tracebloc 1.2.72__tar.gz → 1.2.73__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tracebloc-1.2.72 → tracebloc-1.2.73}/PKG-INFO +1 -1
- {tracebloc-1.2.72 → tracebloc-1.2.73}/pyproject.toml +1 -1
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/acceptance.py +1 -1
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/owncode/assemble.py +12 -2
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/owncode/shake.py +30 -18
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/owncode/source_tree.py +144 -3
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/package.py +17 -8
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/model_upload.py +5 -5
- tracebloc-1.2.73/tracebloc/upload/weights_parts.py +221 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc.egg-info/PKG-INFO +1 -1
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc.egg-info/SOURCES.txt +1 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/MANIFEST.in +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/README.md +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/license.txt +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/setup.cfg +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/__init__.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/_deprecation.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/_framework_utils.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/_http_utils.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/_link_answer.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/_link_regeneration.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/_linking.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/_raw_width_note.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/_stored_link.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/_upload_conversion.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/checks.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/client.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/config.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/contracts/stage_urls.v1.json +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/contracts/stage_urls.v1.source.json +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/contracts/survival_payload.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/contracts/training_requirements.source.json +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/contracts/training_requirements.txt +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/__init__.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/families/__init__.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/families/hf/__init__.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/families/hf/_build.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/families/tabular.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/families/vision/__init__.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/families/vision/_matching.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/gate.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/interfaces.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/licence/__init__.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/licence/spdx.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/outputs.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/owncode/__init__.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/owncode/arguments.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/owncode/checkpoint.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/owncode/gate_rules.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/owncode/generate.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/owncode/head.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/owncode/import_check.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/owncode/layer0.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/owncode/live.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/owncode/model_class.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/owncode/remote.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/owncode/repo.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/report.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/tokenizers.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/convert/training_versions.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/env_login.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/exceptions.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/fetch.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/inference.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/library_fetch.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/lifelines_weights.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/link_model_dataset.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/notebook/__init__.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/notebook/templates/_partials.tpl +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/notebook/templates/embeddings.tpl +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/notebook/templates/nlp_finetune.tpl +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/notebook/templates/nlp_generative.tpl +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/notebook/templates/survival.tpl +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/notebook/templates/tabular_timeseries.tpl +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/notebook/templates/vision_from_scratch.tpl +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/raw_mode.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/raw_mode_frame.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/raw_mode_seed.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/raw_mode_survival.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/raw_mode_torch.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/seed_weights.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/training/__init__.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/training/augmentation.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/training/callbacks.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/training/converted.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/training/features.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/training/image_size.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/training/lora.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/training/lora_validation.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/training/loss.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/training/memory_estimate.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/training/plan.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/training/preprocessing.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/training/shape.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/training/subdataset.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/training/submitter.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/training/summary.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/__init__.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/lifelines_time_to_event.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/scikit_survival_time_to_event.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/sequence_length.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/skl_tabular_base.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/skl_tabular_classifier.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/skl_tabular_regression.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/statistical_base.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/survival_base.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/torch_causal_language_modeler.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/torch_embeddings.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/torch_image_classifier.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/torch_key_point_detector.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/torch_masked_language_modeler.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/torch_nlp_base.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/torch_object_detector.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/torch_semantic_segmentation.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/torch_sentence_pair_classifier.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/torch_seq2seq_modeler.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/torch_tabular_classifier.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/torch_tabular_regression.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/torch_text_classifier.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/torch_time_series_classifier.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/torch_time_series_forecaster.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/torch_time_to_event.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/torch_token_classifier.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/upload/weight_loaders.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/user.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/utils/__init__.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/utils/class_order.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/utils/constants.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/utils/custom_loss_probe.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/utils/detection_utils.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/utils/entry_point.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/utils/feature_dict.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/utils/general_utils.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/utils/key_point_detection_utils.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/utils/registry.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/utils/safetensors_io.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/utils/semantic_segmentation_utils.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/utils/tabular_classification_utils.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/utils/tabular_custom_loss_utils.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/utils/tabular_regression_utils.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/utils/text_classification_utils.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/utils/time_series_utils.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/utils/time_to_event_utils.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/utils/warnings_once.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/validation/__init__.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/validation/checker.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/validation/header_rules.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/validation/lifelines_checks.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/validation/pytorch_checks.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/validation/rewriter.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/validation/scikit_survival_checks.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc/validation/sklearn_checks.py +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc.egg-info/dependency_links.txt +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc.egg-info/requires.txt +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc.egg-info/top_level.txt +0 -0
- {tracebloc-1.2.72 → tracebloc-1.2.73}/tracebloc_package/__init__.py +0 -0
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "tracebloc"
|
|
7
|
-
version = "1.2.
|
|
7
|
+
version = "1.2.73"
|
|
8
8
|
description = "Build AI with anyone. On data that can't move. SDK for the tracebloc collaborative AI workspace."
|
|
9
9
|
license = "MIT"
|
|
10
10
|
license-files = ["license.txt"]
|
|
@@ -115,7 +115,7 @@ def upload_check(package: GeneratedPackage, written: WrittenPackage) -> CheckFin
|
|
|
115
115
|
else:
|
|
116
116
|
problems += _size_problem(model_file, MAX_PY_BYTES, "the model file")
|
|
117
117
|
problems += _size_problem(
|
|
118
|
-
written.weights, package_module.
|
|
118
|
+
written.weights, package_module.weights_in_parts_cap(), "the weights"
|
|
119
119
|
)
|
|
120
120
|
problems += _size_problem(written.tokenizer, MAX_TOKENIZER_BYTES, "the tokenizer")
|
|
121
121
|
pair = (
|
|
@@ -35,6 +35,9 @@ from tracebloc.utils.constants import TaskCategory
|
|
|
35
35
|
#: The name every own-code package records as its family.
|
|
36
36
|
FAMILY = "own_code"
|
|
37
37
|
|
|
38
|
+
#: The class the platform loads from the entry file.
|
|
39
|
+
ENTRY_CLASS = "MyModel"
|
|
40
|
+
|
|
38
41
|
|
|
39
42
|
def require_use_case(task: UseCaseTask) -> None:
|
|
40
43
|
if not task.from_use_case:
|
|
@@ -153,10 +156,17 @@ def build(
|
|
|
153
156
|
)
|
|
154
157
|
resize = (plan.resize, plan.trained) if plan.resize and plan.trained else None
|
|
155
158
|
module = flat_name(state["module"])
|
|
156
|
-
|
|
159
|
+
# The entry file defines ``MyModel`` itself, so a peer class of that name is
|
|
160
|
+
# imported under another one.
|
|
161
|
+
peer = state["class_name"]
|
|
162
|
+
alias = f"Peer{peer}" if peer == ENTRY_CLASS else peer
|
|
163
|
+
imports = [
|
|
164
|
+
f"from {module} import {peer}" + (f" as {alias}" if alias != peer else ""),
|
|
165
|
+
*state.get("imports", ()),
|
|
166
|
+
]
|
|
157
167
|
entry = generate.entry_source(
|
|
158
168
|
imports=imports,
|
|
159
|
-
class_name=
|
|
169
|
+
class_name=alias,
|
|
160
170
|
construct=call,
|
|
161
171
|
adapter=bool(state["adapter"]) and not generate.is_self_supervised(category),
|
|
162
172
|
resize=resize,
|
|
@@ -28,6 +28,8 @@ from tracebloc.convert.owncode.source_tree import (
|
|
|
28
28
|
dotted,
|
|
29
29
|
loaded_names,
|
|
30
30
|
statement_start,
|
|
31
|
+
switch_edits,
|
|
32
|
+
unbound_reads,
|
|
31
33
|
)
|
|
32
34
|
|
|
33
35
|
#: The prefix of every flat file made from one of the peer's modules.
|
|
@@ -115,6 +117,7 @@ class Shaker:
|
|
|
115
117
|
top = set().union(*(bound_names(s) for s in module.tree.body))
|
|
116
118
|
for name in sorted(uses & top):
|
|
117
119
|
self.need(module.name, name)
|
|
120
|
+
self._refuse_local_star(module, stmt, uses - top)
|
|
118
121
|
self._follow_attributes(module, stmt)
|
|
119
122
|
|
|
120
123
|
def _follow_import(self, module: SourceModule, stmt: ast.stmt) -> None:
|
|
@@ -150,6 +153,31 @@ class Shaker:
|
|
|
150
153
|
f"{alias.name.rpartition('.')[0] or alias.name} import ...`",
|
|
151
154
|
)
|
|
152
155
|
|
|
156
|
+
def _refuse_local_star(
|
|
157
|
+
self, module: SourceModule, stmt: ast.stmt, names: set[str]
|
|
158
|
+
) -> None:
|
|
159
|
+
"""A kept statement reading a name only a local ``import *`` supplies, or
|
|
160
|
+
one a local star whose names cannot be listed may supply."""
|
|
161
|
+
for name in sorted(names):
|
|
162
|
+
star = self.tree.local_star(module, name)
|
|
163
|
+
if star is not None:
|
|
164
|
+
raise ConversionRefusedError(
|
|
165
|
+
RefusalCause.NO_SOURCE,
|
|
166
|
+
f"{module.relative}:{star.lineno} imports `*` from "
|
|
167
|
+
f"`{self.tree.absolute(module, star)}` and the model uses "
|
|
168
|
+
f"`{name}` from it; import the names the model uses instead",
|
|
169
|
+
)
|
|
170
|
+
opaque = self.tree.opaque_local_star(module)
|
|
171
|
+
unbound = unbound_reads(stmt, names) if opaque else []
|
|
172
|
+
if unbound:
|
|
173
|
+
raise ConversionRefusedError(
|
|
174
|
+
RefusalCause.NO_SOURCE,
|
|
175
|
+
f"{module.relative}:{opaque.lineno} imports `*` from "
|
|
176
|
+
f"`{self.tree.absolute(module, opaque)}`, whose names cannot be "
|
|
177
|
+
f"listed, and the model uses `{unbound[0]}`; import the names "
|
|
178
|
+
"the model uses instead",
|
|
179
|
+
)
|
|
180
|
+
|
|
153
181
|
def _need_member(self, source: str, member: str) -> None:
|
|
154
182
|
for name in (f"{source}.{member}", source):
|
|
155
183
|
if name in self.tree.unreadable:
|
|
@@ -417,30 +445,13 @@ class Shaker:
|
|
|
417
445
|
)
|
|
418
446
|
for library, switches in DOWNLOAD_SWITCHES.items():
|
|
419
447
|
if name.startswith(library):
|
|
420
|
-
edits +=
|
|
448
|
+
edits += switch_edits(module, node, switches)
|
|
421
449
|
for line, start, end, value, text in sorted(edits, reverse=True):
|
|
422
450
|
lines[line - 1] = lines[line - 1][:start] + value + lines[line - 1][end:]
|
|
423
451
|
self.changes.append(text)
|
|
424
452
|
return lines
|
|
425
453
|
|
|
426
454
|
|
|
427
|
-
def _switch_edits(
|
|
428
|
-
module: SourceModule, call: ast.Call, switches: dict[str, str]
|
|
429
|
-
) -> list:
|
|
430
|
-
edits = []
|
|
431
|
-
for keyword in call.keywords:
|
|
432
|
-
value = keyword.value
|
|
433
|
-
off = switches.get(keyword.arg or "")
|
|
434
|
-
if off is None or value.lineno != value.end_lineno:
|
|
435
|
-
continue
|
|
436
|
-
current = ast.get_source_segment(module.text, value) or ""
|
|
437
|
-
if current in (off, "None") or current == "False":
|
|
438
|
-
continue
|
|
439
|
-
text = f"{module.relative}:{value.lineno} {keyword.arg}={current} -> {off}"
|
|
440
|
-
edits.append((value.lineno, value.col_offset, value.end_col_offset, off, text))
|
|
441
|
-
return edits
|
|
442
|
-
|
|
443
|
-
|
|
444
455
|
def _is_guard(stmt: ast.stmt) -> bool:
|
|
445
456
|
"""A block that never runs when the model is built, with no ``else`` that
|
|
446
457
|
does: ``if __name__ == "__main__":`` or ``if TYPE_CHECKING:``."""
|
|
@@ -461,6 +472,7 @@ def _is_guard(stmt: ast.stmt) -> bool:
|
|
|
461
472
|
|
|
462
473
|
|
|
463
474
|
def _library_star(stmt: ast.stmt) -> bool:
|
|
475
|
+
"""An absolute ``from x import *``."""
|
|
464
476
|
return (
|
|
465
477
|
isinstance(stmt, ast.ImportFrom)
|
|
466
478
|
and not stmt.level
|
|
@@ -8,6 +8,7 @@ statement binds a name. Everything here works on text and ``ast`` alone.
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
10
|
import ast
|
|
11
|
+
import builtins
|
|
11
12
|
import os
|
|
12
13
|
from dataclasses import dataclass
|
|
13
14
|
from pathlib import Path, PurePosixPath
|
|
@@ -98,6 +99,31 @@ def loaded_names(node: ast.AST) -> set[str]:
|
|
|
98
99
|
}
|
|
99
100
|
|
|
100
101
|
|
|
102
|
+
def stored_names(node: ast.AST) -> set[str]:
|
|
103
|
+
"""Every name ``node`` binds below its top level: assignment and loop
|
|
104
|
+
targets, parameters, nested definitions, imports and handler names."""
|
|
105
|
+
found: set[str] = set()
|
|
106
|
+
for n in ast.walk(node):
|
|
107
|
+
if isinstance(n, ast.Name) and not isinstance(n.ctx, ast.Load):
|
|
108
|
+
found.add(n.id)
|
|
109
|
+
elif isinstance(n, ast.arg):
|
|
110
|
+
found.add(n.arg)
|
|
111
|
+
elif isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
112
|
+
found.add(n.name)
|
|
113
|
+
elif isinstance(n, ast.ExceptHandler) and n.name:
|
|
114
|
+
found.add(n.name)
|
|
115
|
+
elif isinstance(n, (ast.Import, ast.ImportFrom)):
|
|
116
|
+
found.update(
|
|
117
|
+
(a.asname or a.name).split(".")[0] for a in n.names if a.name != "*"
|
|
118
|
+
)
|
|
119
|
+
return found
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def unbound_reads(stmt: ast.stmt, names: set[str]) -> list[str]:
|
|
123
|
+
"""The ``names`` ``stmt`` reads that neither it nor the builtins bind."""
|
|
124
|
+
return sorted(n for n in names - stored_names(stmt) if not hasattr(builtins, n))
|
|
125
|
+
|
|
126
|
+
|
|
101
127
|
def dotted(node: ast.AST) -> list[str] | None:
|
|
102
128
|
"""``a.b.c`` as ``["a", "b", "c"]``; ``None`` unless rooted at a name."""
|
|
103
129
|
parts: list[str] = []
|
|
@@ -220,8 +246,14 @@ class SourceTree:
|
|
|
220
246
|
bound[alias.asname or alias.name] = f"{source}.{alias.name}"
|
|
221
247
|
return bound
|
|
222
248
|
|
|
223
|
-
def qualify(
|
|
224
|
-
|
|
249
|
+
def qualify(
|
|
250
|
+
self, module: SourceModule, node: ast.AST, depth: int = RESOLUTION_DEPTH
|
|
251
|
+
) -> str | None:
|
|
252
|
+
"""The dotted name an expression means in ``module``, or ``None``.
|
|
253
|
+
|
|
254
|
+
A name a local star import supplies means what the starred module
|
|
255
|
+
itself means by it, so a library name it only re-exports stays the
|
|
256
|
+
library's."""
|
|
225
257
|
parts = dotted(node)
|
|
226
258
|
if parts is None:
|
|
227
259
|
return None
|
|
@@ -230,11 +262,27 @@ class SourceTree:
|
|
|
230
262
|
return ".".join([bound, *parts[1:]])
|
|
231
263
|
if any(parts[0] in bound_names(stmt) for stmt in module.tree.body):
|
|
232
264
|
return ".".join([module.name, *parts])
|
|
265
|
+
if depth > 0 and (star := self.local_star(module, parts[0])) is not None:
|
|
266
|
+
source = self.modules[self.absolute(module, star)]
|
|
267
|
+
return self.qualify(source, node, depth - 1)
|
|
233
268
|
stars = self.library_stars(module)
|
|
234
269
|
# A name nothing binds may come from a library's star import: read it as
|
|
235
270
|
# that library's, the only reading the source allows.
|
|
236
271
|
return ".".join([stars[0], *parts]) if len(stars) == 1 else None
|
|
237
272
|
|
|
273
|
+
def local_star(self, module: SourceModule, name: str) -> ast.ImportFrom | None:
|
|
274
|
+
"""The local ``from x import *`` in ``module`` that supplies ``name``, the
|
|
275
|
+
last such line (Python binds the last star that has the name); ``None``
|
|
276
|
+
when none does."""
|
|
277
|
+
for node in reversed(module.tree.body):
|
|
278
|
+
if isinstance(node, ast.ImportFrom) and any(
|
|
279
|
+
alias.name == "*" for alias in node.names
|
|
280
|
+
):
|
|
281
|
+
source = self.absolute(module, node)
|
|
282
|
+
if self.is_local(source) and name in self.star_exports(source):
|
|
283
|
+
return node
|
|
284
|
+
return None
|
|
285
|
+
|
|
238
286
|
def library_stars(self, module: SourceModule) -> list[str]:
|
|
239
287
|
"""The libraries ``module`` star-imports (``from torch.nn import *``)."""
|
|
240
288
|
return [
|
|
@@ -285,13 +333,89 @@ class SourceTree:
|
|
|
285
333
|
if name in bound_names(node):
|
|
286
334
|
return module, node
|
|
287
335
|
target = self.bindings(module).get(name)
|
|
288
|
-
if target is None
|
|
336
|
+
if target is None:
|
|
337
|
+
return self._through_stars(module, name, depth)
|
|
338
|
+
if not self.is_local(target):
|
|
289
339
|
return None
|
|
290
340
|
owner, _, member = target.rpartition(".")
|
|
291
341
|
if f"{owner}.{member}" in self.modules:
|
|
292
342
|
return None
|
|
293
343
|
return self.definition(owner, member, depth - 1)
|
|
294
344
|
|
|
345
|
+
def _through_stars(
|
|
346
|
+
self, module: SourceModule, name: str, depth: int
|
|
347
|
+
) -> tuple[SourceModule, ast.stmt] | None:
|
|
348
|
+
"""``name`` as ``module``'s last local ``from x import *`` line that
|
|
349
|
+
supplies it defines it; ``None`` when none does."""
|
|
350
|
+
star = self.local_star(module, name)
|
|
351
|
+
if star is None:
|
|
352
|
+
return None
|
|
353
|
+
return self.definition(self.absolute(module, star), name, depth - 1)
|
|
354
|
+
|
|
355
|
+
def star_exports(self, module_name: str) -> frozenset[str]:
|
|
356
|
+
"""The names ``from <module_name> import *`` binds: a literal ``__all__``,
|
|
357
|
+
else every top-level name without a leading underscore, those its own
|
|
358
|
+
local star imports supply included. Empty when ``__all__`` is built any
|
|
359
|
+
other way (see ``star_unreadable``), so nothing is guessed."""
|
|
360
|
+
return self._star_names(module_name, frozenset()) or frozenset()
|
|
361
|
+
|
|
362
|
+
def star_unreadable(self, module_name: str) -> bool:
|
|
363
|
+
"""Whether ``from <module_name> import *`` binds names this tree cannot
|
|
364
|
+
list: an ``__all__`` built any way but one literal, in the module or in
|
|
365
|
+
a local star it follows."""
|
|
366
|
+
return self._star_names(module_name, frozenset()) is None
|
|
367
|
+
|
|
368
|
+
def opaque_local_star(self, module: SourceModule) -> ast.ImportFrom | None:
|
|
369
|
+
"""The last local ``from x import *`` in ``module`` whose names cannot be
|
|
370
|
+
listed; ``None`` when every local star is readable."""
|
|
371
|
+
for node in reversed(module.tree.body):
|
|
372
|
+
if isinstance(node, ast.ImportFrom) and any(
|
|
373
|
+
alias.name == "*" for alias in node.names
|
|
374
|
+
):
|
|
375
|
+
source = self.absolute(module, node)
|
|
376
|
+
if self.is_local(source) and self.star_unreadable(source):
|
|
377
|
+
return node
|
|
378
|
+
return None
|
|
379
|
+
|
|
380
|
+
def _star_names(
|
|
381
|
+
self, module_name: str, seen: frozenset[str]
|
|
382
|
+
) -> frozenset[str] | None:
|
|
383
|
+
"""``star_exports``, or ``None`` when the names cannot be listed."""
|
|
384
|
+
module = self.modules.get(module_name)
|
|
385
|
+
if module is None or module_name in seen:
|
|
386
|
+
return frozenset()
|
|
387
|
+
touching = [
|
|
388
|
+
node
|
|
389
|
+
for node in module.tree.body
|
|
390
|
+
if "__all__" in bound_names(node) | loaded_names(node)
|
|
391
|
+
]
|
|
392
|
+
if touching:
|
|
393
|
+
node = touching[0]
|
|
394
|
+
if len(touching) > 1 or not isinstance(node, (ast.Assign, ast.AnnAssign)):
|
|
395
|
+
return None
|
|
396
|
+
try:
|
|
397
|
+
listed = ast.literal_eval(node.value) if node.value else None
|
|
398
|
+
except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError):
|
|
399
|
+
return None
|
|
400
|
+
if not isinstance(listed, (list, tuple)) or not all(
|
|
401
|
+
isinstance(item, str) for item in listed
|
|
402
|
+
):
|
|
403
|
+
return None
|
|
404
|
+
return frozenset(listed)
|
|
405
|
+
names: set[str] = set()
|
|
406
|
+
for node in module.tree.body:
|
|
407
|
+
names |= bound_names(node)
|
|
408
|
+
if isinstance(node, ast.ImportFrom) and any(
|
|
409
|
+
alias.name == "*" for alias in node.names
|
|
410
|
+
):
|
|
411
|
+
source = self.absolute(module, node)
|
|
412
|
+
if self.is_local(source):
|
|
413
|
+
nested = self._star_names(source, seen | {module_name})
|
|
414
|
+
if nested is None:
|
|
415
|
+
return None
|
|
416
|
+
names |= nested
|
|
417
|
+
return frozenset(n for n in names if not n.startswith("_") and n != "*")
|
|
418
|
+
|
|
295
419
|
def files(
|
|
296
420
|
self,
|
|
297
421
|
suffixes: tuple[str, ...],
|
|
@@ -313,3 +437,20 @@ class SourceTree:
|
|
|
313
437
|
if name.endswith(suffixes) and (named is None or name == named)
|
|
314
438
|
]
|
|
315
439
|
return sorted(found, key=lambda p: (len(p.relative_to(self.root).parts), p))
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def switch_edits(
|
|
443
|
+
module: SourceModule, call: ast.Call, switches: dict[str, str]
|
|
444
|
+
) -> list:
|
|
445
|
+
edits = []
|
|
446
|
+
for keyword in call.keywords:
|
|
447
|
+
value = keyword.value
|
|
448
|
+
off = switches.get(keyword.arg or "")
|
|
449
|
+
if off is None or value.lineno != value.end_lineno:
|
|
450
|
+
continue
|
|
451
|
+
current = ast.get_source_segment(module.text, value) or ""
|
|
452
|
+
if current in (off, "None") or current == "False":
|
|
453
|
+
continue
|
|
454
|
+
text = f"{module.relative}:{value.lineno} {keyword.arg}={current} -> {off}"
|
|
455
|
+
edits.append((value.lineno, value.col_offset, value.end_col_offset, off, text))
|
|
456
|
+
return edits
|
|
@@ -27,6 +27,7 @@ from dataclasses import dataclass, replace
|
|
|
27
27
|
from pathlib import Path
|
|
28
28
|
from typing import Any, Callable, Mapping
|
|
29
29
|
|
|
30
|
+
from tracebloc.upload.weights_parts import IN_PARTS_CAP_BYTES, SINGLE_REQUEST_CAP_BYTES
|
|
30
31
|
from tracebloc.convert.interfaces import (
|
|
31
32
|
ConversionRefusedError,
|
|
32
33
|
ConverterWiringError,
|
|
@@ -59,10 +60,11 @@ HEADER_ORDER = (
|
|
|
59
60
|
|
|
60
61
|
GIB = 1024**3
|
|
61
62
|
|
|
62
|
-
#:
|
|
63
|
-
|
|
64
|
-
#:
|
|
65
|
-
|
|
63
|
+
#: Weights over one check-model request's cap ship in bf16 (decided 2026-10-28).
|
|
64
|
+
WEIGHTS_CAP_BYTES = SINGLE_REQUEST_CAP_BYTES
|
|
65
|
+
#: The most weights the backend takes at all, sent in parts
|
|
66
|
+
#: (``tracebloc.upload.weights_parts``): over it in bf16 is a hard stop.
|
|
67
|
+
WEIGHTS_IN_PARTS_CAP_BYTES = IN_PARTS_CAP_BYTES
|
|
66
68
|
|
|
67
69
|
#: The two class counts the head is derived with. Neither is 1 or 2, which
|
|
68
70
|
#: several heads special-case, nor a usual default that could equal the
|
|
@@ -410,6 +412,11 @@ def weights_cap() -> int:
|
|
|
410
412
|
return WEIGHTS_CAP_BYTES
|
|
411
413
|
|
|
412
414
|
|
|
415
|
+
def weights_in_parts_cap() -> int:
|
|
416
|
+
"""The most weights the backend takes, read when asked so a test can lower it."""
|
|
417
|
+
return WEIGHTS_IN_PARTS_CAP_BYTES
|
|
418
|
+
|
|
419
|
+
|
|
413
420
|
def _gib(size: int) -> str:
|
|
414
421
|
return f"{size / GIB:.2f} GiB" if size >= GIB / 100 else f"{size} bytes"
|
|
415
422
|
|
|
@@ -419,8 +426,9 @@ def ship_within_weights_cap(
|
|
|
419
426
|
) -> GeneratedPackage:
|
|
420
427
|
"""Ship the weights in bf16 when in fp32 they are over the weights cap.
|
|
421
428
|
|
|
422
|
-
Still over
|
|
423
|
-
bit, so an exact proof becomes a
|
|
429
|
+
Still over the most the backend takes (sent in parts) in bf16 is a hard
|
|
430
|
+
stop. bf16 weights no longer match bit for bit, so an exact proof becomes a
|
|
431
|
+
tolerance one.
|
|
424
432
|
"""
|
|
425
433
|
weights, cap = written.weights, weights_cap()
|
|
426
434
|
if weights is None or weights.suffix != ".safetensors":
|
|
@@ -428,11 +436,12 @@ def ship_within_weights_cap(
|
|
|
428
436
|
size = weights.stat().st_size
|
|
429
437
|
if size <= cap:
|
|
430
438
|
return package
|
|
431
|
-
|
|
439
|
+
limit = weights_in_parts_cap()
|
|
440
|
+
if cast_weights_to_bf16(weights) > limit:
|
|
432
441
|
raise ConversionRefusedError(
|
|
433
442
|
RefusalCause.SIZE_CAP,
|
|
434
443
|
f"the {package.family} weights are {_gib(size)} in fp32 and still over "
|
|
435
|
-
f"the {_gib(
|
|
444
|
+
f"the {_gib(limit)} weights limit in bf16; bring a smaller model",
|
|
436
445
|
)
|
|
437
446
|
note = (
|
|
438
447
|
f"the weights ship in bf16: in fp32 they are {_gib(size)}, over the "
|
|
@@ -40,6 +40,7 @@ from tracebloc.utils.safetensors_io import (
|
|
|
40
40
|
load_torch_safetensors,
|
|
41
41
|
read_weights_envelope,
|
|
42
42
|
)
|
|
43
|
+
from tracebloc.upload.weights_parts import attach_weights
|
|
43
44
|
from tracebloc.upload.weight_loaders import (
|
|
44
45
|
load_seed_state_dict,
|
|
45
46
|
load_torch_state_dict,
|
|
@@ -258,16 +259,15 @@ class Model:
|
|
|
258
259
|
self.progress_bar_2 = tqdm(total=1, desc="Model Upload Progress")
|
|
259
260
|
model_file = open(upload_file_for(self.__model_path, self.conversion), "rb")
|
|
260
261
|
files = {"upload_file": model_file}
|
|
261
|
-
if self.weights:
|
|
262
|
-
# Raises ModelValidationError on any weights-load failure.
|
|
263
|
-
self.check_weights()
|
|
264
|
-
model_weights = open(self.weights_path, "rb")
|
|
265
|
-
files["upload_weights"] = model_weights
|
|
266
262
|
values = {
|
|
267
263
|
"model_name": self.model_name,
|
|
268
264
|
"set_weights": bool(self.weights),
|
|
269
265
|
"type": "functional_test",
|
|
270
266
|
}
|
|
267
|
+
if self.weights:
|
|
268
|
+
# Raises ModelValidationError on any weights-load failure.
|
|
269
|
+
self.check_weights()
|
|
270
|
+
model_weights = attach_weights(self._client, self.weights_path, files, values)
|
|
271
271
|
# Attach a contributor-supplied tokenizer.json as a model artifact
|
|
272
272
|
# (issue #748). The backend stores it against the model and ships
|
|
273
273
|
# it to every client with the weights; ``upload_tokenizer`` mirrors
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
"""Weights over the platform's single-request cap, sent in numbered parts.
|
|
2
|
+
|
|
3
|
+
``check-model/`` takes weights of up to :data:`SINGLE_REQUEST_CAP_BYTES` in its
|
|
4
|
+
own request. Larger ones, up to :data:`IN_PARTS_CAP_BYTES`, go to
|
|
5
|
+
``check-model/weights-uploads/`` first: the upload is started with the file's
|
|
6
|
+
size and SHA-256, each part is sent with its own SHA-256, and once the platform
|
|
7
|
+
has checked the whole file ``check-model/`` names the upload as
|
|
8
|
+
``weights_upload`` instead of carrying the file.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import hashlib
|
|
14
|
+
import os
|
|
15
|
+
import time
|
|
16
|
+
from typing import IO, Any, Callable, MutableMapping
|
|
17
|
+
|
|
18
|
+
from tqdm import tqdm
|
|
19
|
+
|
|
20
|
+
from tracebloc._http_utils import _extract_error_message
|
|
21
|
+
from tracebloc.exceptions import APIError
|
|
22
|
+
from tracebloc.utils.general_utils import format_exception
|
|
23
|
+
|
|
24
|
+
#: The platform's caps (``CHECK_MODEL_MAX_WEIGHTS_BYTES`` and
|
|
25
|
+
#: ``CHECK_MODEL_MAX_WEIGHTS_BYTES_IN_PARTS``). It publishes them in no
|
|
26
|
+
#: contract, so they are restated: a drift is a refusal from the platform.
|
|
27
|
+
GIB = 1024**3
|
|
28
|
+
SINGLE_REQUEST_CAP_BYTES = 2 * GIB
|
|
29
|
+
IN_PARTS_CAP_BYTES = 7 * GIB
|
|
30
|
+
|
|
31
|
+
_UPLOADS = "check-model/weights-uploads/"
|
|
32
|
+
_HASH_CHUNK = 8 * 1024 * 1024
|
|
33
|
+
#: A part is at most 64 MiB; at a slow 1 MB/s that is about a minute.
|
|
34
|
+
_PART_TIMEOUT_SECONDS = 300
|
|
35
|
+
_PART_ATTEMPTS = 3
|
|
36
|
+
#: The platform reads the whole file once to check it (minutes at 7 GiB).
|
|
37
|
+
_VERIFY_TIMEOUT_SECONDS = 30 * 60
|
|
38
|
+
_POLL_SECONDS = (1, 2, 5, 10)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class WeightsUploadError(Exception):
|
|
42
|
+
"""The weights could not be sent in parts; the message says why."""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def needs_parts(path: str | os.PathLike[str]) -> bool:
|
|
46
|
+
"""Whether the weights at ``path`` are over one request's cap."""
|
|
47
|
+
return os.path.getsize(path) > SINGLE_REQUEST_CAP_BYTES
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _sha256_of(path: str | os.PathLike[str]) -> str:
|
|
51
|
+
digest = hashlib.sha256()
|
|
52
|
+
with open(path, "rb") as handle:
|
|
53
|
+
for chunk in iter(lambda: handle.read(_HASH_CHUNK), b""):
|
|
54
|
+
digest.update(chunk)
|
|
55
|
+
return digest.hexdigest()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _json_object(response: Any, *keys: str) -> dict[str, Any]:
|
|
59
|
+
"""A 2xx answer's JSON object carrying ``keys``; a
|
|
60
|
+
:class:`WeightsUploadError` when the platform's answer is anything else."""
|
|
61
|
+
try:
|
|
62
|
+
body = response.json()
|
|
63
|
+
except ValueError as error:
|
|
64
|
+
raise WeightsUploadError(
|
|
65
|
+
f"the platform's answer could not be read: {format_exception(error)}"
|
|
66
|
+
) from error
|
|
67
|
+
if not isinstance(body, dict) or any(key not in body for key in keys):
|
|
68
|
+
raise WeightsUploadError(
|
|
69
|
+
"the platform's answer is not the one expected "
|
|
70
|
+
f"(missing {', '.join(keys) or 'a JSON object'})"
|
|
71
|
+
)
|
|
72
|
+
return body
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _cancel(client: Any, upload_id: str) -> None:
|
|
76
|
+
"""Best-effort delete of the upload; the failure that led here is the one
|
|
77
|
+
to report."""
|
|
78
|
+
try:
|
|
79
|
+
client.delete(f"{_UPLOADS}{upload_id}/", raise_on_error=False)
|
|
80
|
+
except Exception:
|
|
81
|
+
pass
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _start(client: Any, path: str | os.PathLike[str]) -> dict[str, Any]:
|
|
85
|
+
size = os.path.getsize(path)
|
|
86
|
+
if size > IN_PARTS_CAP_BYTES:
|
|
87
|
+
raise WeightsUploadError(
|
|
88
|
+
f"the weights are {size / GIB:.2f} GiB, over the "
|
|
89
|
+
f"{IN_PARTS_CAP_BYTES / GIB:.0f} GiB limit"
|
|
90
|
+
)
|
|
91
|
+
response = client.post(
|
|
92
|
+
_UPLOADS,
|
|
93
|
+
json={
|
|
94
|
+
"size_bytes": size,
|
|
95
|
+
"sha256": _sha256_of(path),
|
|
96
|
+
"file_name": os.path.basename(os.fspath(path)),
|
|
97
|
+
},
|
|
98
|
+
raise_on_error=False,
|
|
99
|
+
)
|
|
100
|
+
if response.status_code == 404:
|
|
101
|
+
raise WeightsUploadError(
|
|
102
|
+
f"this tracebloc platform takes weights of up to "
|
|
103
|
+
f"{SINGLE_REQUEST_CAP_BYTES / GIB:.0f} GiB; these are {size / GIB:.2f} GiB"
|
|
104
|
+
)
|
|
105
|
+
if response.status_code != 201:
|
|
106
|
+
raise WeightsUploadError(
|
|
107
|
+
f"the upload was refused: {_extract_error_message(response)}"
|
|
108
|
+
)
|
|
109
|
+
try:
|
|
110
|
+
return _json_object(response, "id", "part_bytes", "part_count")
|
|
111
|
+
except WeightsUploadError:
|
|
112
|
+
# The platform created the upload; cancel it if its answer names it.
|
|
113
|
+
try:
|
|
114
|
+
created = response.json()
|
|
115
|
+
except ValueError:
|
|
116
|
+
created = None
|
|
117
|
+
if isinstance(created, dict) and "id" in created:
|
|
118
|
+
_cancel(client, created["id"])
|
|
119
|
+
raise
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _send_part(client: Any, upload_id: str, number: int, data: bytes) -> None:
|
|
123
|
+
sha256 = hashlib.sha256(data).hexdigest()
|
|
124
|
+
for attempt in range(1, _PART_ATTEMPTS + 1):
|
|
125
|
+
try:
|
|
126
|
+
response = client.put(
|
|
127
|
+
f"{_UPLOADS}{upload_id}/parts/{number}/",
|
|
128
|
+
files={"part": ("part", data)},
|
|
129
|
+
data={"sha256": sha256},
|
|
130
|
+
raise_on_error=False,
|
|
131
|
+
timeout=_PART_TIMEOUT_SECONDS,
|
|
132
|
+
)
|
|
133
|
+
except APIError as error:
|
|
134
|
+
if attempt == _PART_ATTEMPTS:
|
|
135
|
+
raise WeightsUploadError(
|
|
136
|
+
f"part {number} was not sent: {format_exception(error)}"
|
|
137
|
+
) from error
|
|
138
|
+
continue
|
|
139
|
+
if response.status_code == 200:
|
|
140
|
+
return
|
|
141
|
+
if response.status_code < 500 or attempt == _PART_ATTEMPTS:
|
|
142
|
+
raise WeightsUploadError(
|
|
143
|
+
f"part {number} was refused: {_extract_error_message(response)}"
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _wait_until_checked(
|
|
148
|
+
client: Any, upload_id: str, sleep: Callable[[float], None]
|
|
149
|
+
) -> None:
|
|
150
|
+
deadline = time.monotonic() + _VERIFY_TIMEOUT_SECONDS
|
|
151
|
+
polls = 0
|
|
152
|
+
while time.monotonic() < deadline:
|
|
153
|
+
try:
|
|
154
|
+
response = client.get(f"{_UPLOADS}{upload_id}/", raise_on_error=False)
|
|
155
|
+
except APIError as error:
|
|
156
|
+
raise WeightsUploadError(
|
|
157
|
+
f"the check could not be followed: {format_exception(error)}"
|
|
158
|
+
) from error
|
|
159
|
+
if not 200 <= response.status_code < 300:
|
|
160
|
+
raise WeightsUploadError(
|
|
161
|
+
f"the check could not be followed: {_extract_error_message(response)}"
|
|
162
|
+
)
|
|
163
|
+
upload = _json_object(response, "state")
|
|
164
|
+
if upload["state"] == "ready":
|
|
165
|
+
return
|
|
166
|
+
if upload["state"] != "verifying":
|
|
167
|
+
raise WeightsUploadError(
|
|
168
|
+
f"the platform did not take the weights: {upload.get('reason') or upload['state']}"
|
|
169
|
+
)
|
|
170
|
+
sleep(_POLL_SECONDS[min(polls, len(_POLL_SECONDS) - 1)])
|
|
171
|
+
polls += 1
|
|
172
|
+
raise WeightsUploadError("the platform did not finish checking the weights in time")
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def send_in_parts(
|
|
176
|
+
client: Any,
|
|
177
|
+
path: str | os.PathLike[str],
|
|
178
|
+
progress: Callable[[int], None] = lambda _sent: None,
|
|
179
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
180
|
+
) -> str:
|
|
181
|
+
"""Send the weights at ``path`` in parts and return the upload's id, which
|
|
182
|
+
``check-model/`` takes as ``weights_upload`` once the platform has checked
|
|
183
|
+
the whole file. ``progress`` is called with each part's byte count."""
|
|
184
|
+
upload = _start(client, path)
|
|
185
|
+
upload_id, part_bytes = upload["id"], upload["part_bytes"]
|
|
186
|
+
try:
|
|
187
|
+
with open(path, "rb") as handle:
|
|
188
|
+
for number in range(1, upload["part_count"] + 1):
|
|
189
|
+
data = handle.read(part_bytes)
|
|
190
|
+
_send_part(client, upload_id, number, data)
|
|
191
|
+
progress(len(data))
|
|
192
|
+
response = client.post(f"{_UPLOADS}{upload_id}/finish/", raise_on_error=False)
|
|
193
|
+
if response.status_code != 202:
|
|
194
|
+
raise WeightsUploadError(
|
|
195
|
+
f"the parts could not be finished: {_extract_error_message(response)}"
|
|
196
|
+
)
|
|
197
|
+
_wait_until_checked(client, upload_id, sleep)
|
|
198
|
+
except BaseException:
|
|
199
|
+
_cancel(client, upload_id)
|
|
200
|
+
raise
|
|
201
|
+
return upload_id
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def attach_weights(
|
|
205
|
+
client: Any,
|
|
206
|
+
path: str | os.PathLike[str],
|
|
207
|
+
files: MutableMapping[str, Any],
|
|
208
|
+
values: MutableMapping[str, Any],
|
|
209
|
+
) -> IO[bytes] | None:
|
|
210
|
+
"""Put the weights at ``path`` on a ``check-model/`` request: as its
|
|
211
|
+
``upload_weights`` file within one request's cap, else sent in parts first
|
|
212
|
+
and named as ``weights_upload``. Returns the file it opened, for the caller
|
|
213
|
+
to close."""
|
|
214
|
+
if not needs_parts(path):
|
|
215
|
+
handle = open(path, "rb")
|
|
216
|
+
files["upload_weights"] = handle
|
|
217
|
+
return handle
|
|
218
|
+
size = os.path.getsize(path)
|
|
219
|
+
with tqdm(total=size, unit="B", unit_scale=True, desc="Weights upload") as bar:
|
|
220
|
+
values["weights_upload"] = send_in_parts(client, path, progress=bar.update)
|
|
221
|
+
return None
|
|
@@ -123,6 +123,7 @@ tracebloc/upload/torch_time_series_forecaster.py
|
|
|
123
123
|
tracebloc/upload/torch_time_to_event.py
|
|
124
124
|
tracebloc/upload/torch_token_classifier.py
|
|
125
125
|
tracebloc/upload/weight_loaders.py
|
|
126
|
+
tracebloc/upload/weights_parts.py
|
|
126
127
|
tracebloc/utils/__init__.py
|
|
127
128
|
tracebloc/utils/class_order.py
|
|
128
129
|
tracebloc/utils/constants.py
|
|
File without changes
|