tracebloc 1.2.74__tar.gz → 1.2.76__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tracebloc-1.2.74 → tracebloc-1.2.76}/PKG-INFO +1 -1
- {tracebloc-1.2.74 → tracebloc-1.2.76}/pyproject.toml +1 -1
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/_upload_conversion.py +2 -58
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/acceptance.py +14 -4
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/families/hf/__init__.py +15 -20
- tracebloc-1.2.76/tracebloc/convert/families/hf/_model_type.py +73 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/interfaces.py +5 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/__init__.py +2 -1
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/assemble.py +2 -1
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/live.py +18 -5
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/package.py +23 -2
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/training_versions.py +28 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc.egg-info/PKG-INFO +1 -1
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc.egg-info/SOURCES.txt +1 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/MANIFEST.in +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/README.md +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/license.txt +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/setup.cfg +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/__init__.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/_deprecation.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/_framework_utils.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/_http_utils.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/_link_answer.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/_link_regeneration.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/_linking.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/_raw_width_note.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/_stored_link.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/checks.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/client.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/config.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/contracts/stage_urls.v1.json +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/contracts/stage_urls.v1.source.json +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/contracts/survival_payload.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/contracts/training_requirements.source.json +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/contracts/training_requirements.txt +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/__init__.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/families/__init__.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/families/hf/_build.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/families/tabular.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/families/vision/__init__.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/families/vision/_matching.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/gate.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/licence/__init__.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/licence/spdx.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/outputs.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/arguments.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/checkpoint.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/gate_rules.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/generate.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/head.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/import_check.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/layer0.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/model_class.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/remote.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/repo.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/shake.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/source_tree.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/report.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/tokenizers.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/env_login.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/exceptions.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/fetch.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/inference.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/library_fetch.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/lifelines_weights.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/link_model_dataset.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/notebook/__init__.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/notebook/templates/_partials.tpl +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/notebook/templates/embeddings.tpl +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/notebook/templates/nlp_finetune.tpl +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/notebook/templates/nlp_generative.tpl +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/notebook/templates/survival.tpl +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/notebook/templates/tabular_timeseries.tpl +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/notebook/templates/vision_from_scratch.tpl +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/raw_mode.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/raw_mode_frame.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/raw_mode_seed.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/raw_mode_survival.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/raw_mode_torch.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/seed_weights.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/__init__.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/augmentation.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/callbacks.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/converted.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/features.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/image_size.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/lora.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/lora_validation.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/loss.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/memory_estimate.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/plan.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/preprocessing.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/shape.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/subdataset.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/submitter.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/summary.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/__init__.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/lifelines_time_to_event.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/model_upload.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/scikit_survival_time_to_event.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/sequence_length.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/skl_tabular_base.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/skl_tabular_classifier.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/skl_tabular_regression.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/statistical_base.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/survival_base.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_causal_language_modeler.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_embeddings.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_image_classifier.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_key_point_detector.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_masked_language_modeler.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_nlp_base.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_object_detector.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_semantic_segmentation.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_sentence_pair_classifier.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_seq2seq_modeler.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_tabular_classifier.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_tabular_regression.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_text_classifier.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_time_series_classifier.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_time_series_forecaster.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_time_to_event.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_token_classifier.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/weight_loaders.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/weights_parts.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/user.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/__init__.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/class_order.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/constants.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/custom_loss_probe.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/detection_utils.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/entry_point.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/feature_dict.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/general_utils.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/key_point_detection_utils.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/registry.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/safetensors_io.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/semantic_segmentation_utils.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/tabular_classification_utils.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/tabular_custom_loss_utils.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/tabular_regression_utils.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/text_classification_utils.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/time_series_utils.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/time_to_event_utils.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/warnings_once.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/validation/__init__.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/validation/checker.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/validation/header_rules.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/validation/lifelines_checks.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/validation/pytorch_checks.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/validation/rewriter.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/validation/scikit_survival_checks.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/validation/sklearn_checks.py +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc.egg-info/dependency_links.txt +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc.egg-info/requires.txt +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc.egg-info/top_level.txt +0 -0
- {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc_package/__init__.py +0 -0
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "tracebloc"
|
|
7
|
-
version = "1.2.
|
|
7
|
+
version = "1.2.76"
|
|
8
8
|
description = "Build AI with anyone. On data that can't move. SDK for the tracebloc collaborative AI workspace."
|
|
9
9
|
license = "MIT"
|
|
10
10
|
license-files = ["license.txt"]
|
|
@@ -5,7 +5,7 @@ untouched. Anything else (a Hugging Face folder, a checkpoint, the peer's own
|
|
|
5
5
|
code, a live object) goes through the converter, whose package is uploaded
|
|
6
6
|
only once its acceptance check passed and its licence is confirmed. The use
|
|
7
7
|
case's task comes from ``dataset_id``: ``dataset/<dataset_id>/task/`` answers
|
|
8
|
-
it for a dataset the peer can link a model to.
|
|
8
|
+
it for a dataset the peer can link a model to, and its 404 refuses any other.
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
11
|
from __future__ import annotations
|
|
@@ -32,12 +32,8 @@ from tracebloc.validation.header_rules import NO_OUTPUT_CLASSES
|
|
|
32
32
|
|
|
33
33
|
#: A dataset's category and counts, for a dataset the peer can link a model to.
|
|
34
34
|
DATASET_TASK_PATH = "dataset/{dataset_id}/task/"
|
|
35
|
-
#: The use cases the peer can see, each with its training dataset.
|
|
36
|
-
USE_CASES_PATH = "competition/"
|
|
37
35
|
#: ``licence_confirmed``'s default: ask the peer, in a notebook only.
|
|
38
36
|
ASK = "ask"
|
|
39
|
-
#: How many pages of use cases are read before giving up on a dataset.
|
|
40
|
-
MAX_PAGES = 20
|
|
41
37
|
|
|
42
38
|
|
|
43
39
|
@dataclass(frozen=True)
|
|
@@ -83,67 +79,15 @@ def _not_found(dataset_id: str) -> ModelValidationError:
|
|
|
83
79
|
)
|
|
84
80
|
|
|
85
81
|
|
|
86
|
-
def _route_missing(response: Any) -> bool:
|
|
87
|
-
"""True for a 404 the server's URL routing answered rather than the view:
|
|
88
|
-
the API's own 404 is JSON ``{"detail": ...}``, an unknown URL's page is not."""
|
|
89
|
-
try:
|
|
90
|
-
body = response.json()
|
|
91
|
-
except ValueError:
|
|
92
|
-
return True
|
|
93
|
-
return not (isinstance(body, Mapping) and "detail" in body)
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
def _class_count(dataset: Mapping[str, Any]) -> int | None:
|
|
97
|
-
meta = dataset.get("dataset_meta")
|
|
98
|
-
counts = meta.get("count") if isinstance(meta, Mapping) else None
|
|
99
|
-
labels = counts.get("label") if isinstance(counts, Mapping) else None
|
|
100
|
-
return len(labels) if isinstance(labels, (Mapping, list)) and labels else None
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
def _use_cases(client: Any) -> Any:
|
|
104
|
-
"""Every use case ``competition/`` lists for the peer, page by page."""
|
|
105
|
-
for page in range(1, MAX_PAGES + 1):
|
|
106
|
-
response = client.get(
|
|
107
|
-
USE_CASES_PATH, params={"page": page}, raise_on_error=False
|
|
108
|
-
)
|
|
109
|
-
if response.status_code != 200:
|
|
110
|
-
raise APIError(response.status_code, "the use cases could not be read")
|
|
111
|
-
body = _json(response, "the use cases")
|
|
112
|
-
yield from body.get("results", []) if isinstance(body, Mapping) else body
|
|
113
|
-
if not isinstance(body, Mapping) or not body.get("next"):
|
|
114
|
-
return
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
def _task_from_use_cases(client: Any, dataset_id: str) -> UseCaseTask:
|
|
118
|
-
"""The task of the use case whose training dataset is ``dataset_id``, for a
|
|
119
|
-
server without ``dataset/<dataset_id>/task/``: its category, and its class
|
|
120
|
-
count when the dataset records its labels."""
|
|
121
|
-
for use_case in _use_cases(client):
|
|
122
|
-
dataset = use_case.get("dataset") or {}
|
|
123
|
-
if dataset_id not in (use_case.get("dataset_key"), dataset.get("dataset_key")):
|
|
124
|
-
continue
|
|
125
|
-
category = _category(dataset, dataset_id)
|
|
126
|
-
classes = _class_count(dataset)
|
|
127
|
-
counts = {}
|
|
128
|
-
if classes is not None and category not in NO_OUTPUT_CLASSES:
|
|
129
|
-
counts["output_classes"] = classes
|
|
130
|
-
return UseCaseTask(category, from_use_case=True, counts=counts)
|
|
131
|
-
raise _not_found(dataset_id)
|
|
132
|
-
|
|
133
|
-
|
|
134
82
|
def dataset_task(client: Any, dataset_id: str) -> UseCaseTask:
|
|
135
83
|
"""The task of ``dataset_id``, a dataset the peer can link a model to.
|
|
136
84
|
|
|
137
85
|
Its category, and every count the server records for it; a class count
|
|
138
|
-
only for a task with a class per label.
|
|
86
|
+
only for a task with a class per label. A 404 is the dataset's refusal.
|
|
139
87
|
"""
|
|
140
88
|
path = DATASET_TASK_PATH.format(dataset_id=quote(str(dataset_id), safe=""))
|
|
141
89
|
response = client.get(path, raise_on_error=False)
|
|
142
90
|
if response.status_code == 404:
|
|
143
|
-
if _route_missing(response):
|
|
144
|
-
# TODO(tracebloc/tracebloc-py-package#1059): a server older than
|
|
145
|
-
# dataset/<key>/task/ answers the task from its use cases.
|
|
146
|
-
return _task_from_use_cases(client, dataset_id)
|
|
147
91
|
raise _not_found(dataset_id)
|
|
148
92
|
what = f"the task of dataset '{dataset_id}'"
|
|
149
93
|
if response.status_code != 200:
|
|
@@ -21,6 +21,7 @@ from typing import Any, Iterator, Sequence
|
|
|
21
21
|
|
|
22
22
|
from tracebloc.convert import outputs, training_versions
|
|
23
23
|
from tracebloc.convert.interfaces import (
|
|
24
|
+
OWN_CODE_FAMILY,
|
|
24
25
|
CheckFinding,
|
|
25
26
|
ConverterWiringError,
|
|
26
27
|
Family,
|
|
@@ -127,8 +128,13 @@ def upload_check(package: GeneratedPackage, written: WrittenPackage) -> CheckFin
|
|
|
127
128
|
return _finding("upload", problems, "within the backend's caps, with a handler")
|
|
128
129
|
|
|
129
130
|
|
|
130
|
-
def build_model(
|
|
131
|
-
|
|
131
|
+
def build_model(
|
|
132
|
+
written: WrittenPackage, workdir: Path, own_code: bool = False
|
|
133
|
+
) -> tuple[Any, CheckFinding]:
|
|
134
|
+
"""``MyModel()`` from the written package, seeded; ``None`` when it cannot be built.
|
|
135
|
+
|
|
136
|
+
The peer's ``own_code`` raising is its refusal, :func:`package.code_failure`.
|
|
137
|
+
"""
|
|
132
138
|
try:
|
|
133
139
|
module = load_entry_module(written, workdir)
|
|
134
140
|
if not callable(getattr(module, "MyModel", None)):
|
|
@@ -138,6 +144,8 @@ def build_model(written: WrittenPackage, workdir: Path) -> tuple[Any, CheckFindi
|
|
|
138
144
|
with _seeded():
|
|
139
145
|
model = module.MyModel()
|
|
140
146
|
except Exception as error: # noqa: BLE001 - the check reports, never raises
|
|
147
|
+
if own_code and not isinstance(error, ConverterWiringError):
|
|
148
|
+
raise package_module.code_failure(written, error) from error
|
|
141
149
|
return None, CheckFinding(
|
|
142
150
|
"build", False, f"MyModel() raised {_one_line(error)}"
|
|
143
151
|
)
|
|
@@ -354,7 +362,8 @@ def findings(
|
|
|
354
362
|
|
|
355
363
|
Refuses first, before anything is built, when a training library the
|
|
356
364
|
package imports is installed here at another version than the training
|
|
357
|
-
images pin
|
|
365
|
+
images pin; it refuses the peer's own code, too, when ``MyModel()`` raises.
|
|
366
|
+
Then: today's header check, the backend's upload checks,
|
|
358
367
|
``MyModel()`` built, its parameter count, the shipped weights' strict load,
|
|
359
368
|
the output against the original that ``family``'s ``build_original``
|
|
360
369
|
builds (shape-only when it builds none), and the token ids for a package
|
|
@@ -366,7 +375,8 @@ def findings(
|
|
|
366
375
|
upload_check(package, written),
|
|
367
376
|
]
|
|
368
377
|
with scratch() as raw:
|
|
369
|
-
|
|
378
|
+
own_code = package.family == OWN_CODE_FAMILY
|
|
379
|
+
model, built = build_model(written, Path(raw), own_code)
|
|
370
380
|
found.append(built)
|
|
371
381
|
if model is not None:
|
|
372
382
|
found += model_findings(recognition, package, written, model, family)
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
"""Hugging Face transformers: the use case names the task, any checkpoint fine-tunes.
|
|
2
2
|
|
|
3
3
|
A folder is recognised by its ``config.json`` ``model_type``, read without
|
|
4
|
-
running anything
|
|
4
|
+
running anything; a config saved without one names it through its
|
|
5
|
+
``architectures``, which transformers' own mappings resolve. A config with an
|
|
6
|
+
``auto_map`` ships its own model code, so
|
|
5
7
|
the folder is the own-code converter's, not this family's. A sentence-transformers
|
|
6
8
|
folder is the transformer its ``modules.json`` names. A folder saved for
|
|
7
9
|
EvolutionaryScale's ``esm`` package is refused, naming ESM C's transformers checkpoint.
|
|
@@ -23,6 +25,8 @@ from typing import Any, Mapping
|
|
|
23
25
|
|
|
24
26
|
from tracebloc.convert.families.hf._build import head_prefixes, load, model_source
|
|
25
27
|
from tracebloc.convert.families.hf._build import write_weights
|
|
28
|
+
from tracebloc.convert.families.hf._model_type import _AUTO, CONFIG_FILE, _mapping
|
|
29
|
+
from tracebloc.convert.families.hf._model_type import _names, model_type_of
|
|
26
30
|
from tracebloc.convert.families.vision import read_normalisation
|
|
27
31
|
from tracebloc.convert.interfaces import ComparedOutput, ConversionInput, Proof
|
|
28
32
|
from tracebloc.convert.interfaces import ConversionRefusedError, RefusalCause
|
|
@@ -33,7 +37,6 @@ from tracebloc.utils.constants import Framework, TaskCategory
|
|
|
33
37
|
from tracebloc.validation.header_rules import NO_OUTPUT_CLASSES, SELF_SUPERVISED_NLP
|
|
34
38
|
from tracebloc.validation.header_rules import SHAPE_VALUES
|
|
35
39
|
|
|
36
|
-
CONFIG_FILE = "config.json"
|
|
37
40
|
#: A sentence-transformers folder's module list.
|
|
38
41
|
ST_MODULES = "modules.json"
|
|
39
42
|
#: What transformers loads a folder's PyTorch weights from, shards included.
|
|
@@ -51,8 +54,6 @@ ESM_PACKAGE_WEIGHTS = "data/weights/esm*.pth"
|
|
|
51
54
|
#: ESM C as transformers reads it (``model_type`` ``esmc``).
|
|
52
55
|
ESMC_CHECKPOINTS = "biohub/ESMC-300M-hf, ESMC-600M-hf or ESMC-6B-hf"
|
|
53
56
|
|
|
54
|
-
_AUTO = "transformers.models.auto"
|
|
55
|
-
|
|
56
57
|
#: The Auto class each task is built with, and the mapping in
|
|
57
58
|
#: ``transformers.models.auto.modeling_auto`` naming its class per
|
|
58
59
|
#: ``model_type``. Earlier rows win as a checkpoint's declared head.
|
|
@@ -135,17 +136,6 @@ def _read_json(path: Path) -> Any:
|
|
|
135
136
|
return None
|
|
136
137
|
|
|
137
138
|
|
|
138
|
-
def _names(value: Any) -> tuple[str, ...]:
|
|
139
|
-
"""A mapping's class names for one ``model_type``: one name or several."""
|
|
140
|
-
if isinstance(value, str):
|
|
141
|
-
return (value,)
|
|
142
|
-
return tuple(value) if isinstance(value, (tuple, list)) else ()
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
def _mapping(name: str) -> Mapping[str, Any]:
|
|
146
|
-
return getattr(importlib.import_module(f"{_AUTO}.modeling_auto"), name, {})
|
|
147
|
-
|
|
148
|
-
|
|
149
139
|
def _modality(model_type: str) -> str | None:
|
|
150
140
|
"""``"text"`` or ``"images"``, from transformers' tokenizer and image-processor
|
|
151
141
|
mappings; ``None`` for a model with both or neither."""
|
|
@@ -350,9 +340,8 @@ class HuggingFaceFamily:
|
|
|
350
340
|
return None
|
|
351
341
|
folder, pooling = _sentence_transformer(path)
|
|
352
342
|
config = _read_json(folder / CONFIG_FILE)
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
):
|
|
343
|
+
model_type = model_type_of(folder, config)
|
|
344
|
+
if model_type is None:
|
|
356
345
|
_refuse_esm_package_layout(folder)
|
|
357
346
|
return None
|
|
358
347
|
if "auto_map" in config:
|
|
@@ -360,7 +349,10 @@ class HuggingFaceFamily:
|
|
|
360
349
|
from tracebloc._framework_utils import require_module
|
|
361
350
|
|
|
362
351
|
transformers = require_module("transformers", "converting a Hugging Face model")
|
|
363
|
-
|
|
352
|
+
read_from = CONFIG_FILE
|
|
353
|
+
if config.get("model_type") != model_type:
|
|
354
|
+
read_from = f"{CONFIG_FILE} architectures"
|
|
355
|
+
config = {**config, "model_type": model_type}
|
|
364
356
|
known = importlib.import_module(f"{_AUTO}.configuration_auto")
|
|
365
357
|
if model_type not in known.CONFIG_MAPPING_NAMES:
|
|
366
358
|
raise ConversionRefusedError(
|
|
@@ -376,7 +368,10 @@ class HuggingFaceFamily:
|
|
|
376
368
|
if pooling is not None:
|
|
377
369
|
declared = TaskCategory.EMBEDDINGS
|
|
378
370
|
label = architecture or model_type
|
|
379
|
-
values = {
|
|
371
|
+
values = {
|
|
372
|
+
"entry_point": _sourced(label, CONFIG_FILE, weights),
|
|
373
|
+
"model_type": _sourced(model_type, read_from),
|
|
374
|
+
}
|
|
380
375
|
names = _class_names(config)
|
|
381
376
|
if names:
|
|
382
377
|
values["id2label"] = _sourced(names, f"{CONFIG_FILE} id2label")
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""A Hugging Face folder's ``model_type``: its config's, or its architecture's.
|
|
2
|
+
|
|
3
|
+
ProtBERT's config.json names ``BertForMaskedLM`` and no ``model_type``;
|
|
4
|
+
transformers' own mappings give the model type of an architecture.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import importlib
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any, Mapping
|
|
12
|
+
|
|
13
|
+
from tracebloc.convert.interfaces import ConversionRefusedError, RefusalCause
|
|
14
|
+
|
|
15
|
+
CONFIG_FILE = "config.json"
|
|
16
|
+
_AUTO = "transformers.models.auto"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _names(value: Any) -> tuple[str, ...]:
|
|
20
|
+
"""A mapping's class names for one ``model_type``: one name or several."""
|
|
21
|
+
if isinstance(value, str):
|
|
22
|
+
return (value,)
|
|
23
|
+
return tuple(value) if isinstance(value, (tuple, list)) else ()
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _mapping(name: str) -> Mapping[str, Any]:
|
|
27
|
+
return getattr(importlib.import_module(f"{_AUTO}.modeling_auto"), name, {})
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _architecture_model_type(folder: Path, architecture: str) -> str:
|
|
31
|
+
"""The ``model_type`` transformers' mappings give ``architecture``; refused
|
|
32
|
+
when they give none, or several the class's own config does not settle."""
|
|
33
|
+
import transformers
|
|
34
|
+
|
|
35
|
+
modeling = importlib.import_module(f"{_AUTO}.modeling_auto")
|
|
36
|
+
found = {
|
|
37
|
+
model_type
|
|
38
|
+
for name in vars(modeling)
|
|
39
|
+
if name.endswith("_MAPPING_NAMES")
|
|
40
|
+
for model_type, classes in _mapping(name).items()
|
|
41
|
+
if architecture in _names(classes)
|
|
42
|
+
}
|
|
43
|
+
if len(found) > 1:
|
|
44
|
+
try:
|
|
45
|
+
own = getattr(transformers, architecture).config_class.model_type
|
|
46
|
+
except Exception: # noqa: BLE001 - a class that cannot load settles nothing
|
|
47
|
+
own = None
|
|
48
|
+
found = {own} if own in found else found
|
|
49
|
+
if len(found) == 1:
|
|
50
|
+
return found.pop()
|
|
51
|
+
named = f"several, {', '.join(sorted(found))}" if found else "none"
|
|
52
|
+
raise ConversionRefusedError(
|
|
53
|
+
RefusalCause.NO_TASK_CLASS,
|
|
54
|
+
f"{folder.name}'s {CONFIG_FILE} declares no model_type, and transformers "
|
|
55
|
+
f"{transformers.__version__} gives its architecture {architecture} {named}; "
|
|
56
|
+
f"add the model_type it was saved with to {CONFIG_FILE}",
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def model_type_of(folder: Path, config: Any) -> str | None:
|
|
61
|
+
"""The config's ``model_type``, or its architecture's; ``None`` when the
|
|
62
|
+
folder is not a transformers model or ships its own code."""
|
|
63
|
+
if not isinstance(config, dict):
|
|
64
|
+
return None
|
|
65
|
+
if isinstance(config.get("model_type"), str):
|
|
66
|
+
return config["model_type"]
|
|
67
|
+
architectures = _names(config.get("architectures"))
|
|
68
|
+
if "auto_map" in config or not architectures:
|
|
69
|
+
return None
|
|
70
|
+
from tracebloc._framework_utils import require_module
|
|
71
|
+
|
|
72
|
+
require_module("transformers", "converting a Hugging Face model")
|
|
73
|
+
return _architecture_model_type(folder, architectures[0])
|
|
@@ -31,6 +31,9 @@ FAMILIES_PACKAGE = "tracebloc.convert.families"
|
|
|
31
31
|
#: The module-level name each family module binds its family to.
|
|
32
32
|
FAMILY_ATTRIBUTE = "FAMILY"
|
|
33
33
|
|
|
34
|
+
#: The family every package built from the peer's own code records.
|
|
35
|
+
OWN_CODE_FAMILY = "own_code"
|
|
36
|
+
|
|
34
37
|
#: A family's ``name``: the stable id the report records.
|
|
35
38
|
_FAMILY_NAME = re.compile(r"^[a-z][a-z0-9_]*$")
|
|
36
39
|
|
|
@@ -262,6 +265,8 @@ class RefusalCause(_StringEnum):
|
|
|
262
265
|
BLOCKED_FAMILY = "blocked_family"
|
|
263
266
|
#: Local library versions differ from the training image's.
|
|
264
267
|
VERSIONS = "versions"
|
|
268
|
+
#: The model's own code raises when it is imported, built or read.
|
|
269
|
+
CODE_FAILS = "code_fails"
|
|
265
270
|
|
|
266
271
|
|
|
267
272
|
class ConversionRefusedError(ModelValidationError):
|
|
@@ -27,6 +27,7 @@ from pathlib import Path
|
|
|
27
27
|
from typing import Any, Iterator
|
|
28
28
|
|
|
29
29
|
from tracebloc.convert.interfaces import (
|
|
30
|
+
OWN_CODE_FAMILY,
|
|
30
31
|
ComparedOutput,
|
|
31
32
|
ConversionInput,
|
|
32
33
|
GeneratedPackage,
|
|
@@ -39,7 +40,7 @@ from tracebloc.convert.interfaces import (
|
|
|
39
40
|
class OwnCode:
|
|
40
41
|
"""What runs when no family recognises the input."""
|
|
41
42
|
|
|
42
|
-
name =
|
|
43
|
+
name = OWN_CODE_FAMILY
|
|
43
44
|
|
|
44
45
|
def recognise(self, source: ConversionInput) -> Recognition | None:
|
|
45
46
|
from tracebloc.convert.owncode import layer0, live, remote, repo
|
|
@@ -12,6 +12,7 @@ from pathlib import Path
|
|
|
12
12
|
from typing import Any, Mapping
|
|
13
13
|
|
|
14
14
|
from tracebloc.convert.interfaces import (
|
|
15
|
+
OWN_CODE_FAMILY,
|
|
15
16
|
ComparedOutput,
|
|
16
17
|
ConversionRefusedError,
|
|
17
18
|
ConverterWiringError,
|
|
@@ -33,7 +34,7 @@ from tracebloc.convert.owncode.shake import flat_name, shake
|
|
|
33
34
|
from tracebloc.utils.constants import TaskCategory
|
|
34
35
|
|
|
35
36
|
#: The name every own-code package records as its family.
|
|
36
|
-
FAMILY =
|
|
37
|
+
FAMILY = OWN_CODE_FAMILY
|
|
37
38
|
|
|
38
39
|
#: The class the platform loads from the entry file.
|
|
39
40
|
ENTRY_CLASS = "MyModel"
|
|
@@ -38,6 +38,7 @@ from tracebloc.convert.owncode.assemble import FAMILY
|
|
|
38
38
|
from tracebloc.convert.owncode.model_class import ClassIndex
|
|
39
39
|
from tracebloc.convert.owncode.repo import class_state
|
|
40
40
|
from tracebloc.convert.owncode.source_tree import SourceTree
|
|
41
|
+
from tracebloc.convert.training_versions import refuse_code_failure
|
|
41
42
|
from tracebloc.utils.constants import Framework
|
|
42
43
|
|
|
43
44
|
#: The file a notebook's captured code is shipped from, and named by in messages.
|
|
@@ -266,16 +267,28 @@ def recognise(source: ConversionInput) -> Recognition | None:
|
|
|
266
267
|
obj = source.live_object
|
|
267
268
|
if torch is None or not isinstance(obj, torch.nn.Module):
|
|
268
269
|
return None
|
|
269
|
-
|
|
270
|
+
try:
|
|
271
|
+
model, hyper, notes = _model_of(obj)
|
|
272
|
+
weights = dict(model.state_dict())
|
|
273
|
+
remote_config = (
|
|
274
|
+
model.config.to_dict()
|
|
275
|
+
if type(model).__module__.startswith("transformers_modules.")
|
|
276
|
+
else None
|
|
277
|
+
)
|
|
278
|
+
except ConversionRefusedError:
|
|
279
|
+
raise
|
|
280
|
+
except Exception as error:
|
|
281
|
+
what = "reading the object passed to upload_model"
|
|
282
|
+
raise refuse_code_failure(what, error) from error
|
|
270
283
|
cls = type(model)
|
|
271
284
|
module = sys.modules.get(cls.__module__)
|
|
272
285
|
file = getattr(module, "__file__", None)
|
|
273
286
|
if file and installed(Path(file)):
|
|
274
287
|
raise _refuse(f"is {cls.__module__}.{cls.__name__}, a library class")
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
288
|
+
if remote_config is not None and file:
|
|
289
|
+
found = remote.recognise_parts(
|
|
290
|
+
source, Path(file).parent, remote_config, weights, None
|
|
291
|
+
)
|
|
279
292
|
state = {**found.state, "live_object": model, "proof": Proof.EXACT}
|
|
280
293
|
return dataclasses.replace(found, state=state)
|
|
281
294
|
if cls.__module__.split(".")[0] in LIBRARY_ROOTS:
|
|
@@ -29,6 +29,7 @@ from typing import Any, Callable, Mapping
|
|
|
29
29
|
|
|
30
30
|
from tracebloc.upload.weights_parts import IN_PARTS_CAP_BYTES, SINGLE_REQUEST_CAP_BYTES
|
|
31
31
|
from tracebloc.convert.interfaces import (
|
|
32
|
+
OWN_CODE_FAMILY,
|
|
32
33
|
ConversionRefusedError,
|
|
33
34
|
ConverterWiringError,
|
|
34
35
|
GeneratedPackage,
|
|
@@ -36,6 +37,7 @@ from tracebloc.convert.interfaces import (
|
|
|
36
37
|
RefusalCause,
|
|
37
38
|
SourcedValue,
|
|
38
39
|
)
|
|
40
|
+
from tracebloc.convert.training_versions import refuse_code_failure
|
|
39
41
|
from tracebloc.exceptions import ModelValidationError
|
|
40
42
|
from tracebloc.validation.rewriter import ModelFileRewriter
|
|
41
43
|
|
|
@@ -362,6 +364,16 @@ def state_dict_shapes(written: WrittenPackage) -> dict[str, tuple[int, ...]]:
|
|
|
362
364
|
return {key: tuple(value.shape) for key, value in model.state_dict().items()}
|
|
363
365
|
|
|
364
366
|
|
|
367
|
+
def code_failure(written: WrittenPackage, error: Exception) -> ConversionRefusedError:
|
|
368
|
+
"""The refusal for the peer's own code in ``written`` raising ``error`` when run."""
|
|
369
|
+
with tempfile.TemporaryDirectory(prefix="tracebloc_sources_") as raw:
|
|
370
|
+
sources = [
|
|
371
|
+
path.read_text(encoding="utf-8")
|
|
372
|
+
for path in python_sources(written.model_file, Path(raw))
|
|
373
|
+
]
|
|
374
|
+
return refuse_code_failure("the converted model's code", error, sources)
|
|
375
|
+
|
|
376
|
+
|
|
365
377
|
def derive_seed_excluded_prefixes(
|
|
366
378
|
build_for_classes: Callable[[int, Path], GeneratedPackage],
|
|
367
379
|
shapes: Callable[
|
|
@@ -371,7 +383,8 @@ def derive_seed_excluded_prefixes(
|
|
|
371
383
|
"""The head's prefixes: build with each of :data:`PROBE_CLASSES` and diff.
|
|
372
384
|
|
|
373
385
|
``build_for_classes(count, workdir)`` builds the package for ``count``
|
|
374
|
-
output classes, which for a family is its ``build`` with that count.
|
|
386
|
+
output classes, which for a family is its ``build`` with that count. The
|
|
387
|
+
peer's own code raising when it runs here is its :func:`code_failure`.
|
|
375
388
|
"""
|
|
376
389
|
built = []
|
|
377
390
|
for count in PROBE_CLASSES:
|
|
@@ -379,7 +392,15 @@ def derive_seed_excluded_prefixes(
|
|
|
379
392
|
workdir, out = Path(raw) / "build", Path(raw) / "package"
|
|
380
393
|
workdir.mkdir()
|
|
381
394
|
package = build_for_classes(count, workdir)
|
|
382
|
-
|
|
395
|
+
written = write_package(package, out, "probe")
|
|
396
|
+
try:
|
|
397
|
+
built.append(shapes(written))
|
|
398
|
+
except (ConversionRefusedError, ConverterWiringError):
|
|
399
|
+
raise
|
|
400
|
+
except Exception as error:
|
|
401
|
+
if package.family != OWN_CODE_FAMILY:
|
|
402
|
+
raise
|
|
403
|
+
raise code_failure(written, error) from error
|
|
383
404
|
return head_prefixes(*built)
|
|
384
405
|
|
|
385
406
|
|
|
@@ -25,6 +25,7 @@ from tracebloc.convert.interfaces import (
|
|
|
25
25
|
ConverterWiringError,
|
|
26
26
|
RefusalCause,
|
|
27
27
|
)
|
|
28
|
+
from tracebloc.utils.general_utils import format_exception
|
|
28
29
|
|
|
29
30
|
#: The vendored copy of the training images' requirements.
|
|
30
31
|
PINS_FILE = (
|
|
@@ -52,6 +53,8 @@ TRAINING_LIBRARIES: Mapping[str, tuple[str, ...]] = {
|
|
|
52
53
|
TRAINING_EXTRA = "training"
|
|
53
54
|
|
|
54
55
|
_PIN = re.compile(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*)\s*==\s*([^\s;#]+)")
|
|
56
|
+
#: The ``(/path/to/file.py)`` an ImportError names: this machine's, not the model's.
|
|
57
|
+
_SOURCE_FILE = re.compile(r" \((?:/|[A-Za-z]:\\)[^)]*\)")
|
|
55
58
|
|
|
56
59
|
|
|
57
60
|
def read_pins(path: Path | None = None) -> dict[str, str]:
|
|
@@ -160,3 +163,28 @@ def refuse_other_versions(sources: Iterable[str]) -> CheckFinding:
|
|
|
160
163
|
raise ConversionRefusedError(RefusalCause.VERSIONS, refusal_message(clauses))
|
|
161
164
|
shown = ", ".join(libraries) or "no training library"
|
|
162
165
|
return CheckFinding("versions", True, f"{shown} at the training images' versions")
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def refuse_code_failure(
|
|
169
|
+
what: str, error: BaseException, sources: Iterable[str] = ()
|
|
170
|
+
) -> ConversionRefusedError:
|
|
171
|
+
"""The refusal for the model's own code raising ``error`` while ``what`` ran.
|
|
172
|
+
|
|
173
|
+
``sources`` are the model files run. When a training library they import is
|
|
174
|
+
installed here at another version than training's, that is the refusal.
|
|
175
|
+
"""
|
|
176
|
+
libraries = imported_libraries(sources) or ("torch",)
|
|
177
|
+
clauses = version_mismatches(libraries)
|
|
178
|
+
if clauses:
|
|
179
|
+
return ConversionRefusedError(RefusalCause.VERSIONS, refusal_message(clauses))
|
|
180
|
+
pins = read_pins()
|
|
181
|
+
named = ", ".join(f"{name} {pinned_version(name, pins)}" for name in libraries)
|
|
182
|
+
detail = " ".join(_SOURCE_FILE.sub("", format_exception(error)).split())[:300]
|
|
183
|
+
advice = "fix the code so it runs under them"
|
|
184
|
+
if "transformers" in libraries:
|
|
185
|
+
advice += ", or bring a checkpoint transformers reads without its own code"
|
|
186
|
+
return ConversionRefusedError(
|
|
187
|
+
RefusalCause.CODE_FAILS,
|
|
188
|
+
f"{what} raised {detail}: the model's own code does not run under the "
|
|
189
|
+
f"training versions ({named}); {advice}",
|
|
190
|
+
)
|
|
@@ -52,6 +52,7 @@ tracebloc/convert/families/__init__.py
|
|
|
52
52
|
tracebloc/convert/families/tabular.py
|
|
53
53
|
tracebloc/convert/families/hf/__init__.py
|
|
54
54
|
tracebloc/convert/families/hf/_build.py
|
|
55
|
+
tracebloc/convert/families/hf/_model_type.py
|
|
55
56
|
tracebloc/convert/families/vision/__init__.py
|
|
56
57
|
tracebloc/convert/families/vision/_matching.py
|
|
57
58
|
tracebloc/convert/licence/__init__.py
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|