tracebloc 1.2.74__tar.gz → 1.2.76__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. {tracebloc-1.2.74 → tracebloc-1.2.76}/PKG-INFO +1 -1
  2. {tracebloc-1.2.74 → tracebloc-1.2.76}/pyproject.toml +1 -1
  3. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/_upload_conversion.py +2 -58
  4. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/acceptance.py +14 -4
  5. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/families/hf/__init__.py +15 -20
  6. tracebloc-1.2.76/tracebloc/convert/families/hf/_model_type.py +73 -0
  7. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/interfaces.py +5 -0
  8. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/__init__.py +2 -1
  9. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/assemble.py +2 -1
  10. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/live.py +18 -5
  11. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/package.py +23 -2
  12. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/training_versions.py +28 -0
  13. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc.egg-info/PKG-INFO +1 -1
  14. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc.egg-info/SOURCES.txt +1 -0
  15. {tracebloc-1.2.74 → tracebloc-1.2.76}/MANIFEST.in +0 -0
  16. {tracebloc-1.2.74 → tracebloc-1.2.76}/README.md +0 -0
  17. {tracebloc-1.2.74 → tracebloc-1.2.76}/license.txt +0 -0
  18. {tracebloc-1.2.74 → tracebloc-1.2.76}/setup.cfg +0 -0
  19. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/__init__.py +0 -0
  20. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/_deprecation.py +0 -0
  21. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/_framework_utils.py +0 -0
  22. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/_http_utils.py +0 -0
  23. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/_link_answer.py +0 -0
  24. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/_link_regeneration.py +0 -0
  25. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/_linking.py +0 -0
  26. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/_raw_width_note.py +0 -0
  27. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/_stored_link.py +0 -0
  28. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/checks.py +0 -0
  29. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/client.py +0 -0
  30. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/config.py +0 -0
  31. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/contracts/stage_urls.v1.json +0 -0
  32. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/contracts/stage_urls.v1.source.json +0 -0
  33. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/contracts/survival_payload.py +0 -0
  34. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/contracts/training_requirements.source.json +0 -0
  35. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/contracts/training_requirements.txt +0 -0
  36. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/__init__.py +0 -0
  37. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/families/__init__.py +0 -0
  38. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/families/hf/_build.py +0 -0
  39. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/families/tabular.py +0 -0
  40. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/families/vision/__init__.py +0 -0
  41. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/families/vision/_matching.py +0 -0
  42. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/gate.py +0 -0
  43. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/licence/__init__.py +0 -0
  44. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/licence/spdx.py +0 -0
  45. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/outputs.py +0 -0
  46. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/arguments.py +0 -0
  47. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/checkpoint.py +0 -0
  48. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/gate_rules.py +0 -0
  49. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/generate.py +0 -0
  50. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/head.py +0 -0
  51. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/import_check.py +0 -0
  52. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/layer0.py +0 -0
  53. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/model_class.py +0 -0
  54. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/remote.py +0 -0
  55. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/repo.py +0 -0
  56. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/shake.py +0 -0
  57. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/owncode/source_tree.py +0 -0
  58. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/report.py +0 -0
  59. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/convert/tokenizers.py +0 -0
  60. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/env_login.py +0 -0
  61. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/exceptions.py +0 -0
  62. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/fetch.py +0 -0
  63. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/inference.py +0 -0
  64. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/library_fetch.py +0 -0
  65. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/lifelines_weights.py +0 -0
  66. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/link_model_dataset.py +0 -0
  67. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/notebook/__init__.py +0 -0
  68. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/notebook/templates/_partials.tpl +0 -0
  69. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/notebook/templates/embeddings.tpl +0 -0
  70. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/notebook/templates/nlp_finetune.tpl +0 -0
  71. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/notebook/templates/nlp_generative.tpl +0 -0
  72. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/notebook/templates/survival.tpl +0 -0
  73. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/notebook/templates/tabular_timeseries.tpl +0 -0
  74. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/notebook/templates/vision_from_scratch.tpl +0 -0
  75. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/raw_mode.py +0 -0
  76. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/raw_mode_frame.py +0 -0
  77. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/raw_mode_seed.py +0 -0
  78. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/raw_mode_survival.py +0 -0
  79. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/raw_mode_torch.py +0 -0
  80. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/seed_weights.py +0 -0
  81. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/__init__.py +0 -0
  82. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/augmentation.py +0 -0
  83. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/callbacks.py +0 -0
  84. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/converted.py +0 -0
  85. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/features.py +0 -0
  86. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/image_size.py +0 -0
  87. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/lora.py +0 -0
  88. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/lora_validation.py +0 -0
  89. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/loss.py +0 -0
  90. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/memory_estimate.py +0 -0
  91. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/plan.py +0 -0
  92. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/preprocessing.py +0 -0
  93. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/shape.py +0 -0
  94. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/subdataset.py +0 -0
  95. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/submitter.py +0 -0
  96. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/training/summary.py +0 -0
  97. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/__init__.py +0 -0
  98. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/lifelines_time_to_event.py +0 -0
  99. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/model_upload.py +0 -0
  100. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/scikit_survival_time_to_event.py +0 -0
  101. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/sequence_length.py +0 -0
  102. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/skl_tabular_base.py +0 -0
  103. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/skl_tabular_classifier.py +0 -0
  104. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/skl_tabular_regression.py +0 -0
  105. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/statistical_base.py +0 -0
  106. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/survival_base.py +0 -0
  107. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_causal_language_modeler.py +0 -0
  108. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_embeddings.py +0 -0
  109. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_image_classifier.py +0 -0
  110. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_key_point_detector.py +0 -0
  111. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_masked_language_modeler.py +0 -0
  112. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_nlp_base.py +0 -0
  113. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_object_detector.py +0 -0
  114. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_semantic_segmentation.py +0 -0
  115. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_sentence_pair_classifier.py +0 -0
  116. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_seq2seq_modeler.py +0 -0
  117. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_tabular_classifier.py +0 -0
  118. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_tabular_regression.py +0 -0
  119. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_text_classifier.py +0 -0
  120. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_time_series_classifier.py +0 -0
  121. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_time_series_forecaster.py +0 -0
  122. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_time_to_event.py +0 -0
  123. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/torch_token_classifier.py +0 -0
  124. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/weight_loaders.py +0 -0
  125. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/upload/weights_parts.py +0 -0
  126. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/user.py +0 -0
  127. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/__init__.py +0 -0
  128. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/class_order.py +0 -0
  129. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/constants.py +0 -0
  130. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/custom_loss_probe.py +0 -0
  131. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/detection_utils.py +0 -0
  132. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/entry_point.py +0 -0
  133. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/feature_dict.py +0 -0
  134. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/general_utils.py +0 -0
  135. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/key_point_detection_utils.py +0 -0
  136. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/registry.py +0 -0
  137. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/safetensors_io.py +0 -0
  138. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/semantic_segmentation_utils.py +0 -0
  139. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/tabular_classification_utils.py +0 -0
  140. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/tabular_custom_loss_utils.py +0 -0
  141. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/tabular_regression_utils.py +0 -0
  142. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/text_classification_utils.py +0 -0
  143. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/time_series_utils.py +0 -0
  144. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/time_to_event_utils.py +0 -0
  145. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/utils/warnings_once.py +0 -0
  146. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/validation/__init__.py +0 -0
  147. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/validation/checker.py +0 -0
  148. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/validation/header_rules.py +0 -0
  149. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/validation/lifelines_checks.py +0 -0
  150. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/validation/pytorch_checks.py +0 -0
  151. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/validation/rewriter.py +0 -0
  152. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/validation/scikit_survival_checks.py +0 -0
  153. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc/validation/sklearn_checks.py +0 -0
  154. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc.egg-info/dependency_links.txt +0 -0
  155. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc.egg-info/requires.txt +0 -0
  156. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc.egg-info/top_level.txt +0 -0
  157. {tracebloc-1.2.74 → tracebloc-1.2.76}/tracebloc_package/__init__.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tracebloc
3
- Version: 1.2.74
3
+ Version: 1.2.76
4
4
  Summary: Build AI with anyone. On data that can't move. SDK for the tracebloc collaborative AI workspace.
5
5
  Author-email: tracebloc <lukas@tracebloc.io>
6
6
  License-Expression: MIT
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "tracebloc"
7
- version = "1.2.74"
7
+ version = "1.2.76"
8
8
  description = "Build AI with anyone. On data that can't move. SDK for the tracebloc collaborative AI workspace."
9
9
  license = "MIT"
10
10
  license-files = ["license.txt"]
@@ -5,7 +5,7 @@ untouched. Anything else (a Hugging Face folder, a checkpoint, the peer's own
5
5
  code, a live object) goes through the converter, whose package is uploaded
6
6
  only once its acceptance check passed and its licence is confirmed. The use
7
7
  case's task comes from ``dataset_id``: ``dataset/<dataset_id>/task/`` answers
8
- it for a dataset the peer can link a model to.
8
+ it for a dataset the peer can link a model to, and its 404 refuses any other.
9
9
  """
10
10
 
11
11
  from __future__ import annotations
@@ -32,12 +32,8 @@ from tracebloc.validation.header_rules import NO_OUTPUT_CLASSES
32
32
 
33
33
  #: A dataset's category and counts, for a dataset the peer can link a model to.
34
34
  DATASET_TASK_PATH = "dataset/{dataset_id}/task/"
35
- #: The use cases the peer can see, each with its training dataset.
36
- USE_CASES_PATH = "competition/"
37
35
  #: ``licence_confirmed``'s default: ask the peer, in a notebook only.
38
36
  ASK = "ask"
39
- #: How many pages of use cases are read before giving up on a dataset.
40
- MAX_PAGES = 20
41
37
 
42
38
 
43
39
  @dataclass(frozen=True)
@@ -83,67 +79,15 @@ def _not_found(dataset_id: str) -> ModelValidationError:
83
79
  )
84
80
 
85
81
 
86
- def _route_missing(response: Any) -> bool:
87
- """True for a 404 the server's URL routing answered rather than the view:
88
- the API's own 404 is JSON ``{"detail": ...}``, an unknown URL's page is not."""
89
- try:
90
- body = response.json()
91
- except ValueError:
92
- return True
93
- return not (isinstance(body, Mapping) and "detail" in body)
94
-
95
-
96
- def _class_count(dataset: Mapping[str, Any]) -> int | None:
97
- meta = dataset.get("dataset_meta")
98
- counts = meta.get("count") if isinstance(meta, Mapping) else None
99
- labels = counts.get("label") if isinstance(counts, Mapping) else None
100
- return len(labels) if isinstance(labels, (Mapping, list)) and labels else None
101
-
102
-
103
- def _use_cases(client: Any) -> Any:
104
- """Every use case ``competition/`` lists for the peer, page by page."""
105
- for page in range(1, MAX_PAGES + 1):
106
- response = client.get(
107
- USE_CASES_PATH, params={"page": page}, raise_on_error=False
108
- )
109
- if response.status_code != 200:
110
- raise APIError(response.status_code, "the use cases could not be read")
111
- body = _json(response, "the use cases")
112
- yield from body.get("results", []) if isinstance(body, Mapping) else body
113
- if not isinstance(body, Mapping) or not body.get("next"):
114
- return
115
-
116
-
117
- def _task_from_use_cases(client: Any, dataset_id: str) -> UseCaseTask:
118
- """The task of the use case whose training dataset is ``dataset_id``, for a
119
- server without ``dataset/<dataset_id>/task/``: its category, and its class
120
- count when the dataset records its labels."""
121
- for use_case in _use_cases(client):
122
- dataset = use_case.get("dataset") or {}
123
- if dataset_id not in (use_case.get("dataset_key"), dataset.get("dataset_key")):
124
- continue
125
- category = _category(dataset, dataset_id)
126
- classes = _class_count(dataset)
127
- counts = {}
128
- if classes is not None and category not in NO_OUTPUT_CLASSES:
129
- counts["output_classes"] = classes
130
- return UseCaseTask(category, from_use_case=True, counts=counts)
131
- raise _not_found(dataset_id)
132
-
133
-
134
82
  def dataset_task(client: Any, dataset_id: str) -> UseCaseTask:
135
83
  """The task of ``dataset_id``, a dataset the peer can link a model to.
136
84
 
137
85
  Its category, and every count the server records for it; a class count
138
- only for a task with a class per label.
86
+ only for a task with a class per label. A 404 is the dataset's refusal.
139
87
  """
140
88
  path = DATASET_TASK_PATH.format(dataset_id=quote(str(dataset_id), safe=""))
141
89
  response = client.get(path, raise_on_error=False)
142
90
  if response.status_code == 404:
143
- if _route_missing(response):
144
- # TODO(tracebloc/tracebloc-py-package#1059): a server older than
145
- # dataset/<key>/task/ answers the task from its use cases.
146
- return _task_from_use_cases(client, dataset_id)
147
91
  raise _not_found(dataset_id)
148
92
  what = f"the task of dataset '{dataset_id}'"
149
93
  if response.status_code != 200:
@@ -21,6 +21,7 @@ from typing import Any, Iterator, Sequence
21
21
 
22
22
  from tracebloc.convert import outputs, training_versions
23
23
  from tracebloc.convert.interfaces import (
24
+ OWN_CODE_FAMILY,
24
25
  CheckFinding,
25
26
  ConverterWiringError,
26
27
  Family,
@@ -127,8 +128,13 @@ def upload_check(package: GeneratedPackage, written: WrittenPackage) -> CheckFin
127
128
  return _finding("upload", problems, "within the backend's caps, with a handler")
128
129
 
129
130
 
130
- def build_model(written: WrittenPackage, workdir: Path) -> tuple[Any, CheckFinding]:
131
- """``MyModel()`` from the written package, seeded; ``None`` when it cannot be built."""
131
+ def build_model(
132
+ written: WrittenPackage, workdir: Path, own_code: bool = False
133
+ ) -> tuple[Any, CheckFinding]:
134
+ """``MyModel()`` from the written package, seeded; ``None`` when it cannot be built.
135
+
136
+ The peer's ``own_code`` raising is its refusal, :func:`package.code_failure`.
137
+ """
132
138
  try:
133
139
  module = load_entry_module(written, workdir)
134
140
  if not callable(getattr(module, "MyModel", None)):
@@ -138,6 +144,8 @@ def build_model(written: WrittenPackage, workdir: Path) -> tuple[Any, CheckFindi
138
144
  with _seeded():
139
145
  model = module.MyModel()
140
146
  except Exception as error: # noqa: BLE001 - the check reports, never raises
147
+ if own_code and not isinstance(error, ConverterWiringError):
148
+ raise package_module.code_failure(written, error) from error
141
149
  return None, CheckFinding(
142
150
  "build", False, f"MyModel() raised {_one_line(error)}"
143
151
  )
@@ -354,7 +362,8 @@ def findings(
354
362
 
355
363
  Refuses first, before anything is built, when a training library the
356
364
  package imports is installed here at another version than the training
357
- images pin. Then: today's header check, the backend's upload checks,
365
+ images pin; it refuses the peer's own code, too, when ``MyModel()`` raises.
366
+ Then: today's header check, the backend's upload checks,
358
367
  ``MyModel()`` built, its parameter count, the shipped weights' strict load,
359
368
  the output against the original that ``family``'s ``build_original``
360
369
  builds (shape-only when it builds none), and the token ids for a package
@@ -366,7 +375,8 @@ def findings(
366
375
  upload_check(package, written),
367
376
  ]
368
377
  with scratch() as raw:
369
- model, built = build_model(written, Path(raw))
378
+ own_code = package.family == OWN_CODE_FAMILY
379
+ model, built = build_model(written, Path(raw), own_code)
370
380
  found.append(built)
371
381
  if model is not None:
372
382
  found += model_findings(recognition, package, written, model, family)
@@ -1,7 +1,9 @@
1
1
  """Hugging Face transformers: the use case names the task, any checkpoint fine-tunes.
2
2
 
3
3
  A folder is recognised by its ``config.json`` ``model_type``, read without
4
- running anything. A config with an ``auto_map`` ships its own model code, so
4
+ running anything; a config saved without one names it through its
5
+ ``architectures``, which transformers' own mappings resolve. A config with an
6
+ ``auto_map`` ships its own model code, so
5
7
  the folder is the own-code converter's, not this family's. A sentence-transformers
6
8
  folder is the transformer its ``modules.json`` names. A folder saved for
7
9
  EvolutionaryScale's ``esm`` package is refused, naming ESM C's transformers checkpoint.
@@ -23,6 +25,8 @@ from typing import Any, Mapping
23
25
 
24
26
  from tracebloc.convert.families.hf._build import head_prefixes, load, model_source
25
27
  from tracebloc.convert.families.hf._build import write_weights
28
+ from tracebloc.convert.families.hf._model_type import _AUTO, CONFIG_FILE, _mapping
29
+ from tracebloc.convert.families.hf._model_type import _names, model_type_of
26
30
  from tracebloc.convert.families.vision import read_normalisation
27
31
  from tracebloc.convert.interfaces import ComparedOutput, ConversionInput, Proof
28
32
  from tracebloc.convert.interfaces import ConversionRefusedError, RefusalCause
@@ -33,7 +37,6 @@ from tracebloc.utils.constants import Framework, TaskCategory
33
37
  from tracebloc.validation.header_rules import NO_OUTPUT_CLASSES, SELF_SUPERVISED_NLP
34
38
  from tracebloc.validation.header_rules import SHAPE_VALUES
35
39
 
36
- CONFIG_FILE = "config.json"
37
40
  #: A sentence-transformers folder's module list.
38
41
  ST_MODULES = "modules.json"
39
42
  #: What transformers loads a folder's PyTorch weights from, shards included.
@@ -51,8 +54,6 @@ ESM_PACKAGE_WEIGHTS = "data/weights/esm*.pth"
51
54
  #: ESM C as transformers reads it (``model_type`` ``esmc``).
52
55
  ESMC_CHECKPOINTS = "biohub/ESMC-300M-hf, ESMC-600M-hf or ESMC-6B-hf"
53
56
 
54
- _AUTO = "transformers.models.auto"
55
-
56
57
  #: The Auto class each task is built with, and the mapping in
57
58
  #: ``transformers.models.auto.modeling_auto`` naming its class per
58
59
  #: ``model_type``. Earlier rows win as a checkpoint's declared head.
@@ -135,17 +136,6 @@ def _read_json(path: Path) -> Any:
135
136
  return None
136
137
 
137
138
 
138
- def _names(value: Any) -> tuple[str, ...]:
139
- """A mapping's class names for one ``model_type``: one name or several."""
140
- if isinstance(value, str):
141
- return (value,)
142
- return tuple(value) if isinstance(value, (tuple, list)) else ()
143
-
144
-
145
- def _mapping(name: str) -> Mapping[str, Any]:
146
- return getattr(importlib.import_module(f"{_AUTO}.modeling_auto"), name, {})
147
-
148
-
149
139
  def _modality(model_type: str) -> str | None:
150
140
  """``"text"`` or ``"images"``, from transformers' tokenizer and image-processor
151
141
  mappings; ``None`` for a model with both or neither."""
@@ -350,9 +340,8 @@ class HuggingFaceFamily:
350
340
  return None
351
341
  folder, pooling = _sentence_transformer(path)
352
342
  config = _read_json(folder / CONFIG_FILE)
353
- if not isinstance(config, dict) or not isinstance(
354
- config.get("model_type"), str
355
- ):
343
+ model_type = model_type_of(folder, config)
344
+ if model_type is None:
356
345
  _refuse_esm_package_layout(folder)
357
346
  return None
358
347
  if "auto_map" in config:
@@ -360,7 +349,10 @@ class HuggingFaceFamily:
360
349
  from tracebloc._framework_utils import require_module
361
350
 
362
351
  transformers = require_module("transformers", "converting a Hugging Face model")
363
- model_type = config["model_type"]
352
+ read_from = CONFIG_FILE
353
+ if config.get("model_type") != model_type:
354
+ read_from = f"{CONFIG_FILE} architectures"
355
+ config = {**config, "model_type": model_type}
364
356
  known = importlib.import_module(f"{_AUTO}.configuration_auto")
365
357
  if model_type not in known.CONFIG_MAPPING_NAMES:
366
358
  raise ConversionRefusedError(
@@ -376,7 +368,10 @@ class HuggingFaceFamily:
376
368
  if pooling is not None:
377
369
  declared = TaskCategory.EMBEDDINGS
378
370
  label = architecture or model_type
379
- values = {"entry_point": _sourced(label, CONFIG_FILE, weights)}
371
+ values = {
372
+ "entry_point": _sourced(label, CONFIG_FILE, weights),
373
+ "model_type": _sourced(model_type, read_from),
374
+ }
380
375
  names = _class_names(config)
381
376
  if names:
382
377
  values["id2label"] = _sourced(names, f"{CONFIG_FILE} id2label")
@@ -0,0 +1,73 @@
1
+ """A Hugging Face folder's ``model_type``: its config's, or its architecture's.
2
+
3
+ ProtBERT's config.json names ``BertForMaskedLM`` and no ``model_type``;
4
+ transformers' own mappings give the model type of an architecture.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import importlib
10
+ from pathlib import Path
11
+ from typing import Any, Mapping
12
+
13
+ from tracebloc.convert.interfaces import ConversionRefusedError, RefusalCause
14
+
15
+ CONFIG_FILE = "config.json"
16
+ _AUTO = "transformers.models.auto"
17
+
18
+
19
+ def _names(value: Any) -> tuple[str, ...]:
20
+ """A mapping's class names for one ``model_type``: one name or several."""
21
+ if isinstance(value, str):
22
+ return (value,)
23
+ return tuple(value) if isinstance(value, (tuple, list)) else ()
24
+
25
+
26
+ def _mapping(name: str) -> Mapping[str, Any]:
27
+ return getattr(importlib.import_module(f"{_AUTO}.modeling_auto"), name, {})
28
+
29
+
30
+ def _architecture_model_type(folder: Path, architecture: str) -> str:
31
+ """The ``model_type`` transformers' mappings give ``architecture``; refused
32
+ when they give none, or several the class's own config does not settle."""
33
+ import transformers
34
+
35
+ modeling = importlib.import_module(f"{_AUTO}.modeling_auto")
36
+ found = {
37
+ model_type
38
+ for name in vars(modeling)
39
+ if name.endswith("_MAPPING_NAMES")
40
+ for model_type, classes in _mapping(name).items()
41
+ if architecture in _names(classes)
42
+ }
43
+ if len(found) > 1:
44
+ try:
45
+ own = getattr(transformers, architecture).config_class.model_type
46
+ except Exception: # noqa: BLE001 - a class that cannot load settles nothing
47
+ own = None
48
+ found = {own} if own in found else found
49
+ if len(found) == 1:
50
+ return found.pop()
51
+ named = f"several, {', '.join(sorted(found))}" if found else "none"
52
+ raise ConversionRefusedError(
53
+ RefusalCause.NO_TASK_CLASS,
54
+ f"{folder.name}'s {CONFIG_FILE} declares no model_type, and transformers "
55
+ f"{transformers.__version__} gives its architecture {architecture} {named}; "
56
+ f"add the model_type it was saved with to {CONFIG_FILE}",
57
+ )
58
+
59
+
60
+ def model_type_of(folder: Path, config: Any) -> str | None:
61
+ """The config's ``model_type``, or its architecture's; ``None`` when the
62
+ folder is not a transformers model or ships its own code."""
63
+ if not isinstance(config, dict):
64
+ return None
65
+ if isinstance(config.get("model_type"), str):
66
+ return config["model_type"]
67
+ architectures = _names(config.get("architectures"))
68
+ if "auto_map" in config or not architectures:
69
+ return None
70
+ from tracebloc._framework_utils import require_module
71
+
72
+ require_module("transformers", "converting a Hugging Face model")
73
+ return _architecture_model_type(folder, architectures[0])
@@ -31,6 +31,9 @@ FAMILIES_PACKAGE = "tracebloc.convert.families"
31
31
  #: The module-level name each family module binds its family to.
32
32
  FAMILY_ATTRIBUTE = "FAMILY"
33
33
 
34
+ #: The family every package built from the peer's own code records.
35
+ OWN_CODE_FAMILY = "own_code"
36
+
34
37
  #: A family's ``name``: the stable id the report records.
35
38
  _FAMILY_NAME = re.compile(r"^[a-z][a-z0-9_]*$")
36
39
 
@@ -262,6 +265,8 @@ class RefusalCause(_StringEnum):
262
265
  BLOCKED_FAMILY = "blocked_family"
263
266
  #: Local library versions differ from the training image's.
264
267
  VERSIONS = "versions"
268
+ #: The model's own code raises when it is imported, built or read.
269
+ CODE_FAILS = "code_fails"
265
270
 
266
271
 
267
272
  class ConversionRefusedError(ModelValidationError):
@@ -27,6 +27,7 @@ from pathlib import Path
27
27
  from typing import Any, Iterator
28
28
 
29
29
  from tracebloc.convert.interfaces import (
30
+ OWN_CODE_FAMILY,
30
31
  ComparedOutput,
31
32
  ConversionInput,
32
33
  GeneratedPackage,
@@ -39,7 +40,7 @@ from tracebloc.convert.interfaces import (
39
40
  class OwnCode:
40
41
  """What runs when no family recognises the input."""
41
42
 
42
- name = "own_code"
43
+ name = OWN_CODE_FAMILY
43
44
 
44
45
  def recognise(self, source: ConversionInput) -> Recognition | None:
45
46
  from tracebloc.convert.owncode import layer0, live, remote, repo
@@ -12,6 +12,7 @@ from pathlib import Path
12
12
  from typing import Any, Mapping
13
13
 
14
14
  from tracebloc.convert.interfaces import (
15
+ OWN_CODE_FAMILY,
15
16
  ComparedOutput,
16
17
  ConversionRefusedError,
17
18
  ConverterWiringError,
@@ -33,7 +34,7 @@ from tracebloc.convert.owncode.shake import flat_name, shake
33
34
  from tracebloc.utils.constants import TaskCategory
34
35
 
35
36
  #: The name every own-code package records as its family.
36
- FAMILY = "own_code"
37
+ FAMILY = OWN_CODE_FAMILY
37
38
 
38
39
  #: The class the platform loads from the entry file.
39
40
  ENTRY_CLASS = "MyModel"
@@ -38,6 +38,7 @@ from tracebloc.convert.owncode.assemble import FAMILY
38
38
  from tracebloc.convert.owncode.model_class import ClassIndex
39
39
  from tracebloc.convert.owncode.repo import class_state
40
40
  from tracebloc.convert.owncode.source_tree import SourceTree
41
+ from tracebloc.convert.training_versions import refuse_code_failure
41
42
  from tracebloc.utils.constants import Framework
42
43
 
43
44
  #: The file a notebook's captured code is shipped from, and named by in messages.
@@ -266,16 +267,28 @@ def recognise(source: ConversionInput) -> Recognition | None:
266
267
  obj = source.live_object
267
268
  if torch is None or not isinstance(obj, torch.nn.Module):
268
269
  return None
269
- model, hyper, notes = _model_of(obj)
270
+ try:
271
+ model, hyper, notes = _model_of(obj)
272
+ weights = dict(model.state_dict())
273
+ remote_config = (
274
+ model.config.to_dict()
275
+ if type(model).__module__.startswith("transformers_modules.")
276
+ else None
277
+ )
278
+ except ConversionRefusedError:
279
+ raise
280
+ except Exception as error:
281
+ what = "reading the object passed to upload_model"
282
+ raise refuse_code_failure(what, error) from error
270
283
  cls = type(model)
271
284
  module = sys.modules.get(cls.__module__)
272
285
  file = getattr(module, "__file__", None)
273
286
  if file and installed(Path(file)):
274
287
  raise _refuse(f"is {cls.__module__}.{cls.__name__}, a library class")
275
- weights = dict(model.state_dict())
276
- if cls.__module__.startswith("transformers_modules.") and file:
277
- config = model.config.to_dict()
278
- found = remote.recognise_parts(source, Path(file).parent, config, weights, None)
288
+ if remote_config is not None and file:
289
+ found = remote.recognise_parts(
290
+ source, Path(file).parent, remote_config, weights, None
291
+ )
279
292
  state = {**found.state, "live_object": model, "proof": Proof.EXACT}
280
293
  return dataclasses.replace(found, state=state)
281
294
  if cls.__module__.split(".")[0] in LIBRARY_ROOTS:
@@ -29,6 +29,7 @@ from typing import Any, Callable, Mapping
29
29
 
30
30
  from tracebloc.upload.weights_parts import IN_PARTS_CAP_BYTES, SINGLE_REQUEST_CAP_BYTES
31
31
  from tracebloc.convert.interfaces import (
32
+ OWN_CODE_FAMILY,
32
33
  ConversionRefusedError,
33
34
  ConverterWiringError,
34
35
  GeneratedPackage,
@@ -36,6 +37,7 @@ from tracebloc.convert.interfaces import (
36
37
  RefusalCause,
37
38
  SourcedValue,
38
39
  )
40
+ from tracebloc.convert.training_versions import refuse_code_failure
39
41
  from tracebloc.exceptions import ModelValidationError
40
42
  from tracebloc.validation.rewriter import ModelFileRewriter
41
43
 
@@ -362,6 +364,16 @@ def state_dict_shapes(written: WrittenPackage) -> dict[str, tuple[int, ...]]:
362
364
  return {key: tuple(value.shape) for key, value in model.state_dict().items()}
363
365
 
364
366
 
367
+ def code_failure(written: WrittenPackage, error: Exception) -> ConversionRefusedError:
368
+ """The refusal for the peer's own code in ``written`` raising ``error`` when run."""
369
+ with tempfile.TemporaryDirectory(prefix="tracebloc_sources_") as raw:
370
+ sources = [
371
+ path.read_text(encoding="utf-8")
372
+ for path in python_sources(written.model_file, Path(raw))
373
+ ]
374
+ return refuse_code_failure("the converted model's code", error, sources)
375
+
376
+
365
377
  def derive_seed_excluded_prefixes(
366
378
  build_for_classes: Callable[[int, Path], GeneratedPackage],
367
379
  shapes: Callable[
@@ -371,7 +383,8 @@ def derive_seed_excluded_prefixes(
371
383
  """The head's prefixes: build with each of :data:`PROBE_CLASSES` and diff.
372
384
 
373
385
  ``build_for_classes(count, workdir)`` builds the package for ``count``
374
- output classes, which for a family is its ``build`` with that count.
386
+ output classes, which for a family is its ``build`` with that count. The
387
+ peer's own code raising when it runs here is its :func:`code_failure`.
375
388
  """
376
389
  built = []
377
390
  for count in PROBE_CLASSES:
@@ -379,7 +392,15 @@ def derive_seed_excluded_prefixes(
379
392
  workdir, out = Path(raw) / "build", Path(raw) / "package"
380
393
  workdir.mkdir()
381
394
  package = build_for_classes(count, workdir)
382
- built.append(shapes(write_package(package, out, "probe")))
395
+ written = write_package(package, out, "probe")
396
+ try:
397
+ built.append(shapes(written))
398
+ except (ConversionRefusedError, ConverterWiringError):
399
+ raise
400
+ except Exception as error:
401
+ if package.family != OWN_CODE_FAMILY:
402
+ raise
403
+ raise code_failure(written, error) from error
383
404
  return head_prefixes(*built)
384
405
 
385
406
 
@@ -25,6 +25,7 @@ from tracebloc.convert.interfaces import (
25
25
  ConverterWiringError,
26
26
  RefusalCause,
27
27
  )
28
+ from tracebloc.utils.general_utils import format_exception
28
29
 
29
30
  #: The vendored copy of the training images' requirements.
30
31
  PINS_FILE = (
@@ -52,6 +53,8 @@ TRAINING_LIBRARIES: Mapping[str, tuple[str, ...]] = {
52
53
  TRAINING_EXTRA = "training"
53
54
 
54
55
  _PIN = re.compile(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*)\s*==\s*([^\s;#]+)")
56
+ #: The ``(/path/to/file.py)`` an ImportError names: this machine's, not the model's.
57
+ _SOURCE_FILE = re.compile(r" \((?:/|[A-Za-z]:\\)[^)]*\)")
55
58
 
56
59
 
57
60
  def read_pins(path: Path | None = None) -> dict[str, str]:
@@ -160,3 +163,28 @@ def refuse_other_versions(sources: Iterable[str]) -> CheckFinding:
160
163
  raise ConversionRefusedError(RefusalCause.VERSIONS, refusal_message(clauses))
161
164
  shown = ", ".join(libraries) or "no training library"
162
165
  return CheckFinding("versions", True, f"{shown} at the training images' versions")
166
+
167
+
168
+ def refuse_code_failure(
169
+ what: str, error: BaseException, sources: Iterable[str] = ()
170
+ ) -> ConversionRefusedError:
171
+ """The refusal for the model's own code raising ``error`` while ``what`` ran.
172
+
173
+ ``sources`` are the model files run. When a training library they import is
174
+ installed here at another version than training's, that is the refusal.
175
+ """
176
+ libraries = imported_libraries(sources) or ("torch",)
177
+ clauses = version_mismatches(libraries)
178
+ if clauses:
179
+ return ConversionRefusedError(RefusalCause.VERSIONS, refusal_message(clauses))
180
+ pins = read_pins()
181
+ named = ", ".join(f"{name} {pinned_version(name, pins)}" for name in libraries)
182
+ detail = " ".join(_SOURCE_FILE.sub("", format_exception(error)).split())[:300]
183
+ advice = "fix the code so it runs under them"
184
+ if "transformers" in libraries:
185
+ advice += ", or bring a checkpoint transformers reads without its own code"
186
+ return ConversionRefusedError(
187
+ RefusalCause.CODE_FAILS,
188
+ f"{what} raised {detail}: the model's own code does not run under the "
189
+ f"training versions ({named}); {advice}",
190
+ )
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tracebloc
3
- Version: 1.2.74
3
+ Version: 1.2.76
4
4
  Summary: Build AI with anyone. On data that can't move. SDK for the tracebloc collaborative AI workspace.
5
5
  Author-email: tracebloc <lukas@tracebloc.io>
6
6
  License-Expression: MIT
@@ -52,6 +52,7 @@ tracebloc/convert/families/__init__.py
52
52
  tracebloc/convert/families/tabular.py
53
53
  tracebloc/convert/families/hf/__init__.py
54
54
  tracebloc/convert/families/hf/_build.py
55
+ tracebloc/convert/families/hf/_model_type.py
55
56
  tracebloc/convert/families/vision/__init__.py
56
57
  tracebloc/convert/families/vision/_matching.py
57
58
  tracebloc/convert/licence/__init__.py
File without changes
File without changes
File without changes
File without changes
File without changes