ellf-recipes-gpu 0.1.1__tar.gz

ellf_recipes_gpu-0.1.1/LICENSE

@@ -0,0 +1,3 @@
+All Rights Reserved
+
+Copyright (C) 2023 ExplosionAI GmbH

ellf_recipes_gpu-0.1.1/MANIFEST.in

@@ -0,0 +1,2 @@
+include LICENSE
+include requirements.in

ellf_recipes_gpu-0.1.1/PKG-INFO

@@ -0,0 +1,23 @@
+Metadata-Version: 2.4
+Name: ellf-recipes-gpu
+Version: 0.1.1
+Summary: GPU recipes for Ellf (spaCy curated transformers training & inference)
+Home-page: https://prodi.gy
+Author: ExplosionAI GmbH
+Author-email: contact@explosion.ai
+License: All rights reserved
+License-File: LICENSE
+Requires-Dist: ellf-recipes
+Requires-Dist: ellf_recipes_sdk
+Requires-Dist: huggingface-hub>=0.20.0
+Requires-Dist: prodigy-company-plugins
+Requires-Dist: prodigy>=1.16.0
+Requires-Dist: spacy-curated-transformers<1.0.0,>=0.3.0
+Requires-Dist: transformers>=4.30.0
+Dynamic: author
+Dynamic: author-email
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: summary

ellf_recipes_gpu-0.1.1/ellf_recipes_gpu/__init__.py

@@ -0,0 +1,3 @@
+from .about import __version__
+
+__all__ = ["__version__"]

ellf_recipes_gpu-0.1.1/ellf_recipes_gpu/about.json

@@ -0,0 +1,13 @@
+{
+  "name": "ellf-recipes-gpu",
+  "version": "0.1.1",
+  "summary": "GPU recipes for Ellf (spaCy curated transformers training & inference)",
+  "uri": "https://prodi.gy",
+  "prog": "ptr",
+  "package_data": [],
+  "entry_points": {
+    "ellf_recipes": [
+      "ellf_recipes_gpu = ellf_recipes_gpu.recipes"
+    ]
+  }
+}

ellf_recipes_gpu-0.1.1/ellf_recipes_gpu/about.py

@@ -0,0 +1,23 @@
+"""
+This file originates in shared/shared_about.py, and
+is copied verbatim into the different projects. It should
+not be edited directly, edit the shared version and replicate
+it everywhere.
+"""
+
+import json
+from pathlib import Path
+
+PWD = Path(__file__).parent
+
+
+with (PWD / "about.json").open("r", encoding="utf8") as file_:
+    _about_data = json.load(file_)
+
+
+__name__ = _about_data["name"]
+__title__ = _about_data.get("title", "")
+__description__ = _about_data.get("description", "")
+__summary__ = _about_data.get("summary", "")
+__version__ = _about_data["version"]
+__prog__ = _about_data.get("prog", "")

ellf_recipes_gpu-0.1.1/ellf_recipes_gpu/recipes/__init__.py

@@ -0,0 +1,11 @@
+# isort: skip_file
+# The import order determines the order of recipes shown in the UI
+from . import (
+    train_curated_transformer,
+    apply_curated_transformer,
+)
+
+__all__ = [
+    "train_curated_transformer",
+    "apply_curated_transformer",
+]

ellf_recipes_gpu-0.1.1/ellf_recipes_gpu/recipes/apply_curated_transformer.py

@@ -0,0 +1,118 @@
+from typing import Any, Dict, List, Union
+
+import spacy
+from prodigy.errors import RecipeError
+from prodigy.util import msg
+
+from ellf_recipes_sdk import (
+    Input,
+    InputDataset,
+    IntProps,
+    Model,
+    TextProps,
+    action_recipe,
+    save_examples,
+)
+
+
+def _doc_predictions(doc) -> Dict[str, Any]:  # type: ignore[no-untyped-def]
+    """Extract a generic, component-agnostic prediction payload from a Doc.
+
+    Deliberately simple rather than reproducing every task-specific
+    Prodigy annotation format (span boundaries, textcat "accept" shape,
+    etc.) for every possible component — this is a batch-inference
+    action recipe, not an annotation task recipe, so its output is meant
+    to be inspected or consumed downstream rather than re-annotated.
+    """
+    predictions: Dict[str, Any] = {}
+    if doc.ents:
+        predictions["ents"] = [
+            {"start": ent.start_char, "end": ent.end_char, "label": ent.label_}
+            for ent in doc.ents
+        ]
+    if doc.spans:
+        predictions["spans"] = {
+            key: [
+                {"start": span.start_char, "end": span.end_char, "label": span.label_}
+                for span in spans
+            ]
+            for key, spans in doc.spans.items()
+        }
+    if doc.cats:
+        predictions["cats"] = dict(doc.cats)
+    return predictions
+
+
+@action_recipe(
+    title="Apply a spacy-curated-transformers model (GPU)",
+    description=(
+        "Run inference with a trained spacy-curated-transformers model "
+        "over an input dataset or asset, writing predictions to an "
+        "output dataset."
+    ),
+    field_props={
+        # fmt: off
+        "model": TextProps(
+            title="Trained model",
+            description="Model asset produced by train-curated-transformer",
+        ),
+        "gpu_id": IntProps(
+            title="GPU device id", description="Use 0 for the first GPU"
+        ),
+        # fmt: on
+    },
+    resources={"gpu": 1},
+)
+def apply_curated_transformer(
+    *,
+    model: Model,
+    input: Union[Input, InputDataset],
+    output_dataset: str,
+    gpu_id: int = 0,
+    batch_size: int = 32,
+) -> None:
+    """Run batch inference with a trained spacy-curated-transformers model.
+
+    Writes ``{"text": ..., "predictions": {...}}`` rows to
+    ``output_dataset`` — see ``_doc_predictions`` for the prediction
+    shape. Predictions are model output, not user annotation, so they're
+    saved as ``server_ann_keys`` rather than ``user_ann_keys``.
+
+    Contract:
+        Preconditions:
+            - Every example loaded from ``input`` must have a ``"text"``
+              key; a missing one raises ``RecipeError``.
+
+        Raises:
+            RecipeError: if an input example has no ``"text"`` key.
+            Various spaCy/torch errors (not caught) if the requested GPU
+            device is unavailable or the model fails to load.
+    """
+    if not spacy.prefer_gpu(gpu_id):  # pyright: ignore[reportPrivateImportUsage]
+        msg.warn(
+            f"Could not activate GPU {gpu_id} — falling back to CPU. "
+            "Inference will be slower than expected."
+        )
+    nlp = model.load()
+
+    texts: List[str] = []
+    for eg in input.load():
+        text = eg.get("text")
+        if not text:
+            raise RecipeError("Every input example must have a non-empty 'text' field.")
+        texts.append(text)
+
+    examples = []
+    for text, doc in zip(texts, nlp.pipe(texts, batch_size=batch_size)):
+        examples.append({"text": text, "predictions": _doc_predictions(doc)})
+
+    save_examples(
+        output_dataset,
+        examples,
+        input_keys=["text"],
+        server_ann_keys=["predictions"],
+    )
+    msg.good(f"Wrote {len(examples)} predictions to dataset '{output_dataset}'")
+
+
+__all__ = ["apply_curated_transformer"]

ellf_recipes_gpu-0.1.1/ellf_recipes_gpu/recipes/train_curated_transformer.py

@@ -0,0 +1,374 @@
+import tempfile
+from enum import Enum
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+import prodigy.recipes.train as prodigy_train
+import spacy
+from prodigy.errors import RecipeError
+from spacy.cli.init_config import init_config as spacy_init_config
+from spacy.cli.init_config import save_config
+from spacy.util import load_config
+from thinc.api import Config
+
+from ellf_recipes.recipes.train import (
+    COMPONENTS,
+    Data,
+    OutputSettings,
+    _cleanup_temp_datasets,
+    _get_pam_context,
+    _load_meta_json,
+    _parse_config_overrides,
+    _preflight_outputs,
+    _save_outputs,
+)
+from ellf_recipes_sdk import (
+    BoolProps,
+    FloatProps,
+    IntProps,
+    ListProps,
+    TextProps,
+    action_recipe,
+)
+
+
+class HfModelType(Enum):
+    """Hugging Face model families spacy-curated-transformers can load.
+
+    Deliberately narrower than the 5 architectures curated-transformers
+    registers (it also has ``AlbertTransformer.v1``): its HF-tokenizer
+    loader (``tokenization/hf_loader.py``) only converts Bert/Roberta/
+    XLMRoberta/Camembert fast tokenizers, so a real Albert checkpoint
+    would register cleanly but fail at model-init time. Leaving it out
+    here avoids shipping a family that looks supported but isn't.
+    """
+
+    bert = "bert"
+    roberta = "roberta"
+    camembert = "camembert"
+    xlm_roberta = "xlm-roberta"
+
+
+# (curated-transformer architecture, piece-encoder architecture) per HF
+# model family — from spacy-curated-transformers' own setup.cfg entry
+# points, cross-checked against tokenization/hf_loader.py's tokenizer
+# dispatch so the pairing is actually loadable, not just registered.
+_ARCHITECTURE_BY_MODEL_TYPE: Dict[HfModelType, Tuple[str, str]] = {
+    HfModelType.bert: (
+        "spacy-curated-transformers.BertTransformer.v1",
+        "spacy-curated-transformers.BertWordpieceEncoder.v1",
+    ),
+    HfModelType.roberta: (
+        "spacy-curated-transformers.RobertaTransformer.v1",
+        "spacy-curated-transformers.ByteBpeEncoder.v1",
+    ),
+    HfModelType.camembert: (
+        "spacy-curated-transformers.CamembertTransformer.v1",
+        "spacy-curated-transformers.CamembertSentencepieceEncoder.v1",
+    ),
+    HfModelType.xlm_roberta: (
+        "spacy-curated-transformers.XlmrTransformer.v1",
+        "spacy-curated-transformers.XlmrSentencepieceEncoder.v1",
+    ),
+}
+
+TRANSFORMER_PIPE_NAME = "transformer"
+
+
+def _default_curated_transformer_base_config(
+    pipes: Dict[str, Tuple[List[str], List[str]]], lang: str
+) -> Config:
+    """Build a tok2vec-listener-based default config for ``pipes``, always
+    using spaCy's "accuracy" preset rather than
+    ``prodigy.recipes.train.generate_default_config``'s default choice of
+    "efficiency" for blank models.
+
+    Efficiency-mode uses architectures that skip the tok2vec listener
+    entirely for some components (e.g. textcat's bag-of-words
+    ``TextCatBOW.v3``) — there would be nothing for
+    ``_install_curated_transformer`` to redirect at the transformer.
+    Accuracy-mode consistently wires every supported component through a
+    ``model.tok2vec`` listener (verified against the installed spaCy:
+    ner/spancat/tagger/parser/senter/textcat/textcat_multilabel all use
+    it under "accuracy"), which is what makes the swap possible. This
+    also matches practice: pairing an efficiency-oriented head with an
+    already-expensive transformer forward pass isn't a combination
+    anyone actually wants.
+    """
+    config = spacy_init_config(
+        lang=lang, pipeline=list(pipes), optimize="accuracy", gpu=False
+    )
+    if "ner" in config["components"]:
+        config["components"]["ner"]["incorrect_spans_key"] = (
+            prodigy_train.NER_DEFAULT_INCORRECT_KEY
+        )
+    if "spancat" in config["components"]:
+        config["components"]["spancat"]["spans_key"] = prodigy_train.SPANCAT_DEFAULT_KEY
+        prefix = f"spans_{prodigy_train.SPANCAT_DEFAULT_KEY}"
+        config["training"]["score_weights"][f"{prefix}_f"] = 1.0
+        config["training"]["score_weights"][f"{prefix}_p"] = 0.0
+        config["training"]["score_weights"][f"{prefix}_r"] = 0.0
+        train_sets, dev_sets = pipes["spancat"]
+        db = prodigy_train.connect()
+        examples = prodigy_train.load_examples(db, [*train_sets, *dev_sets])
+        suggester = prodigy_train.infer_spancat_suggester(examples, spacy.blank(lang))
+        config["components"]["spancat"]["suggester"] = suggester
+    return config
+
+
+def _install_curated_transformer(
+    config: Config, *, pipes: List[str], hf_model_type: HfModelType
+) -> None:
+    """Replace ``config``'s tok2vec pipe with a curated-transformer
+    component in place, redirecting every downstream component's
+    listener at it.
+
+    Raises ``RecipeError`` for any requested component whose default
+    "accuracy" architecture doesn't use a ``model.tok2vec`` listener —
+    there would be nothing correct to redirect.
+    """
+    transformer_arch, piece_encoder_arch = _ARCHITECTURE_BY_MODEL_TYPE[hf_model_type]
+    pipeline = list(config["nlp"]["pipeline"])
+    if "tok2vec" in pipeline:
+        pipeline.remove("tok2vec")
+        config["components"].pop("tok2vec", None)
+    if TRANSFORMER_PIPE_NAME not in pipeline:
+        pipeline.insert(0, TRANSFORMER_PIPE_NAME)
+    config["nlp"]["pipeline"] = pipeline
+    config["components"][TRANSFORMER_PIPE_NAME] = {
+        "factory": "curated_transformer",
+        "model": {
+            "@architectures": transformer_arch,
+            # Placeholder — overwritten by init_fill_config_curated_transformer
+            # (via _fill_curated_transformer_from_hf below) with the real HF
+            # model config's vocab_size, hidden_width, num_hidden_layers, etc.
+            "vocab_size": 1,
+            "piece_encoder": {"@architectures": piece_encoder_arch},
+            "with_spans": {
+                "@architectures": "spacy-curated-transformers.WithStridedSpans.v1",
+                "stride": 96,
+                "window": 128,
+            },
+        },
+    }
+    for component in pipes:
+        model = config["components"][component]["model"]
+        if "tok2vec" not in model:
+            raise RecipeError(
+                f"Component {component!r} has no tok2vec listener to redirect "
+                f"to the curated transformer (its default architecture is "
+                f"{model.get('@architectures')!r}). GPU training via "
+                f"spacy-curated-transformers currently supports components "
+                f"whose default config wires a tok2vec listener."
+            )
+        model["tok2vec"] = {
+            "@architectures": "spacy-curated-transformers.LastTransformerLayerListener.v1",
+            # Placeholder — corrected below to match the transformer's real
+            # hidden width, once known.
+            "width": 768,
+            "pooling": {"@layers": "reduce_mean.v1"},
+            "upstream": "*",
+        }
+
+
+def _fill_curated_transformer_from_hf(
+    config: Config, *, hf_model_name: str, hf_model_revision: str
+) -> Config:
+    """Fill the transformer component's model hyperparameters (vocab
+    size, hidden width, layer count, ...) from the real Hugging Face
+    model config, and set its encoder/piece-encoder loaders to pull
+    weights from ``hf_model_name`` at load time.
+
+    Delegates to spacy-curated-transformers' own ``init fill-curated-
+    transformer`` implementation rather than re-deriving the HF-config
+    field mapping here — that mapping is version-specific to the
+    installed spacy-curated-transformers and re-implementing it would
+    drift out of sync silently. It also validates the HF model's actual
+    type against the architecture chosen in ``_install_curated_transformer``
+    and fails with a clear message on mismatch (e.g. picking "roberta"
+    but naming a BERT checkpoint).
+
+    Only fills the transformer's own ``model`` block — listener
+    ``width`` on every downstream component must still be propagated
+    from the resolved ``hidden_width`` by the caller, since this step
+    doesn't know about them.
+    """
+    from spacy_curated_transformers.cli.fill_config_transformer import (
+        init_fill_config_curated_transformer,
+    )
+
+    with tempfile.NamedTemporaryFile(suffix=".cfg", mode="w", delete=False) as tmp:
+        tmp_path = Path(tmp.name)
+    try:
+        save_config(config, tmp_path, is_stdout=False, silent=True)
+        init_fill_config_curated_transformer(
+            tmp_path,
+            tmp_path,
+            cli_model_name=hf_model_name,
+            cli_model_revision=hf_model_revision,
+            cli_transformer_name=TRANSFORMER_PIPE_NAME,
+            config_overrides={},
+        )
+        filled = load_config(tmp_path)
+    finally:
+        tmp_path.unlink(missing_ok=True)
+
+    hidden_width = filled["components"][TRANSFORMER_PIPE_NAME]["model"]["hidden_width"]
+    for component in filled["components"].values():
+        model = component.get("model", {})
+        tok2vec = model.get("tok2vec")
+        if (
+            isinstance(tok2vec, dict)
+            and tok2vec.get("@architectures")
+            == "spacy-curated-transformers.LastTransformerLayerListener.v1"
+        ):
+            tok2vec["width"] = hidden_width
+    return filled
+
+
+@action_recipe(
+    title="Train a spaCy pipeline with a curated transformer (GPU)",
+    description=(
+        "Train a spaCy pipeline using spacy-curated-transformers for GPU-"
+        "accelerated training and inference, loading weights from a "
+        "Hugging Face checkpoint."
+    ),
+    field_props={
+        # fmt: off
+        "hf_model_name": TextProps(
+            title="Hugging Face model name",
+            description="e.g. 'roberta-base', 'bert-base-cased', 'xlm-roberta-base'",
+        ),
+        "hf_model_revision": TextProps(
+            title="Hugging Face model revision",
+            description="Branch or commit to load. Defaults to 'main'.",
+        ),
+        "lang": TextProps(title="Language code", description="e.g. 'en'"),
+        "gpu_id": IntProps(
+            title="GPU device id", description="Use 0 for the first GPU"
+        ),
+        "eval_split": FloatProps(
+            title="Portion of examples to split off for evaluation",
+            description="Applied if no dedicated evaluation datasets are provided for a component.",
+            min=0.0,
+            max=1.0,
+            step=0.05,
+        ),
+        "label_stats": BoolProps(title="Show per-label scores"),
+        "verbose": BoolProps(title="Enable verbose logging"),
+        "config_overrides": ListProps(
+            title="Config overrides",
+            description="Override config values for training",
+            exists=False,
+            min=0,
+        ),
+        # fmt: on
+    },
+    cli_names={
+        **{f"data.{c}.training-input-dataset": f"{c}.train" for c in COMPONENTS},
+        **{f"data.{c}.training-input": f"{c}.train-asset" for c in COMPONENTS},
+        **{f"data.{c}.evaluation-input-dataset": f"{c}.eval" for c in COMPONENTS},
+        **{f"data.{c}.evaluation-input": f"{c}.eval-asset" for c in COMPONENTS},
+    },
+    resources={"gpu": 1},
+)
+def train_curated_transformer(
+    *,
+    data: Data,
+    output: OutputSettings,
+    hf_model_type: HfModelType,
+    hf_model_name: str,
+    hf_model_revision: str = "main",
+    lang: str = "en",
+    gpu_id: int = 0,
+    eval_split: float = 0.2,
+    label_stats: bool = False,
+    verbose: bool = False,
+    config_overrides: Optional[List[str]] = None,
+) -> None:
+    """Train a spaCy pipeline with a spacy-curated-transformers component
+    on GPU, loading weights from a Hugging Face checkpoint.
+
+    Mirrors the base ``ellf_recipes`` ``train`` recipe's data handling,
+    validation, and output persistence, but builds a curated-transformer
+    config instead of a tok2vec one and trains with a real ``gpu_id``
+    instead of the CPU recipe's hardcoded ``-1``.
+
+    Contract:
+        Preconditions:
+            - Same as ``ellf_recipes.recipes.train.train``: ``data`` must
+              contain at least one non-None component; ``output.name``
+              required unless ``output.skip``; each ``config_overrides``
+              entry must contain ``"="``.
+            - ``hf_model_type`` must match the actual type of the model at
+              ``hf_model_name`` (validated against the real HF config
+              during the fill-in step; raises a clear error on mismatch).
+            - Every requested component's default "accuracy" architecture
+              must use a ``model.tok2vec`` listener (raises ``RecipeError``
+              otherwise — see ``_install_curated_transformer``).
+
+        Raises:
+            RecipeError: on the precondition failures above, or from
+                spacy-curated-transformers' own HF-loading errors
+                (unreachable model name/revision, unsupported HF
+                tokenizer type).
+    """
+    output.validate()
+    config_overrides_dict = _parse_config_overrides(config_overrides or [])
+    prodigy_train.set_log_level(verbose=verbose)
+    pam_client = None
+    cluster_id = None
+    if not output.skip:
+        pam_client, cluster_id = _get_pam_context()
+        _preflight_outputs(output, pam_client, cluster_id)
+    pipes, temp_datasets = data.materialize()
+    try:
+        if not pipes:
+            raise RecipeError("No components to train")
+        base_config = _default_curated_transformer_base_config(pipes, lang)
+        _install_curated_transformer(
+            base_config, pipes=list(pipes), hf_model_type=hf_model_type
+        )
+        transformer_config = _fill_curated_transformer_from_hf(
+            base_config,
+            hf_model_name=hf_model_name,
+            hf_model_revision=hf_model_revision,
+        )
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            tmp_dir = Path(tmp_dir)
+            train_config = prodigy_train._prodigy_config(
+                pipes,
+                None,
+                lang=lang,
+                eval_split=eval_split,
+                config=transformer_config,
+                verbose=verbose,
+            )
+            prodigy_train._train(
+                train_config,
+                output_dir=tmp_dir,
+                overrides=config_overrides_dict,
+                gpu_id=gpu_id,
+                show_label_stats=label_stats,
+            )
+            if output.skip:
+                prodigy_train.msg.warn(
+                    "Skipping output registration. Model and results were NOT "
+                    "saved. Re-run with 'Skip saving outputs' unticked to "
+                    "persist them."
+                )
+            else:
+                assert pam_client is not None and cluster_id is not None
+                model_best = tmp_dir / "model-best"
+                _save_outputs(
+                    output,
+                    pam_client,
+                    cluster_id,
+                    model_dir=model_best,
+                    results=_load_meta_json(model_best),
+                )
+    finally:
+        _cleanup_temp_datasets(temp_datasets)
+
+
+__all__ = ["train_curated_transformer", "HfModelType"]

ellf_recipes_gpu-0.1.1/ellf_recipes_gpu.egg-info/PKG-INFO

@@ -0,0 +1,23 @@
+Metadata-Version: 2.4
+Name: ellf-recipes-gpu
+Version: 0.1.1
+Summary: GPU recipes for Ellf (spaCy curated transformers training & inference)
+Home-page: https://prodi.gy
+Author: ExplosionAI GmbH
+Author-email: contact@explosion.ai
+License: All rights reserved
+License-File: LICENSE
+Requires-Dist: ellf-recipes
+Requires-Dist: ellf_recipes_sdk
+Requires-Dist: huggingface-hub>=0.20.0
+Requires-Dist: prodigy-company-plugins
+Requires-Dist: prodigy>=1.16.0
+Requires-Dist: spacy-curated-transformers<1.0.0,>=0.3.0
+Requires-Dist: transformers>=4.30.0
+Dynamic: author
+Dynamic: author-email
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: summary

ellf_recipes_gpu-0.1.1/ellf_recipes_gpu.egg-info/SOURCES.txt

@@ -0,0 +1,19 @@
+LICENSE
+MANIFEST.in
+pyproject.toml
+requirements.in
+setup.cfg
+setup.py
+ellf_recipes_gpu/__init__.py
+ellf_recipes_gpu/about.json
+ellf_recipes_gpu/about.py
+ellf_recipes_gpu.egg-info/PKG-INFO
+ellf_recipes_gpu.egg-info/SOURCES.txt
+ellf_recipes_gpu.egg-info/dependency_links.txt
+ellf_recipes_gpu.egg-info/entry_points.txt
+ellf_recipes_gpu.egg-info/not-zip-safe
+ellf_recipes_gpu.egg-info/requires.txt
+ellf_recipes_gpu.egg-info/top_level.txt
+ellf_recipes_gpu/recipes/__init__.py
+ellf_recipes_gpu/recipes/apply_curated_transformer.py
+ellf_recipes_gpu/recipes/train_curated_transformer.py

ellf_recipes_gpu-0.1.1/ellf_recipes_gpu.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

ellf_recipes_gpu-0.1.1/ellf_recipes_gpu.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[ellf_recipes]
+ellf_recipes_gpu = ellf_recipes_gpu.recipes

ellf_recipes_gpu-0.1.1/ellf_recipes_gpu.egg-info/not-zip-safe

@@ -0,0 +1 @@
+

ellf_recipes_gpu-0.1.1/ellf_recipes_gpu.egg-info/requires.txt

@@ -0,0 +1,7 @@
+ellf-recipes
+ellf_recipes_sdk
+huggingface-hub>=0.20.0
+prodigy-company-plugins
+prodigy>=1.16.0
+spacy-curated-transformers<1.0.0,>=0.3.0
+transformers>=4.30.0

ellf_recipes_gpu-0.1.1/ellf_recipes_gpu.egg-info/top_level.txt

@@ -0,0 +1 @@
+ellf_recipes_gpu

ellf_recipes_gpu-0.1.1/pyproject.toml

@@ -0,0 +1,9 @@
+[build-system]
+build-backend = "setuptools.build_meta"
+requires = ["setuptools", "wheel"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.pyright]
+extraPaths = [".", "../sdk_recipes"]

ellf_recipes_gpu-0.1.1/requirements.in

@@ -0,0 +1,9 @@
+# This is a generated file! Do not edit directly.
+# Instead run: pdcli repo write-requirements
+ellf-recipes
+ellf_recipes_sdk
+huggingface-hub>=0.20.0
+prodigy-company-plugins
+prodigy>=1.16.0
+spacy-curated-transformers<1.0.0,>=0.3.0
+transformers>=4.30.0

ellf_recipes_gpu-0.1.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

ellf_recipes_gpu-0.1.1/setup.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+"""
+This file originates in shared/shared_setup.py, and
+is copied verbatim into the different projects. It should
+not be edited directly, edit the shared version and replicate
+it everywhere.
+"""
+
+import json
+from pathlib import Path
+from typing import Any, Dict, List
+
+from setuptools import find_packages, setup
+
+PWD = Path(__file__).parent
+EXCLUDED_PACKAGE_DATA_PARTS = {".git", "__pycache__", "tests", "backups", "evals"}
+EXCLUDED_PACKAGE_DATA_FILENAMES = {"CLAUDE.md", "README.md"}
+
+
+def read_about(path: Path) -> Dict[str, Any]:
+    with (path / "about.json").open("r", encoding="utf8") as file_:
+        data = json.loads(file_.read())
+    return data
+
+
+def read_requirements(path: Path) -> List[str]:
+    # Read in requirements and split into packages and URLs
+    requirements_path = path / "requirements.in"
+    with requirements_path.open("r", encoding="utf8") as f:
+        requirements = [line.strip() for line in f]
+    return requirements
+
+
+def should_include_package_data(path: Path, package_path: Path) -> bool:
+    rel_path = path.relative_to(package_path)
+    if not path.is_file():
+        return False
+    if any(part in EXCLUDED_PACKAGE_DATA_PARTS for part in rel_path.parts):
+        return False
+    if path.name in EXCLUDED_PACKAGE_DATA_FILENAMES:
+        return False
+    if path.suffix == ".pyc":
+        return False
+    return True
+
+
+def setup_package():
+    root = PWD.resolve()
+    packages = [
+        p for p in find_packages() if not p.startswith("tests") and p != "migrations"
+    ]
+    package_name = packages[0]
+    package_path = root / package_name
+    about = read_about(root / package_name)
+    requirements = read_requirements(root)
+    version = about["version"]
+    package_data = about.get("package_data", [])
+    package_data_dirs = about.get("package_data_dirs", [])
+    for directory in package_data_dirs:
+        # Package data doesn't properly support recursive globs, so we fix them.
+        package_data.extend(
+            [
+                str(path.relative_to(package_path))
+                for path in (package_path / directory).glob("**/*")
+                if should_include_package_data(path, package_path)
+            ]
+        )
+    # Ensure the about.json is packaged
+    if "about.json" not in package_data:
+        package_data.append("about.json")
+    setup(
+        author=about.get("author", "ExplosionAI GmbH"),
+        author_email=about.get("email", "contact@explosion.ai"),
+        url=about.get("uri", "https://github.com/explosion/ellf"),
+        license=about.get("license", "All rights reserved"),
+        name=about["name"],
+        description=about["summary"],
+        version=version,
+        packages=packages,
+        package_data={package_name: package_data},
+        entry_points=about.get("entry_points", {}),
+        install_requires=requirements,
+        scripts=[],
+        zip_safe=False,
+    )
+
+
+if __name__ == "__main__":
+    setup_package()