hypernix 0.45.0__tar.gz → 0.45.2__tar.gz

{hypernix-0.45.0 → hypernix-0.45.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hypernix
-Version: 0.45.0
+Version: 0.45.2
 Summary: Download and quantize the HyperNix PyTorch model to GGUF (fp32/fp16/Q8_0/Q6_K/Q4_K_M).
 Home-page: https://github.com/minerofthesoal/hypernix-pip
 Author: HyperNix contributors

{hypernix-0.45.0 → hypernix-0.45.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hypernix"
-version = "0.45.0"
+version = "0.45.2"
 description = "Download and quantize the HyperNix PyTorch model to GGUF (fp32/fp16/Q8_0/Q6_K/Q4_K_M)."
 readme = { file = "README.md", content-type = "text/markdown" }
 license = "Apache-2.0"

{hypernix-0.45.0 → hypernix-0.45.2}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = hypernix
-version = 0.45.0
+version = 0.45.2
 description = Download and quantize the HyperNix PyTorch model to GGUF (fp32/fp16/Q8_0/Q6_K/Q4_K_M).
 long_description = file: README.md
 long_description_content_type = text/markdown

{hypernix-0.45.0 → hypernix-0.45.2}/src/hypernix/__init__.py

@@ -157,5 +157,5 @@ __all__ = [
     "verify_snapshot",
 ]
 
-__version__ = "0.45.0"
+__version__ = "0.45.2"
 DEFAULT_REPO_ID = "ray0rf1re/hyper-nix.1"

{hypernix-0.45.0 → hypernix-0.45.2}/src/hypernix/pans.py

@@ -25,6 +25,7 @@ import re
 from collections.abc import Iterable, Iterator
 from dataclasses import dataclass, field
 from pathlib import Path
+from typing import Any, ClassVar
 
 _WS_RE = re.compile(r"[ \t]+")
 _MULTINEWLINE_RE = re.compile(r"\n{3,}")
@@ -37,10 +38,41 @@ _MULTINEWLINE_RE = re.compile(r"\n{3,}")
 @dataclass
 class Pan:
     """Abstract pan.  Subclasses override :meth:`cook` (one line in,
-    zero-or-one line out) or :meth:`iter` (full iterator override)."""
+    zero-or-one line out) or :meth:`iter` (full iterator override).
+
+    ``name`` is deliberately a :class:`ClassVar` rather than a
+    ``@dataclass`` field — it's a human-readable label used in
+    ``__repr__`` / logs, not state.  Making it a field exposed it as
+    the *second positional argument* of every subclass, which meant
+    ``Skillet("file.txt", "instruct")`` silently set ``name="instruct"``
+    instead of ``mode="instruct"`` and left the caller confused.
+
+    Every pan accepts two *keyword-only* length caps:
+
+    * ``max_chars`` — hard per-line character cap.  Longer lines are
+      truncated (not split).
+    * ``context_length`` — convenience for training-script callers.
+      Treated as an approximate token budget and converted to
+      ``max_chars = context_length * 4`` (a reasonable English-BPE
+      heuristic).  Takes precedence over ``max_chars`` when both are
+      set.  For precise chunking by tokens / chars use
+      :mod:`hypernix.food_processor` (``SliceBlade`` / ``ShredBlade``)
+      instead.
+    """
 
     source: Path | str | Iterable[str]
-    name: str = "Pan"
+    max_chars: int | None = field(default=None, kw_only=True)
+    context_length: int | None = field(default=None, kw_only=True)
+    name: ClassVar[str] = "Pan"
+
+    # Rough chars-per-token ratio for English BPE tokenizers.  Only
+    # used when ``context_length`` is set and ``max_chars`` is not.
+    _CHARS_PER_TOKEN: ClassVar[int] = 4
+
+    def _effective_max_chars(self) -> int | None:
+        if self.context_length is not None:
+            return self.context_length * self._CHARS_PER_TOKEN
+        return self.max_chars
 
     def _source_lines(self) -> Iterator[str]:
         if isinstance(self.source, (str, Path)):
@@ -55,10 +87,14 @@ class Pan:
         return line
 
     def iter(self) -> Iterator[str]:
+        cap = self._effective_max_chars()
         for raw in self._source_lines():
             out = self.cook(raw)
-            if out is not None:
-                yield out
+            if out is None:
+                continue
+            if cap is not None and len(out) > cap:
+                out = out[:cap]
+            yield out
 
     def __iter__(self) -> Iterator[str]:
         return self.iter()
@@ -74,7 +110,7 @@ class FryingPan(Pan):
     through.  Useful when the input is already clean (already-formatted
     corpora, pre-tokenized chunks)."""
 
-    name: str = "FryingPan"
+    name: ClassVar[str] = "FryingPan"
 
     def cook(self, line: str) -> str | None:
         return line.rstrip()
@@ -90,7 +126,7 @@ class SaucePan(Pan):
     strip leading/trailing whitespace.  Standard mild cleaning for
     scraped or OCR'd text."""
 
-    name: str = "SaucePan"
+    name: ClassVar[str] = "SaucePan"
 
     def cook(self, line: str) -> str | None:
         line = _WS_RE.sub(" ", line.strip())
@@ -111,10 +147,10 @@ class Skillet(Pan):
     replies on the following line (or in a separate corpus).
     """
 
-    name: str = "Skillet"
     mode: str = "chat"            # "chat" | "instruct"
     user_tag: str = "<USER>"
     assistant_tag: str = "<ASSISTANT>"
+    name: ClassVar[str] = "Skillet"
 
     def cook(self, line: str) -> str | None:
         line = line.strip()
@@ -133,11 +169,15 @@ class Skillet(Pan):
 class GrillPan(Pan):
     """High direct heat.  SaucePan-style cleaning plus deduplication
     (SHA1 hash set) and a minimum length filter.  Safe on arbitrary
-    web-scraped text: collapses boilerplate, drops single-word lines."""
+    web-scraped text: collapses boilerplate, drops single-word lines.
+
+    ``_seen`` is internal dedupe state and is not part of the public
+    ``__init__`` signature.
+    """
 
-    name: str = "GrillPan"
     min_chars: int = 8
-    _seen: set[str] = field(default_factory=set, repr=False)
+    name: ClassVar[str] = "GrillPan"
+    _seen: set[str] = field(default_factory=set, repr=False, init=False)
 
     def cook(self, line: str) -> str | None:
         line = _WS_RE.sub(" ", line.strip())
@@ -162,13 +202,14 @@ class Wok(Pan):
     small corpora where order-randomization matters and you can afford
     the memory."""
 
-    name: str = "Wok"
     seed: int = 0
     reverse_ratio: float = 0.0
+    name: ClassVar[str] = "Wok"
 
     def iter(self) -> Iterator[str]:
         # Pre-clean with SaucePan semantics.
         rng = random.Random(self.seed)
+        cap = self._effective_max_chars()
         buffer: list[str] = []
         for raw in self._source_lines():
             line = _WS_RE.sub(" ", raw.strip())
@@ -176,10 +217,14 @@ class Wok(Pan):
                 buffer.append(line)
         rng.shuffle(buffer)
         for line in buffer:
-            if self.reverse_ratio > 0 and rng.random() < self.reverse_ratio:
-                yield " ".join(reversed(line.split()))
-            else:
-                yield line
+            out = (
+                " ".join(reversed(line.split()))
+                if self.reverse_ratio > 0 and rng.random() < self.reverse_ratio
+                else line
+            )
+            if cap is not None and len(out) > cap:
+                out = out[:cap]
+            yield out
 
 
 # ---------------------------------------------------------------------------
@@ -195,7 +240,29 @@ TIERS: dict[str, type[Pan]] = {
 }
 
 
-def pick_pan(tier: str, source: Path | str | Iterable[str], **kwargs: str) -> Pan:
-    """Return a pan instance by tier name (``"frying-pan"``..``"wok"``)."""
-    cls = TIERS[tier.lower().replace("_", "-")]
-    return cls(source=source, **kwargs)
+def pick_pan(tier: str, source: Path | str | Iterable[str], **kwargs: Any) -> Pan:
+    """Return a pan instance by tier name (``"frying-pan"``..``"wok"``).
+
+    Raises :class:`ValueError` if ``tier`` is unknown (with the list of
+    valid tiers in the message) or if ``**kwargs`` contains a keyword
+    the selected pan doesn't accept (with the list of valid kwargs).
+    """
+    key = tier.lower().replace("_", "-")
+    if key not in TIERS:
+        raise ValueError(
+            f"unknown pan tier {tier!r}; valid tiers are: {sorted(TIERS)}"
+        )
+    cls = TIERS[key]
+    try:
+        return cls(source=source, **kwargs)
+    except TypeError as exc:
+        # Re-raise with a message that lists the keywords the tier
+        # actually accepts — the raw dataclass message ("unexpected
+        # keyword argument 'X'") doesn't say what *is* accepted.
+        import inspect
+        valid = [p for p in inspect.signature(cls).parameters if p != "source"]
+        raise ValueError(
+            f"{cls.__name__} rejected an argument ({exc}). "
+            f"Valid keyword arguments for {cls.__name__} (besides "
+            f"'source'): {valid}"
+        ) from exc

{hypernix-0.45.0 → hypernix-0.45.2}/tests/test_kitchen_v044.py

@@ -89,6 +89,112 @@ def test_pick_pan_factory() -> None:
         assert isinstance(p, pans.Pan)
 
 
+# Regression: second positional arg used to silently bind to `name`
+# (inherited from Pan.__init__), so ``Skillet(src, "instruct")`` would
+# set ``name="instruct"`` and leave ``mode="chat"``.  `name` is now a
+# ClassVar, so positional arg #2 is the first real field on each pan.
+
+def test_skillet_second_positional_is_mode_not_name() -> None:
+    from hypernix import pans
+
+    s = pans.Skillet(["hi"], "instruct")
+    assert s.mode == "instruct"
+    assert s.name == "Skillet"   # unchanged — name is a ClassVar
+
+
+def test_grill_pan_second_positional_is_min_chars() -> None:
+    from hypernix import pans
+
+    g = pans.GrillPan(["hello", "world", "hi", "hello"], 4)
+    assert g.min_chars == 4
+    # "hi" (2 chars) dropped; second "hello" dropped as duplicate.
+    assert list(g) == ["hello", "world"]
+
+
+def test_grill_pan_seen_is_not_in_init_signature() -> None:
+    import inspect
+
+    from hypernix import pans
+
+    params = inspect.signature(pans.GrillPan).parameters
+    assert "_seen" not in params
+
+
+def test_pick_pan_unknown_tier_gives_useful_error() -> None:
+    import pytest
+
+    from hypernix import pans
+
+    with pytest.raises(ValueError, match="unknown pan tier"):
+        pans.pick_pan("microwave-pan", source=["x"])
+
+
+def test_pick_pan_bad_kwarg_lists_valid_ones() -> None:
+    import pytest
+
+    from hypernix import pans
+
+    with pytest.raises(ValueError, match="Skillet rejected"):
+        pans.pick_pan("skillet", source=["x"], min_chars=16)
+
+
+# Regression: v0.45 shipped without a context_length / max_chars knob,
+# so a training script calling ``FryingPan(context_length=256)`` got
+# a bare TypeError.  Every pan now accepts both as keyword-only fields.
+
+def test_frying_pan_accepts_context_length() -> None:
+    from hypernix import pans
+
+    long_line = "x" * 10_000
+    out = list(pans.FryingPan(source=[long_line], context_length=256))
+    # context_length=256 → max_chars = 256 * 4 = 1024.
+    assert len(out[0]) == 1024
+
+
+def test_pan_max_chars_truncates() -> None:
+    from hypernix import pans
+
+    long_line = "x" * 10_000
+    out = list(pans.SaucePan(source=[long_line], max_chars=500))
+    assert len(out[0]) == 500
+
+
+def test_pan_context_length_wins_over_max_chars() -> None:
+    from hypernix import pans
+
+    long_line = "x" * 10_000
+    out = list(pans.FryingPan(
+        source=[long_line], max_chars=99, context_length=50,
+    ))
+    # ctx=50 → 50 * 4 = 200, not 99.
+    assert len(out[0]) == 200
+
+
+def test_pan_length_cap_is_noop_when_unset() -> None:
+    from hypernix import pans
+
+    line = "x" * 10_000
+    out = list(pans.FryingPan(source=[line]))
+    assert len(out[0]) == 10_000
+
+
+def test_wok_honors_context_length() -> None:
+    from hypernix import pans
+
+    lines = ["x" * 10_000, "y" * 10_000]
+    out = list(pans.Wok(source=lines, seed=0, context_length=64))
+    # 64 * 4 = 256
+    assert all(len(s) <= 256 for s in out)
+
+
+def test_pick_pan_forwards_context_length() -> None:
+    from hypernix import pans
+
+    src = ["abcd " * 1000]
+    pan = pans.pick_pan("grill-pan", source=src, context_length=64)
+    assert all(len(s) <= 256 for s in pan)
+
+
 # ---------------------------------------------------------------------------
 # microwave
 # ---------------------------------------------------------------------------