cat-stack 1.6.7__tar.gz → 1.6.9__tar.gz

{cat_stack-1.6.7 → cat_stack-1.6.9}/.gitignore

@@ -41,3 +41,11 @@ survey-summarizer
 # R package build/check artifacts
 *.Rcheck/
 *.tar.gz
+
+# Local experiment scratch (not part of the package)
+/patches/
+/AGENTS.md
+/test_live_*.py
+/test_smoke_*.py
+/test_stress_*.py
+/test_parallel_*.py

{cat_stack-1.6.7 → cat_stack-1.6.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cat-stack
-Version: 1.6.7
+Version: 1.6.9
 Summary: Domain-agnostic text, image, PDF, and DOCX classification engine powered by LLMs
 Project-URL: Documentation, https://github.com/chrissoria/cat-stack#readme
 Project-URL: Issues, https://github.com/chrissoria/cat-stack/issues

{cat_stack-1.6.7 → cat_stack-1.6.9}/src/catstack/__about__.py

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: 2025-present Christopher Soria <chrissoria@berkeley.edu>
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
-__version__ = "1.6.7"
+__version__ = "1.6.9"
 __author__ = "Chris Soria"
 __email__ = "chrissoria@berkeley.edu"
 __title__ = "cat-stack"

{cat_stack-1.6.7 → cat_stack-1.6.9}/src/catstack/_formatter.py

@@ -45,6 +45,11 @@ def _check_dependencies():
 def _check_dependencies_installed() -> bool:
     """Pure check — returns True if all formatter deps import successfully.
     No side effects, no install attempts."""
+    # If a dep was just pip-installed in this process's lifetime, the import
+    # system may have cached its earlier absence; clear that so the re-check
+    # actually sees the freshly-installed package.
+    import importlib
+    importlib.invalidate_caches()
     try:
         import torch  # noqa: F401
         import transformers  # noqa: F401
@@ -165,7 +170,31 @@ def _ensure_dependencies(verbose: bool = True) -> bool:
             "  To skip this and disable the formatter, pass json_formatter=False."
         )
 
-    return _install_dependencies(verbose=verbose)
+    ok = _install_dependencies(verbose=verbose)
+    if not ok:
+        # Freshly pip-installed packages (esp. compiled ones like torch) often
+        # cannot be imported by the SAME running process — but they ARE on disk
+        # now. Tell the user to re-run rather than silently degrading every row
+        # to an error.
+        if verbose and _deps_on_disk():
+            print(
+                "[CatLLM] Formatter dependencies were just installed but cannot "
+                "be imported into the already-running process. Please RE-RUN your "
+                "command — they will load on the next start. (Avoid this by "
+                "pre-installing: pip install 'cat-stack[formatter]'.)"
+            )
+    return ok
+
+
+def _deps_on_disk() -> bool:
+    """True if the formatter deps are findable on disk (importable in a FRESH
+    process) even if they failed to import in the current one."""
+    import importlib.util
+    try:
+        return all(importlib.util.find_spec(m) is not None
+                   for m in ("torch", "transformers", "accelerate"))
+    except (ImportError, ValueError):
+        return False
 
 
 def _is_model_cached() -> bool:
@@ -205,6 +234,51 @@ def ensure_formatter_available() -> bool:
     return True  # actual download happens in load_formatter()
 
 
+def _load_formatter_tokenizer(AutoTokenizer):
+    """Load the formatter tokenizer, defending against a malformed
+    `tokenizer_config.json`.
+
+    Some published configs store `extra_special_tokens` as a LIST, but
+    transformers 4.56–4.57.x expect a dict and crash in
+    `_set_model_specific_special_tokens` with
+    `'list' object has no attribute 'keys'`. On that failure we snapshot the
+    repo locally, normalize a list-valued `extra_special_tokens` to `{}`
+    (the tokens already live in `added_tokens`/`special_tokens_map`, so
+    dropping the field is lossless), and load from the patched local copy.
+    """
+    try:
+        return AutoTokenizer.from_pretrained(
+            _MERGED_MODEL_REPO, trust_remote_code=True
+        )
+    except (AttributeError, TypeError) as e:
+        if "keys" not in str(e) and "extra_special_tokens" not in str(e):
+            raise
+        import json
+        import os
+        from huggingface_hub import snapshot_download
+
+        local_dir = snapshot_download(_MERGED_MODEL_REPO)
+        cfg_path = os.path.join(local_dir, "tokenizer_config.json")
+        with open(cfg_path) as f:
+            cfg = json.load(f)
+        if isinstance(cfg.get("extra_special_tokens"), list):
+            cfg["extra_special_tokens"] = {}
+            # snapshot dirs are often read-only symlink caches; patch a copy.
+            import tempfile
+            import shutil
+            patched = tempfile.mkdtemp(prefix="catllm_formatter_tok_")
+            for fn in os.listdir(local_dir):
+                src = os.path.join(local_dir, fn)
+                if os.path.isfile(src):
+                    shutil.copy(src, os.path.join(patched, fn))
+            with open(os.path.join(patched, "tokenizer_config.json"), "w") as f:
+                json.dump(cfg, f)
+            print("[CatLLM] Patched malformed extra_special_tokens in the "
+                  "formatter tokenizer config (list -> {}).")
+            return AutoTokenizer.from_pretrained(patched, trust_remote_code=True)
+        raise
+
+
 def load_formatter(device=None):
     """
     Load the merged formatter model and tokenizer.
@@ -230,15 +304,21 @@ def load_formatter(device=None):
     dtype = torch.float16 if device == "cuda" else torch.float32
 
     print(f"[CatLLM] Loading JSON formatter on {device}...")
-    tokenizer = AutoTokenizer.from_pretrained(
-        _MERGED_MODEL_REPO, trust_remote_code=True
-    )
+    tokenizer = _load_formatter_tokenizer(AutoTokenizer)
     if tokenizer.pad_token is None:
         tokenizer.pad_token = tokenizer.eos_token
 
-    model = AutoModelForCausalLM.from_pretrained(
-        _MERGED_MODEL_REPO, dtype=dtype, trust_remote_code=True
-    )
+    # `dtype=` is the transformers >=4.56 kwarg; older versions only accept
+    # `torch_dtype=` and crash if `dtype=` leaks into the config. Try the new
+    # name, fall back to the old one.
+    try:
+        model = AutoModelForCausalLM.from_pretrained(
+            _MERGED_MODEL_REPO, dtype=dtype, trust_remote_code=True
+        )
+    except TypeError:
+        model = AutoModelForCausalLM.from_pretrained(
+            _MERGED_MODEL_REPO, torch_dtype=dtype, trust_remote_code=True
+        )
     model = model.to(device)
     model.eval()
 
@@ -281,7 +361,9 @@ def run_formatter(raw_output, categories, model, tokenizer, device):
     with torch.no_grad():
         out = model.generate(
             **inputs,
-            max_new_tokens=128,
+            # 512 (was 128): a large category set produces a long N-key JSON
+            # object; 128 tokens truncated it for 28/48-category tasks.
+            max_new_tokens=512,
             do_sample=False,
             temperature=None,
             top_p=None,

{cat_stack-1.6.7 → cat_stack-1.6.9}/src/catstack/_providers.py

@@ -126,6 +126,27 @@ def _hf_model_needs_enable_thinking_off(model: str) -> bool:
     return any(model.startswith(p) for p in _HF_NEEDS_ENABLE_THINKING_OFF)
 
 
+# Router-served models measured (2026-06-12 reasoning audit) to reason by
+# default with NO honored off-switch through the OpenAI-compatible router:
+# the router 400-rejects `chat_template_kwargs.enable_thinking` for their
+# templates, and they expose no reasoning_effort. classify() warns once per
+# client so users know the provider default applies.
+_HF_DEFAULT_REASONING_PREFIXES = (
+    "openai/gpt-oss",
+    "moonshotai/kimi-k2",
+)
+
+
+def _hf_model_reasons_by_default(model: str) -> bool:
+    m = (model or "").lower()
+    return any(m.startswith(p) for p in _HF_DEFAULT_REASONING_PREFIXES)
+
+
+# Module-level: models already warned about uncontrolled reasoning, so the
+# warning fires once per process even though a fresh client is built per row.
+_WARNED_UNCONTROLLED_REASONING: set = set()
+
+
 # ---------------------------------------------------------------------------
 # Anthropic deprecated the `temperature` parameter starting with the Opus 4.7 /
 # 4.8 generation: these models return 400 "`temperature` is deprecated for this
@@ -545,8 +566,15 @@ class UnifiedLLMClient:
             # accept booleans). Without this, gpt-oss family models emit long
             # <think> blocks by default that bloat per-row generation 3-5x.
             return self._build_openai_payload(messages, json_schema, creativity, force_json, thinking_budget)
+        elif self.provider == "xai":
+            # v1.6.8: forward the reasoning request. grok-4.3+ hybrids reason
+            # by default (2026-06-12 audit: 214 reasoning tokens on a trivial
+            # probe with no control sent); non-reasoning variants reject
+            # reasoning_effort and are handled by the 400 fallback in
+            # complete(), which caches the rejection on the client.
+            return self._build_openai_payload(messages, json_schema, creativity, force_json, thinking_budget)
         else:
-            # Other OpenAI-compatible providers (xai, mistral, etc.)
+            # Other OpenAI-compatible providers (mistral, etc.)
             return self._build_openai_payload(messages, json_schema, creativity, force_json)
 
     def _build_openai_payload(
@@ -620,6 +648,25 @@ class UnifiedLLMClient:
         elif creativity is not None:
             payload["temperature"] = creativity
 
+        # xAI (v1.6.8): hybrid grok models accept reasoning_effort alongside
+        # temperature. "low" is the lowest tier xAI exposes (no "none" /
+        # "minimal"); explicitly non-reasoning variants 400 on the field —
+        # complete() pops it and caches `_xai_no_reasoning_effort` so later
+        # rows on this client skip the doomed field up front.
+        if (
+            self.provider == "xai"
+            and thinking_budget is not None
+            and not getattr(self, "_xai_no_reasoning_effort", False)
+            # Variants whose name already encodes "non-reasoning" are off by
+            # model choice; sending reasoning_effort to them is not just
+            # redundant but HARMFUL — verified 2026-06-13 that
+            # grok-4-1-fast-non-reasoning returns 0 reasoning tokens with no
+            # field but 207 when sent reasoning_effort="low", i.e. the field
+            # turns reasoning back ON. Leave these alone.
+            and "non-reasoning" not in (self.model or "").lower()
+        ):
+            payload["reasoning_effort"] = "low" if thinking_budget == 0 else "high"
+
         # Ollama: per-model-family reasoning control via the top-level
         # `think` field. gpt-oss expects an enum ("low"/"medium"/"high");
         # qwen3/deepseek-r1 expect a boolean. Models not in the
@@ -648,6 +695,24 @@ class UnifiedLLMClient:
             and _hf_model_needs_enable_thinking_off(self.model)
         ):
             payload["chat_template_kwargs"] = {"enable_thinking": False}
+        elif (
+            self.provider in ("huggingface", "huggingface-together")
+            and thinking_budget == 0
+            and _hf_model_reasons_by_default(self.model)
+            and self.model not in _WARNED_UNCONTROLLED_REASONING
+        ):
+            # v1.6.8: these router-served models reason by default and honor
+            # no off-switch through the router (enable_thinking is
+            # 400-rejected for their templates). Warn once per process (a
+            # fresh client is built per row, so a per-instance flag would
+            # warn every row) so the uniform "reasoning off" request isn't
+            # silently unmet.
+            print(
+                f"\n[CatLLM] WARNING: no effective reasoning control delivered "
+                f"for '{self.model}'; the provider's default reasoning "
+                f"behavior applies. See docs/reasoning-controls.md.\n"
+            )
+            _WARNED_UNCONTROLLED_REASONING.add(self.model)
 
         return payload
 
@@ -759,11 +824,19 @@ class UnifiedLLMClient:
         if creativity is not None:
             payload["generationConfig"]["temperature"] = creativity
 
-        # Add thinking budget for extended thinking (Google-specific)
-        # Must be inside generationConfig, not at top level
-        # Google requires a reasonable minimum budget (enforce 128 tokens minimum)
-        if thinking_budget and thinking_budget > 0:
-            budget = max(thinking_budget, 128)
+        # Reasoning control (Google-specific). Must be inside generationConfig.
+        # v1.6.8: an explicit zero budget is now SENT at thinking_budget = 0.
+        # Previously nothing was sent at 0 and Gemini ran at its provider
+        # default, which the 2026-06-12 audit measured as thinking ON
+        # (~200+ thought tokens on a trivial classification call). Models
+        # that reject 0 (minimum-budget tiers) are handled by the 400
+        # fallback in complete(), which caches the discovered floor on the
+        # client (`_google_thinking_floor`).
+        if thinking_budget is not None:
+            if thinking_budget > 0:
+                budget = max(thinking_budget, 128)
+            else:
+                budget = getattr(self, "_google_thinking_floor", 0)
             payload["generationConfig"]["thinkingConfig"] = {"thinkingBudget": budget}
 
         return payload
@@ -946,6 +1019,10 @@ class UnifiedLLMClient:
         # transient error this call? Only strip once per call so we don't
         # mutate payload on every retry tick.
         stripped_response_format = False
+        # v1.6.8: consecutive-timeout counter + one-shot Google schema drop
+        # (see the Timeout handler below).
+        timeout_count = 0
+        dropped_google_schema = False
 
         for attempt in range(max_retries):
             endpoint = self._get_endpoint()
@@ -1019,9 +1096,32 @@ class UnifiedLLMClient:
                             payload["reasoning_effort"] = "low"
                             continue
                         elif current == "low" and "reasoning_effort" in payload:
+                            # Model takes no reasoning_effort at all (e.g.
+                            # xAI's explicitly non-reasoning variants).
+                            # Cache so later rows on this client skip the
+                            # doomed field up front (v1.6.8).
+                            self._xai_no_reasoning_effort = True
                             payload.pop("reasoning_effort")
                             continue
 
+                    # Google (v1.6.8): minimum-budget thinking tiers reject
+                    # thinkingBudget: 0. Fall back to 128 (Google's stated
+                    # minimum) and cache on the client.
+                    if (
+                        self.provider == "google"
+                        and "thinking" in error_text
+                        and ("budget" in error_text or "invalid" in error_text
+                             or "unsupported" in error_text)
+                        and payload.get("generationConfig", {})
+                                   .get("thinkingConfig", {})
+                                   .get("thinkingBudget") == 0
+                    ):
+                        self._google_thinking_floor = 128
+                        payload["generationConfig"]["thinkingConfig"]["thinkingBudget"] = 128
+                        print(f"\n[CatLLM] Model '{self.model}' rejected thinkingBudget=0; "
+                              f"falling back to the minimum (128) and caching for this client.\n")
+                        continue
+
                     # Anthropic deprecated `temperature` for newer models
                     # (Opus 4.7+): they 400 with "`temperature` is deprecated
                     # for this model." Strip it, cache on the client so the
@@ -1113,6 +1213,27 @@ class UnifiedLLMClient:
                 return result, None
 
             except requests.exceptions.Timeout:
+                timeout_count += 1
+                # v1.6.8: Gemini can reproducibly hang on specific inputs
+                # when a strict responseSchema is attached (constrained-
+                # decoding pathology; 2026-06-12 audit — a trivial input
+                # timed out 6/6 attempts WITH the schema and answered
+                # instantly without it). After two consecutive timeouts with
+                # a schema attached, drop the schema once and re-ask: the
+                # prompt still requests JSON and extract_json() parses it
+                # from the free-text response.
+                if (
+                    self.provider == "google"
+                    and timeout_count >= 2
+                    and not dropped_google_schema
+                    and "responseSchema" in payload.get("generationConfig", {})
+                ):
+                    dropped_google_schema = True
+                    payload["generationConfig"].pop("responseSchema", None)
+                    print(f"[CatLLM] Repeated timeouts from '{self.model}' with "
+                          f"responseSchema attached; retrying schema-less "
+                          f"(prompt-based JSON parsing).")
+                    continue
                 wait_time = _backoff_with_jitter(initial_delay, attempt)
                 elapsed = time.monotonic() - start
                 if attempt < max_retries - 1 and elapsed + wait_time <= max_total_wait: