tacl-llm 2.0.0__tar.gz

tacl_llm-2.0.0/PKG-INFO

@@ -0,0 +1,26 @@
+Metadata-Version: 2.4
+Name: tacl-llm
+Version: 2.0.0
+Summary: Ternary Additive Contrastive LLM — CPU-friendly integer neural network
+Author: TACL Contributors
+License-Expression: MIT
+Requires-Python: >=3.10
+Requires-Dist: numpy>=1.24
+Requires-Dist: torch>=2.2
+Requires-Dist: safetensors>=0.4
+Requires-Dist: tokenizers>=0.19
+Requires-Dist: huggingface_hub>=0.23
+Requires-Dist: tqdm
+Requires-Dist: pyyaml
+Provides-Extra: colab
+Requires-Dist: ipywidgets; extra == "colab"
+Requires-Dist: matplotlib; extra == "colab"
+Provides-Extra: logging
+Requires-Dist: tensorboard; extra == "logging"
+Provides-Extra: gpu
+Provides-Extra: datasets
+Requires-Dist: datasets>=2.14; extra == "datasets"
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: datasets>=2.14; extra == "dev"

tacl_llm-2.0.0/README.md

@@ -0,0 +1,17 @@
+# TACL — Ternary Additive Contrastive LLM
+
+CPU-friendly integer neural network. Not a Transformer — uses CHL (Equilibrium Propagation), sparsemax, linear attention, and ternary weights.
+
+## Quick Install
+
+```bash
+pip install tacl-llm
+```
+
+## Documentation
+
+See [ts/00-overview-and-index.md](ts/00-overview-and-index.md) for the full specification, and [wiki/](wiki/) for research wiki and verification reports.
+
+## License
+
+MIT

tacl_llm-2.0.0/pyproject.toml

@@ -0,0 +1,46 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "tacl-llm"
+version = "2.0.0"
+description = "Ternary Additive Contrastive LLM — CPU-friendly integer neural network"
+requires-python = ">=3.10"
+authors = [{ name = "TACL Contributors" }]
+license = "MIT"
+
+dependencies = [
+    "numpy>=1.24",
+    "torch>=2.2",
+    "safetensors>=0.4",
+    "tokenizers>=0.19",
+    "huggingface_hub>=0.23",
+    "tqdm",
+    "pyyaml",
+]
+
+[project.optional-dependencies]
+colab = ["ipywidgets", "matplotlib"]
+logging = ["tensorboard"]
+gpu = []
+datasets = ["datasets>=2.14"]
+dev = ["pytest>=7", "pytest-cov", "datasets>=2.14"]
+
+[project.scripts]
+tacl-train = "tacl.scripts.train:main"
+tacl-generate = "tacl.scripts.generate:main"
+tacl-smoke-check = "tacl.scripts.smoke_check:run_smoke_check"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+tacl = ["py.typed"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]
+filterwarnings = [
+    "ignore::tacl.exceptions.TACLCaveatWarning",
+]

tacl_llm-2.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

tacl_llm-2.0.0/src/tacl/__init__.py

@@ -0,0 +1,29 @@
+from .configuration_tacl import TACLConfig
+from .modeling_tacl import TACLModel, TACLForCausalLM
+from .tokenization_tacl import TACLTokenizer
+from .training.trainer import TACLTrainer, TACLTrainingArguments
+from .auto import TACLAutoModel, TACLAutoForCausalLM, TACLAutoConfig, TACLAutoTokenizer
+from .presets import PRESETS, list_presets
+from .generation.streamer import TACLTextStreamer, TACLTextIteratorStreamer
+from .data import PretrainPackedDataset, SFTDataset
+
+__all__ = [
+    "TACLConfig",
+    "TACLModel",
+    "TACLForCausalLM",
+    "TACLTokenizer",
+    "TACLTrainer",
+    "TACLTrainingArguments",
+    "TACLAutoModel",
+    "TACLAutoForCausalLM",
+    "TACLAutoConfig",
+    "TACLAutoTokenizer",
+    "PRESETS",
+    "list_presets",
+    "TACLTextStreamer",
+    "TACLTextIteratorStreamer",
+    "PretrainPackedDataset",
+    "SFTDataset",
+]
+
+__version__ = "2.0.0"

tacl_llm-2.0.0/src/tacl/auto.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Optional
+
+from .configuration_tacl import TACLConfig
+from .modeling_tacl import TACLForCausalLM, TACLModel
+from .tokenization_tacl import TACLTokenizer
+from .presets import PRESETS
+
+
+class TACLAutoModel:
+    @classmethod
+    def from_pretrained(cls, path_or_repo: str) -> TACLModel:
+        config = TACLConfig.from_pretrained(path_or_repo)
+        model = TACLModel(config)
+        from .serialization import load_state_dict
+        load_state_dict(model, path_or_repo)
+        return model
+
+    @classmethod
+    def from_preset(cls, name: str, **overrides: Any) -> TACLModel:
+        if name not in PRESETS:
+            raise KeyError(f"Unknown preset {name!r}. Available: {list(PRESETS.keys())}")
+        config = PRESETS[name](**overrides)
+        return TACLModel(config)
+
+
+class TACLAutoForCausalLM:
+    @classmethod
+    def from_pretrained(cls, path_or_repo: str) -> TACLForCausalLM:
+        return TACLForCausalLM.from_pretrained(path_or_repo)
+
+    @classmethod
+    def from_preset(cls, name: str, **overrides: Any) -> TACLForCausalLM:
+        return TACLForCausalLM.from_preset(name, **overrides)
+
+
+class TACLAutoConfig:
+    @classmethod
+    def from_pretrained(cls, path_or_repo: str) -> TACLConfig:
+        return TACLConfig.from_pretrained(path_or_repo)
+
+
+class TACLAutoTokenizer:
+    @classmethod
+    def from_pretrained(cls, path_or_repo: str, use_hf: bool = False, **kwargs: Any) -> TACLTokenizer:
+        from pathlib import Path as _Path
+        if use_hf or (not _Path(path_or_repo).exists() and "/" in path_or_repo):
+            return TACLTokenizer.from_pretrained_hf(path_or_repo, **kwargs)
+        return TACLTokenizer.from_pretrained(path_or_repo)

tacl_llm-2.0.0/src/tacl/configuration_tacl.py

@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+import json
+import warnings
+from dataclasses import dataclass, asdict
+from pathlib import Path
+from typing import Any
+
+from .exceptions import TACLCaveatWarning, warn_caveat
+
+
+_CAVEATS = {
+    "chl_validated_on": "MLP proxy only, full-stack integration PENDING (verification-report.md §2.1, §2.3)",
+    "language_modeling_quality": "unmeasured — no perplexity/accuracy benchmark yet",
+    "context_decay": "S_t has no decay mechanism, uniform weighting over all past tokens (see final-architecture.md §2.2 'Known limitation')",
+    "max_safe_context_int32": 4096,
+}
+
+_TACL_VERSION = "2.0.0"
+
+
+@dataclass
+class TACLConfig:
+    tacl_version: str = "2.0"
+    vocab_size: int = 8192
+    hidden_size: int = 512
+    ffn_size: int = 2048
+    num_layers: int = 8
+    max_position_embeddings: int = 4096
+
+    ternary_sparsity_target: float = 0.5
+    quant_format: str = "bfp_int8"
+    fixed_point_qf: int = 12
+    stochastic_rounding: bool = True
+
+    attn_kernel: str = "elu1"
+    use_thought_loop: bool = False
+    thought_max_iters: int = 8
+    thought_epsilon_qf_units: int = 1
+
+    learning_rule: str = "chl"
+    chl_alpha: float = 0.05
+
+    tie_word_embeddings: bool = True
+    pad_token_id: int = 0
+    bos_token_id: int = 1
+    eos_token_id: int = 2
+    unk_token_id: int = 3
+
+    use_fixed_residual: bool = False
+
+    _caveats: dict[str, str] | None = None
+
+    def __post_init__(self) -> None:
+        if self.learning_rule not in ("chl",):
+            raise ValueError(
+                f"learning_rule={self.learning_rule!r} не поддерживается: DFA отвергнут "
+                "численно (cos~0.19 c BP, +332% loss при обучении, см. verification-report.md §1.3/§2.2)."
+            )
+        if self.hidden_size % 2 != 0:
+            raise ValueError("hidden_size должен быть чётным (используется в d/2 расчётах FLOPs)")
+        if self.max_position_embeddings > 4096:
+            warnings.warn(
+                f"max_position_embeddings={self.max_position_embeddings} > 4096: "
+                "int32 overflow risk in linear attention state. "
+                "See verification-report.md §3.2 for details.",
+                stacklevel=2,
+            )
+        if self._caveats is None:
+            self._caveats = dict(_CAVEATS)
+
+    def to_dict(self) -> dict[str, Any]:
+        d = {}
+        for k, v in asdict(self).items():
+            if k == "_caveats" and not v:
+                continue
+            d[k] = v
+        d["auto_map"] = {
+            "AutoConfig": "tacl.configuration_tacl.TACLConfig",
+            "AutoModel": "tacl.auto.TACLAutoModel",
+            "AutoModelForCausalLM": "tacl.auto.TACLAutoForCausalLM",
+        }
+        return d
+
+    def to_json_string(self) -> str:
+        return json.dumps(self.to_dict(), indent=2, ensure_ascii=False)
+
+    @classmethod
+    def from_dict(cls, d: dict[str, Any]) -> "TACLConfig":
+        return cls(**{k: v for k, v in d.items() if k in cls.__dataclass_fields__})
+
+    @classmethod
+    def from_json_file(cls, path: str | Path) -> "TACLConfig":
+        with open(path, "r", encoding="utf-8") as f:
+            return cls.from_dict(json.load(f))
+
+    @classmethod
+    def _version_tuple(cls, v: str) -> tuple[int, ...]:
+        parts = v.split(".")
+        return tuple(int(p) if p.isdigit() else 0 for p in parts)
+
+    def save_pretrained(self, directory: str | Path) -> None:
+        directory = Path(directory)
+        directory.mkdir(parents=True, exist_ok=True)
+        self.tacl_version = _TACL_VERSION
+        with open(directory / "config.json", "w", encoding="utf-8") as f:
+            f.write(self.to_json_string())
+
+    @classmethod
+    def from_pretrained(cls, directory_or_repo: str | Path) -> "TACLConfig":
+        path = Path(directory_or_repo)
+        if not path.exists():
+            from huggingface_hub import hf_hub_download
+            config_path = hf_hub_download(str(directory_or_repo), filename="config.json")
+        else:
+            config_path = path / "config.json"
+        config = cls.from_json_file(config_path)
+
+        saved_version = config.tacl_version
+        if saved_version and cls._version_tuple(saved_version) != cls._version_tuple(_TACL_VERSION):
+            warnings.warn(
+                f"TACL version mismatch: saved model uses tacl_version={saved_version}, "
+                f"current library is {_TACL_VERSION}. "
+                "Upgrade/downgrade may be needed.",
+                stacklevel=2,
+            )
+
+        if config._caveats:
+            for key, val in config._caveats.items():
+                warn_caveat(f"{key}: {val}")
+
+        return config

tacl_llm-2.0.0/src/tacl/data/__init__.py

@@ -0,0 +1,14 @@
+from .base import TACLDataset
+from .pretrain_dataset import PretrainPackedDataset, pretrain_collate_fn
+from .sft_dataset import SFTDataset, DEFAULT_TACL_CHAT_TEMPLATE, sft_collate_fn
+from .streaming import StreamingTextDataset
+
+__all__ = [
+    "TACLDataset",
+    "PretrainPackedDataset",
+    "pretrain_collate_fn",
+    "SFTDataset",
+    "DEFAULT_TACL_CHAT_TEMPLATE",
+    "sft_collate_fn",
+    "StreamingTextDataset",
+]

tacl_llm-2.0.0/src/tacl/data/base.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from typing import Any, Optional, Union
+
+import torch
+from torch.utils.data import Dataset, IterableDataset
+
+
+class TACLDataset(Dataset):
+    def __init__(
+        self,
+        source: Union[str, list[dict]],
+        tokenizer: Any,
+        max_length: int = 512,
+    ):
+        self._tokenizer = tokenizer
+        self._max_length = max_length
+        self._samples: list[dict[str, torch.Tensor]] = []
+        self._load(source)
+
+    def _load(self, source: Union[str, list[dict]]) -> None:
+        raise NotImplementedError
+
+    def __len__(self) -> int:
+        return len(self._samples)
+
+    def __getitem__(self, idx: int) -> dict[str, torch.Tensor]:
+        return self._samples[idx]
+
+    def _get_eos_id(self) -> int:
+        if self._tokenizer._config is not None and hasattr(self._tokenizer._config, 'eos_token_id'):
+            return self._tokenizer._config.eos_token_id
+        return 2

tacl_llm-2.0.0/src/tacl/data/pretrain_dataset.py

@@ -0,0 +1,171 @@
+from __future__ import annotations
+
+import csv
+import glob
+import logging
+import time
+from pathlib import Path
+from typing import Any, Iterable, Iterator, Optional, Union
+
+import torch
+from torch.utils.data import DataLoader, IterableDataset
+
+logger = logging.getLogger(__name__)
+
+
+def _get_eos_id(tokenizer: Any) -> int:
+    if hasattr(tokenizer, '_config') and tokenizer._config is not None and hasattr(tokenizer._config, 'eos_token_id'):
+        return tokenizer._config.eos_token_id
+    return 2
+
+
+class PretrainPackedDataset(IterableDataset):
+    """
+    Tokenises a text stream and packs into contiguous blocks of seq_len,
+    without padding inside each block (standard GPT-style "packing").
+
+    Supports:
+    - Local text files (.txt, .csv, .parquet, glob patterns)
+    - HF Datasets (requires datasets package)
+    - Arbitrary Python iterator of strings
+
+    Shuffle is done via a fixed-size reservoir buffer, not full shuffle.
+    """
+
+    def __init__(
+        self,
+        source: Union[str, Iterable[str]],
+        tokenizer: Any,
+        seq_len: int = 1024,
+        text_column: str = "text",
+        hf_dataset_config: Optional[str] = None,
+        hf_split: str = "train",
+        shuffle_buffer_size: int = 10000,
+        add_eos_between_docs: bool = True,
+    ):
+        self._source = source
+        self._tokenizer = tokenizer
+        self._seq_len = seq_len
+        self._text_column = text_column
+        self._hf_dataset_config = hf_dataset_config
+        self._hf_split = hf_split
+        self._shuffle_buffer_size = shuffle_buffer_size
+        self._add_eos = add_eos_between_docs
+
+        self._doc_iter: Optional[Iterator[str]] = None
+        self._token_buffer: list[int] = []
+        self._docs_processed = 0
+        self._tokens_processed = 0
+        self._start_time = time.time()
+
+    def _get_iterator(self) -> Iterator[str]:
+        if isinstance(self._source, str):
+            matched_files = glob.glob(self._source)
+            if matched_files:
+                for path_str in matched_files:
+                    path = Path(path_str)
+                    ext = path.suffix.lower()
+                    if ext == ".csv":
+                        with open(path, "r", encoding="utf-8", newline="") as f:
+                            reader = csv.DictReader(f)
+                            for row in reader:
+                                yield row.get(self._text_column, "")
+                    elif ext == ".parquet":
+                        try:
+                            import pandas as pd
+                            df = pd.read_parquet(path)
+                            for val in df[self._text_column].tolist():
+                                yield str(val)
+                        except ImportError:
+                            raise ImportError("Install pandas and pyarrow for .parquet support")
+                    else:
+                        with open(path, "r", encoding="utf-8", errors="ignore") as f:
+                            yield from f
+            else:
+                try:
+                    from datasets import load_dataset
+                    ds = load_dataset(
+                        self._source,
+                        self._hf_dataset_config,
+                        split=self._hf_split,
+                        streaming=True,
+                    )
+                    for example in ds:
+                        yield example[self._text_column]
+                except ImportError:
+                    raise ImportError(
+                        f"To use HF datasets, install with: pip install tacl[datasets]"
+                    )
+                except Exception:
+                    with open(self._source, "r", encoding="utf-8", errors="ignore") as f:
+                        yield from f
+        elif isinstance(self._source, Iterable):
+            yield from self._source
+        else:
+            raise TypeError(f"Unsupported source type: {type(self._source)}")
+
+    def __iter__(self) -> Iterator[dict[str, torch.Tensor]]:
+        self._token_buffer = []
+        self._docs_processed = 0
+        self._tokens_processed = 0
+        self._start_time = time.time()
+        doc_iter = self._get_iterator()
+
+        buf: list[dict[str, torch.Tensor]] = []
+
+        for doc_text in doc_iter:
+            doc_text = doc_text.strip()
+            if not doc_text:
+                continue
+
+            ids = self._tokenizer.encode(doc_text)
+            self._docs_processed += 1
+
+            if self._add_eos:
+                eos_id = _get_eos_id(self._tokenizer)
+                ids = ids + [eos_id]
+
+            self._token_buffer.extend(ids)
+
+            while len(self._token_buffer) >= self._seq_len:
+                chunk = self._token_buffer[:self._seq_len]
+                self._token_buffer = self._token_buffer[self._seq_len:]
+                item = {
+                    "input_ids": torch.tensor(chunk, dtype=torch.long),
+                    "labels": torch.tensor(chunk, dtype=torch.long),
+                }
+                self._tokens_processed += self._seq_len
+
+                if self._shuffle_buffer_size > 0:
+                    buf.append(item)
+                    if len(buf) >= self._shuffle_buffer_size:
+                        idx = torch.randint(0, len(buf), (1,)).item()
+                        yield buf.pop(idx)
+                else:
+                    yield item
+
+            if self._docs_processed % 1000 == 0:
+                elapsed = time.time() - self._start_time
+                tok_per_sec = self._tokens_processed / max(elapsed, 1e-6)
+                logger.info(
+                    f"PretrainPackedDataset: {self._docs_processed} docs, "
+                    f"{self._tokens_processed} tokens, {tok_per_sec:.0f} tok/s"
+                )
+
+        while buf:
+            idx = torch.randint(0, len(buf), (1,)).item()
+            yield buf.pop(idx)
+
+        remainder = self._token_buffer[:self._seq_len]
+        if len(remainder) > 0:
+            remainder = remainder + [0] * (self._seq_len - len(remainder))
+            yield {
+                "input_ids": torch.tensor(remainder, dtype=torch.long),
+                "labels": torch.tensor(remainder, dtype=torch.long),
+            }
+
+
+def pretrain_collate_fn(batch: list[dict[str, torch.Tensor]]) -> dict[str, torch.Tensor]:
+    input_ids = torch.stack([item["input_ids"] for item in batch])
+    labels = torch.stack([item["labels"] for item in batch])
+    return {"input_ids": input_ids, "labels": labels}