tensorless 0.1.0__py3-none-any.whl

tensorless/config.py

@@ -0,0 +1,121 @@
+"""Configuration objects.
+
+`TrainConfig` holds every knob a user can override in `tl.train(...)`.
+Any field left as `None` means "let Tensorless decide automatically".
+`ResolvedConfig` is what the auto-configuration system produces after
+filling in every `None` with a concrete value.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field, asdict
+from typing import Optional, Any, Dict
+
+
+@dataclass
+class TrainConfig:
+    """User-facing training configuration. Every field is optional;
+    unset fields are chosen automatically by the auto-config system.
+    """
+
+    # --- output / lifecycle ---
+    out: Optional[str] = None                 # output .tl path, default "model.tl"
+    force: bool = False                        # force retraining even if unchanged
+    resume: Optional[bool] = None               # force/forbid resume (None = auto)
+    ask_on_data_change: bool = False            # raise instead of auto-retrain on data change
+
+    # --- task / architecture ---
+    task: Optional[str] = None                  # "text-generation", "classification", "regression"
+    model_type: Optional[str] = None            # "transformer", "mlp"
+    d_model: Optional[int] = None
+    layers: Optional[int] = None
+    heads: Optional[int] = None
+    ff_mult: Optional[int] = None
+    dropout: Optional[float] = None
+    max_seq_len: Optional[int] = None
+
+    # --- optimization ---
+    optimizer: Optional[str] = None             # "adamw", "adam", "sgd"
+    learning_rate: Optional[float] = None
+    weight_decay: Optional[float] = None
+    batch_size: Optional[int] = None
+    epochs: Optional[int] = None
+    max_steps: Optional[int] = None
+    grad_clip: Optional[float] = None
+    warmup_steps: Optional[int] = None
+
+    # --- validation / early stopping ---
+    val_split: Optional[float] = None
+    patience: Optional[int] = None
+    min_delta: Optional[float] = None
+
+    # --- hardware ---
+    device: Optional[str] = None                # "cpu", "cuda", "tpu", or None = auto
+    precision: Optional[str] = None              # "fp32", "fp16", "bf16"
+
+    # --- checkpointing ---
+    checkpoint_every: Optional[int] = None       # steps between checkpoints
+    checkpoint_dir: Optional[str] = None
+
+    # --- misc ---
+    seed: int = 42
+    verbose: bool = True
+    extra: Dict[str, Any] = field(default_factory=dict)
+
+    def overrides(self) -> Dict[str, Any]:
+        """Return only the fields the user explicitly set (non-None)."""
+        d = asdict(self)
+        d.pop("extra", None)
+        return {k: v for k, v in d.items() if v is not None and v is not False}
+
+
+@dataclass
+class ResolvedConfig:
+    """Fully resolved configuration -- every field has a concrete value.
+    This is what actually gets used for training and what gets embedded
+    in checkpoints and the final `.tl` file.
+    """
+
+    out: str
+    force: bool
+    resume: Optional[bool]
+    ask_on_data_change: bool
+
+    task: str
+    model_type: str
+    d_model: int
+    layers: int
+    heads: int
+    ff_mult: int
+    dropout: float
+    max_seq_len: int
+
+    optimizer: str
+    learning_rate: float
+    weight_decay: float
+    batch_size: int
+    epochs: int
+    max_steps: Optional[int]
+    grad_clip: float
+    warmup_steps: int
+
+    val_split: float
+    patience: int
+    min_delta: float
+
+    device: str
+    precision: str
+
+    checkpoint_every: int
+    checkpoint_dir: str
+
+    seed: int
+    verbose: bool
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> "ResolvedConfig":
+        known = {f for f in cls.__dataclass_fields__.keys()}
+        return cls(**{k: v for k, v in d.items() if k in known})

tensorless/data/__init__.py

@@ -0,0 +1,11 @@
+from .loader import load_dataset, Dataset
+from .fingerprint import fingerprint_path
+from .inspector import inspect_path, InspectionReport
+
+__all__ = [
+    "load_dataset",
+    "Dataset",
+    "fingerprint_path",
+    "inspect_path",
+    "InspectionReport",
+]

tensorless/data/fingerprint.py

@@ -0,0 +1,71 @@
+"""Dataset fingerprinting.
+
+We hash the *content* of every data file under a path (plus filenames and
+sizes) so that Tensorless can cheaply detect whether a dataset has changed
+between runs -- this is what powers the "don't retrain if nothing changed"
+and "resume if interrupted" behaviour.
+
+The fingerprint is intentionally content-based (not mtime-based) so that
+copying a dataset to a new machine, or touching a file without changing it,
+does not trigger an unnecessary retrain.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+from typing import List
+
+# Cap how many bytes of large files we hash, to keep fingerprinting fast on
+# huge datasets. We hash the size + a content sample (head/tail) rather than
+# the whole file when it exceeds this threshold.
+_MAX_FULL_HASH_BYTES = 25 * 1024 * 1024  # 25 MB
+_SAMPLE_BYTES = 1 * 1024 * 1024  # 1 MB head + 1 MB tail for big files
+
+
+def _iter_files(path: str) -> List[str]:
+    if os.path.isfile(path):
+        return [path]
+    files = []
+    for root, dirs, names in os.walk(path):
+        dirs.sort()
+        for name in sorted(names):
+            if name.startswith("."):
+                continue
+            files.append(os.path.join(root, name))
+    return files
+
+
+def _hash_file(path: str, hasher: "hashlib._Hash") -> None:
+    size = os.path.getsize(path)
+    hasher.update(str(size).encode("utf-8"))
+    with open(path, "rb") as f:
+        if size <= _MAX_FULL_HASH_BYTES:
+            while True:
+                chunk = f.read(1024 * 1024)
+                if not chunk:
+                    break
+                hasher.update(chunk)
+        else:
+            hasher.update(f.read(_SAMPLE_BYTES))
+            f.seek(max(0, size - _SAMPLE_BYTES))
+            hasher.update(f.read(_SAMPLE_BYTES))
+
+
+def fingerprint_path(path: str) -> str:
+    """Return a stable hex digest fingerprinting the dataset at `path`.
+
+    The fingerprint changes if any file's content, size, name, or the set
+    of files itself changes.
+    """
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"Path '{path}' does not exist.")
+    path = os.path.abspath(path)
+    hasher = hashlib.sha256()
+    files = _iter_files(path)
+    for f in files:
+        rel = os.path.relpath(f, path if os.path.isdir(path) else os.path.dirname(path))
+        hasher.update(rel.encode("utf-8"))
+        _hash_file(f, hasher)
+    hasher.update(str(len(files)).encode("utf-8"))
+    return hasher.hexdigest()

tensorless/data/inspector.py

@@ -0,0 +1,161 @@
+"""Dataset inspection: `tl.inspect("./data")`.
+
+Loads the dataset, detects its task type, and reports size, samples,
+detected problems, and recommendations -- without training anything.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List
+
+from .loader import load_dataset, Dataset
+from .fingerprint import fingerprint_path
+
+
+@dataclass
+class InspectionReport:
+    path: str
+    fingerprint: str
+    kind: str
+    task: str
+    n_examples: int
+    n_files: int
+    columns: List[str] = field(default_factory=list)
+    sample: Any = None
+    warnings: List[str] = field(default_factory=list)
+    recommendations: List[str] = field(default_factory=list)
+    stats: Dict[str, Any] = field(default_factory=dict)
+
+    def __str__(self) -> str:
+        lines = [
+            f"Dataset: {self.path}",
+            f"  fingerprint : {self.fingerprint[:16]}...",
+            f"  detected kind : {self.kind}",
+            f"  detected task : {self.task}",
+            f"  examples      : {self.n_examples}",
+            f"  files         : {self.n_files}",
+        ]
+        if self.columns:
+            lines.append(f"  columns       : {', '.join(self.columns)}")
+        for k, v in self.stats.items():
+            lines.append(f"  {k:14s}: {v}")
+        if self.warnings:
+            lines.append("  Warnings:")
+            lines += [f"    - {w}" for w in self.warnings]
+        if self.recommendations:
+            lines.append("  Recommendations:")
+            lines += [f"    - {r}" for r in self.recommendations]
+        return "\n".join(lines)
+
+
+def _compute_stats(ds: Dataset) -> Dict[str, Any]:
+    stats: Dict[str, Any] = {}
+    if ds.kind in ("text", "text_labeled"):
+        lengths = [len(t) for t in ds.texts]
+        if lengths:
+            stats["avg_chars"] = round(sum(lengths) / len(lengths), 1)
+            stats["min_chars"] = min(lengths)
+            stats["max_chars"] = max(lengths)
+        if ds.kind == "text_labeled":
+            classes = sorted(set(ds.labels))
+            stats["n_classes"] = len(classes)
+            stats["classes"] = classes[:20]
+    else:
+        stats["n_rows"] = len(ds.records)
+        stats["n_columns"] = len(ds.columns)
+    return stats
+
+
+def _warnings_and_recommendations(ds: Dataset, task: str) -> (List[str], List[str]):
+    warnings: List[str] = []
+    recs: List[str] = []
+    n = len(ds)
+
+    if n == 0:
+        warnings.append("Dataset is empty.")
+        return warnings, recs
+
+    if n < 20:
+        warnings.append(
+            f"Only {n} example(s) found. This is very small for training a "
+            f"useful model; results may be poor / mostly memorization."
+        )
+        recs.append("Collect more data if possible (aim for hundreds+ examples).")
+
+    if ds.kind == "text_labeled":
+        from collections import Counter
+
+        counts = Counter(ds.labels)
+        if len(counts) < 2:
+            warnings.append("Only one class detected; classification needs 2+ classes.")
+        else:
+            majority = max(counts.values())
+            minority = min(counts.values())
+            if majority > 3 * max(minority, 1):
+                warnings.append(
+                    f"Class imbalance detected (largest class {majority} vs "
+                    f"smallest {minority}). Consider balancing or using "
+                    f"class weights."
+                )
+                recs.append(
+                    "Tensorless will still train, but consider collecting more "
+                    "examples for underrepresented classes."
+                )
+
+    if ds.kind == "tabular":
+        missing_cols = set()
+        for r in ds.records[:200]:
+            for c in ds.columns:
+                v = r.get(c, "")
+                if v is None or (isinstance(v, str) and v.strip() == ""):
+                    missing_cols.add(c)
+        if missing_cols:
+            warnings.append(
+                f"Missing values detected in column(s): {', '.join(sorted(missing_cols))}."
+            )
+            recs.append(
+                "Tensorless will impute missing numeric values with the column "
+                "mean and missing categorical values with a placeholder token."
+            )
+
+    if ds.kind in ("text", "text_labeled"):
+        avg_len = sum(len(t) for t in ds.texts) / max(1, len(ds.texts))
+        if avg_len > 20000:
+            recs.append(
+                "Texts are long; Tensorless will truncate to the configured "
+                "max_seq_len. Pass max_seq_len=... to change this."
+            )
+
+    return warnings, recs
+
+
+def inspect_path(path: str) -> InspectionReport:
+    # Local import to avoid a circular import between `data` and `auto`.
+    from ..auto.detector import detect_task
+
+    ds = load_dataset(path)
+    task = detect_task(ds)
+    fp = fingerprint_path(path)
+    stats = _compute_stats(ds)
+    warnings, recs = _warnings_and_recommendations(ds, task)
+
+    sample: Any = None
+    if ds.kind in ("text", "text_labeled") and ds.texts:
+        sample = ds.texts[0][:300]
+    elif ds.records:
+        sample = ds.records[0]
+
+    return InspectionReport(
+        path=path,
+        fingerprint=fp,
+        kind=ds.kind,
+        task=task,
+        n_examples=len(ds),
+        n_files=ds.n_files,
+        columns=list(ds.columns),
+        sample=sample,
+        warnings=warnings,
+        recommendations=recs,
+        stats=stats,
+    )

tensorless/data/loader.py

@@ -0,0 +1,255 @@
+"""Dataset loading.
+
+Turns a path (file or directory) into a normalized `Dataset` object that
+the rest of Tensorless can reason about, regardless of whether the data
+started life as .txt, .json, .jsonl, .csv, or a directory of any of those
+(optionally organized into class subfolders).
+
+Design goal: never silently drop or mutate user data. If something looks
+wrong (empty dataset, unreadable file, inconsistent columns) we raise a
+`DataError` with an actionable message rather than guessing.
+"""
+
+from __future__ import annotations
+
+import csv
+import json
+import os
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+from ..errors import DataError
+
+TEXT_EXTENSIONS = {".txt", ".md"}
+JSON_EXTENSIONS = {".json"}
+JSONL_EXTENSIONS = {".jsonl", ".ndjson"}
+CSV_EXTENSIONS = {".csv", ".tsv"}
+
+# Common field names we look for when a JSON/JSONL record represents a
+# single piece of free text (e.g. for language modeling).
+_TEXT_FIELD_CANDIDATES = ("text", "content", "body", "document", "sentence")
+_LABEL_FIELD_CANDIDATES = ("label", "target", "class", "category", "y")
+
+
+@dataclass
+class Dataset:
+    """Normalized in-memory representation of a loaded dataset.
+
+    kind is one of:
+      - "text"        : a corpus of raw text (language modeling)
+      - "text_labeled": (text, label) pairs (text classification)
+      - "tabular"      : rows of named columns (classification/regression)
+    """
+
+    kind: str
+    source: str
+    texts: List[str] = field(default_factory=list)
+    labels: List[Any] = field(default_factory=list)
+    records: List[Dict[str, Any]] = field(default_factory=list)
+    columns: List[str] = field(default_factory=list)
+    n_files: int = 0
+
+    def __len__(self) -> int:
+        if self.kind in ("text", "text_labeled"):
+            return len(self.texts)
+        return len(self.records)
+
+
+def _read_text_file(path: str) -> str:
+    try:
+        with open(path, "r", encoding="utf-8", errors="strict") as f:
+            return f.read()
+    except UnicodeDecodeError as e:
+        raise DataError(
+            f"File '{path}' is not valid UTF-8 text. Tensorless expects "
+            f"text datasets to be UTF-8 encoded."
+        ) from e
+
+
+def _read_json_records(path: str) -> List[Dict[str, Any]]:
+    with open(path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    if isinstance(data, dict):
+        # Either a single record, or {"data": [...]}
+        for key in ("data", "records", "items", "examples"):
+            if key in data and isinstance(data[key], list):
+                data = data[key]
+                break
+        else:
+            data = [data]
+    if not isinstance(data, list):
+        raise DataError(f"Unsupported JSON structure in '{path}': expected a list of records.")
+    return data
+
+
+def _read_jsonl_records(path: str) -> List[Dict[str, Any]]:
+    records = []
+    with open(path, "r", encoding="utf-8") as f:
+        for i, line in enumerate(f):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                records.append(json.loads(line))
+            except json.JSONDecodeError as e:
+                raise DataError(f"Malformed JSON on line {i + 1} of '{path}': {e}") from e
+    return records
+
+
+def _read_csv_records(path: str) -> List[Dict[str, Any]]:
+    delimiter = "\t" if path.lower().endswith(".tsv") else ","
+    records = []
+    with open(path, "r", encoding="utf-8", newline="") as f:
+        reader = csv.DictReader(f, delimiter=delimiter)
+        if reader.fieldnames is None:
+            raise DataError(f"'{path}' has no header row / is empty.")
+        for row in reader:
+            records.append(dict(row))
+    return records
+
+
+def _extract_text_field(record: Dict[str, Any]) -> Optional[str]:
+    for key in _TEXT_FIELD_CANDIDATES:
+        if key in record and isinstance(record[key], str):
+            return record[key]
+    return None
+
+
+def _extract_label_field(record: Dict[str, Any]) -> Optional[Any]:
+    for key in _LABEL_FIELD_CANDIDATES:
+        if key in record:
+            return record[key]
+    return None
+
+
+def _records_to_dataset(records: List[Dict[str, Any]], source: str) -> Dataset:
+    if not records:
+        raise DataError(f"No records found in '{source}'.")
+
+    text_field_present = all(_extract_text_field(r) is not None for r in records[:50])
+    if text_field_present:
+        texts = [_extract_text_field(r) or "" for r in records]
+        labels = [_extract_label_field(r) for r in records]
+        if any(l is not None for l in labels):
+            return Dataset(kind="text_labeled", source=source, texts=texts, labels=labels)
+        return Dataset(kind="text", source=source, texts=texts)
+
+    # Otherwise treat as generic tabular data.
+    # Preserve column order as it appears in the data (important: the
+    # "use the last column as the target" heuristic in auto/detector.py
+    # depends on this being the *original* column order, not alphabetical).
+    columns: List[str] = []
+    seen = set()
+    for r in records:
+        for k in r.keys():
+            if k not in seen:
+                seen.add(k)
+                columns.append(k)
+    return Dataset(kind="tabular", source=source, records=records, columns=columns)
+
+
+def _load_directory(path: str) -> Dataset:
+    entries = [e for e in sorted(os.listdir(path)) if not e.startswith(".")]
+    if not entries:
+        raise DataError(f"Directory '{path}' is empty.")
+
+    subdirs = [e for e in entries if os.path.isdir(os.path.join(path, e))]
+    files = [e for e in entries if os.path.isfile(os.path.join(path, e))]
+
+    # Case 1: class subfolders of text files -> text classification.
+    if subdirs and not files:
+        texts, labels = [], []
+        n_files = 0
+        for label in subdirs:
+            sub = os.path.join(path, label)
+            for fname in sorted(os.listdir(sub)):
+                fpath = os.path.join(sub, fname)
+                if not os.path.isfile(fpath):
+                    continue
+                ext = os.path.splitext(fname)[1].lower()
+                if ext in TEXT_EXTENSIONS:
+                    texts.append(_read_text_file(fpath))
+                    labels.append(label)
+                    n_files += 1
+        if not texts:
+            raise DataError(
+                f"Directory '{path}' contains subfolders but no readable .txt files inside them."
+            )
+        ds = Dataset(kind="text_labeled", source=path, texts=texts, labels=labels, n_files=n_files)
+        return ds
+
+    # Case 2: flat directory of files -> merge by type.
+    all_records: List[Dict[str, Any]] = []
+    all_texts: List[str] = []
+    n_files = 0
+    for fname in files:
+        fpath = os.path.join(path, fname)
+        ext = os.path.splitext(fname)[1].lower()
+        if ext in TEXT_EXTENSIONS:
+            all_texts.append(_read_text_file(fpath))
+            n_files += 1
+        elif ext in JSON_EXTENSIONS:
+            all_records.extend(_read_json_records(fpath))
+            n_files += 1
+        elif ext in JSONL_EXTENSIONS:
+            all_records.extend(_read_jsonl_records(fpath))
+            n_files += 1
+        elif ext in CSV_EXTENSIONS:
+            all_records.extend(_read_csv_records(fpath))
+            n_files += 1
+        # silently skip unknown extensions (e.g. README, .gitkeep) -- but
+        # never silently skip *data*-looking files; this is only for
+        # incidental non-data files.
+
+    if all_records and all_texts:
+        raise DataError(
+            f"Directory '{path}' mixes plain text files with structured "
+            f"(json/csv) files. Please keep one data format per directory."
+        )
+    if all_records:
+        ds = _records_to_dataset(all_records, path)
+        ds.n_files = n_files
+        return ds
+    if all_texts:
+        return Dataset(kind="text", source=path, texts=all_texts, n_files=n_files)
+
+    raise DataError(
+        f"No supported data files found in '{path}'. Supported: "
+        f".txt, .md, .json, .jsonl, .csv, .tsv"
+    )
+
+
+def load_dataset(path: str) -> Dataset:
+    """Load a dataset from `path` (file or directory) into a `Dataset`."""
+    if not os.path.exists(path):
+        raise DataError(f"Path '{path}' does not exist.")
+
+    if os.path.isdir(path):
+        return _load_directory(path)
+
+    ext = os.path.splitext(path)[1].lower()
+    if ext in TEXT_EXTENSIONS:
+        text = _read_text_file(path)
+        if not text.strip():
+            raise DataError(f"'{path}' is empty.")
+        return Dataset(kind="text", source=path, texts=[text], n_files=1)
+    if ext in JSON_EXTENSIONS:
+        records = _read_json_records(path)
+        ds = _records_to_dataset(records, path)
+        ds.n_files = 1
+        return ds
+    if ext in JSONL_EXTENSIONS:
+        records = _read_jsonl_records(path)
+        ds = _records_to_dataset(records, path)
+        ds.n_files = 1
+        return ds
+    if ext in CSV_EXTENSIONS:
+        records = _read_csv_records(path)
+        ds = _records_to_dataset(records, path)
+        ds.n_files = 1
+        return ds
+
+    raise DataError(
+        f"Unsupported file type '{ext}' for '{path}'. Supported: "
+        f".txt, .md, .json, .jsonl, .csv, .tsv, or a directory containing these."
+    )