tcf-format 0.7.1__py3-none-any.whl

tcf/__init__.py

@@ -0,0 +1,106 @@
+"""TCF — Tabular Compact Format (pré-1.0; formato `#TCF.6` default, `#TCF.7`
+opt-in — ADR-0024 versionamento pré-1.0).
+
+API publica unificada (ADR-0014):
+
+    from tcf import encode, decode, SideOutputs
+
+    # Single-column (lista)
+    text = encode(["abc", "abcd", "abcde"])
+    values = decode(text)
+    assert values == ["abc", "abcd", "abcde"]
+
+    # Multi-column (dict)
+    text = encode({"id": ["1", "2"], "name": ["a", "b"]})
+    table = decode(text)
+    assert table == {"id": ["1", "2"], "name": ["a", "b"]}
+
+    # Side outputs opcional (debug, stats, schema)
+    side = SideOutputs()
+    text = encode(data, side_outputs=side)
+    print(side.hcc_trace)        # detector iterations
+    print(side.column_features)  # pre-pass features
+    # ... etc
+
+Encoder dispatcha por tipo (list vs dict). Decoder dispatcha pelo
+shebang (`#TCF.6 M` -> multi, senao -> single). Self-describing.
+
+## Componentes canonicos
+
+- `tcf.core.online`: **OBAT** (Online Bidirectional Affix Tokenizer).
+  Camada 1. Tokeniza strings via LCP + LCS. Codnome origem: `alg16`.
+- `tcf.composicional.syntax`: **HCC** (Hierarchical Compositional
+  Coding). Camada 2. Detector unificado + emit composicional. Codnome
+  origem: `M8.A`.
+- `tcf.composicional.hcc_seqrle`: HCC + seq-RLE near-identical (M10).
+- `tcf.encoder` / `tcf.decoder`: API publica unificada.
+- `tcf.multi`: implementacao interna multi-col + aliases deprecated.
+- `tcf.side_outputs`: recipiente opcional pra debug/stats.
+
+Ver `docs/algorithms/` para documentacao tecnica detalhada.
+
+## Validacao
+
+> Numeros abaixo sao probatorios: o TESTE mede, a prosa aponta. Guardioes
+> byte-canonical: `tests/test_core_rt.py` + `tests/test_regression_v1_baseline.py`
+> (baselines D1-D9/D17a), `tests/test_multi_col_rt.py` (multi-col),
+> `tests/test_real_world_snapshots.py` (bytes reais; GATE de qualquer mudanca
+> em pre-pass/OBAT/HCC). As % sao derivadas — ver o ADR citado em cada linha.
+
+Single-column (M10 canonical, ADR-0011):
+- D1-D9 sint: 1523B em 2981 raw = 51.1% ratio (RT 9/9)
+  [1523B pinado em test_core_rt.py + test_regression_v1_baseline.py]
+- Real-world Adult+TPC-H 57 cols: -11.73% weighted vs M9 puro
+  [bytes em test_real_world_snapshots.py; % derivada, ADR-0011]
+
+Multi-column (M10 + ADR-0013, T-EXP-MULTI-COL-SCALING):
+- D17a sint 13x4: 322B INVARIANT (preservado vs EXP-011)
+  [322B pinado em test_multi_col_rt.py + test_regression_v1_baseline.py]
+- Real-world 9 tabelas (Adult + TPC-H tier 1+2, 136k linhas):
+  -33.02% weighted vs raw, -31.46% vs single-col concat, RT 9/9
+  [bytes em test_real_world_snapshots.py; % derivada, ADR-0013]
+
+## Backward compat
+
+- `encode_table` / `decode_table` permanecem como aliases DEPRECATED
+  (emitem `DeprecationWarning`). Use `encode(dict)` / `decode(text)`.
+
+Para historia: `experiments/lab/dirty/notas/historia-dirty-lab.md`.
+"""
+
+from tcf.decoder import decode
+from tcf.encoder import encode
+from tcf.multi import decode_table, encode_table  # deprecated aliases
+from tcf.natures import (
+    SPEC_CPF, SPEC_CNPJ, SPEC_IP,
+    TemplatedCheckedSpec, TemplatedPaddedSpec,
+)
+from tcf.pipeline import PipelineConfig
+from tcf.schema import ColumnSchema, TableSchema, build_schema
+from tcf.side_outputs import SideOutputs
+
+# Pré-1.0 (ADR-0024): minor acompanha o formato (#TCF.7 -> 0.7); o PATCH (.1) e'
+# contador de release/correcao, DESACOPLADO do comportamento (nao muda a logica
+# nem o byte-output canonical). Sem compat rigida entre minors de dev; git e' o
+# mecanismo de reproducao. v1.0 = release solido futuro.
+__version__ = "0.7.1"
+
+__all__ = [
+    "encode",
+    "decode",
+    "SideOutputs",
+    "build_schema",
+    "TableSchema",
+    "ColumnSchema",
+    # Natures (ADR-0015):
+    "TemplatedCheckedSpec",
+    "TemplatedPaddedSpec",
+    "SPEC_CPF",
+    "SPEC_CNPJ",
+    "SPEC_IP",
+    # Pipeline toggles (T-CODE-LAYERED-PIPELINE Fase 1):
+    "PipelineConfig",
+    # Deprecated (mantidos pra migracao):
+    "encode_table",
+    "decode_table",
+]

tcf/_core/__init__.py

@@ -0,0 +1,13 @@
+"""tcf._core — aceleradores compilados OPCIONAIS (Cython).
+
+Cada modulo aqui tem fallback pure-Python no codigo canonical de src/tcf.
+Se a extensao compilada nao estiver presente (install sem compilador), o
+import falha silenciosamente e o pure-Python e' usado — output byte-identico.
+
+Atual:
+- detect.pyx -> _detect_compositions (H-PERF-06-v2 Fase B, ADR-0020).
+  Fallback: M8AVirtualRefsSyntax._detect_compositions em composicional/syntax.py.
+
+Build: best-effort durante `pip install` (hatch_build.py); ou local via
+`python -m build` / extensao compilada in-place.
+"""

tcf/_core/detect.pyx

@@ -0,0 +1,153 @@
+# cython: language_level=3, boundscheck=False, wraparound=False
+"""Acelerador Cython OPCIONAL de _detect_compositions (HCC, H-PERF-06-v2 Fase B).
+
+ADR-0020. Logica IDENTICA ao metodo pure-Python em
+composicional/syntax.py (M8AVirtualRefsSyntax._detect_compositions, pos-weld
+#15). Estruturas de dados continuam Python (Counter/dict/tuple/list) -> ordem
+de insercao e tie-break first-wins preservados byte-exato. So' adiciona
+cdef Py_ssize_t em contadores/comprimentos e cdef list nas listas quentes.
+Genexprs usam j/y pra evitar clash de escopo com os cdef.
+
+Se esta extensao nao compilar/importar, syntax.py usa o fallback pure-Python
+(output byte-identico). Validado: D1-D9=1523B, D17a=322B, fixtures real-world.
+Speedup ~2.1-2.3x no _detect_compositions.
+"""
+from collections import Counter
+
+
+def _detect_compositions(self, pieces_per_line, atom_count):
+    cdef Py_ssize_t next_alias = 1
+    cdef Py_ssize_t comp_acc_k = 0
+    alias_to_sub = {}
+    iter_traces = []
+
+    cdef Py_ssize_t li, a, b, i, K, R, n_refs
+    cdef Py_ssize_t n_est_ub, n_tam_min, n_tam, baseline, net, ub_net
+    cdef Py_ssize_t best_net, virtual_count, virt_pos, virt_alias
+    cdef Py_ssize_t alias_temp, virtual_id, ai
+    cdef list refs, new_refs, novos
+    cdef bint line_had_sub
+
+    while True:
+        contagem = Counter()
+        sub_first_line = {}
+        for li, pieces in enumerate(pieces_per_line):
+            if pieces is None:
+                continue
+            for p in pieces:
+                if p[0] == 'refs':
+                    refs = p[1]
+                    n_refs = len(refs)
+                    for a in range(n_refs):
+                        for b in range(a + 2, n_refs + 1):
+                            sub = tuple(refs[a:b])
+                            contagem[sub] += 1
+                            if sub not in sub_first_line:
+                                sub_first_line[sub] = li
+
+        # alias_first_line: primeiro li onde -alias aparece em body
+        alias_first_line = {}
+        for li, pieces in enumerate(pieces_per_line):
+            if pieces is None:
+                continue
+            for p in pieces:
+                if p[0] == 'refs':
+                    for ref in p[1]:
+                        if ref < 0:
+                            ai = -ref
+                            if ai not in alias_first_line:
+                                alias_first_line[ai] = li
+
+        # cheap upper-bound prune + running-max inline (ADR-0019)
+        n_est_ub = max(2, len(str(atom_count + comp_acc_k + len(contagem) + 9)))
+        n_tam_min = len(str(atom_count + comp_acc_k + 1))
+
+        candidates = []
+        best = None
+        best_net = 0
+        for sub, R in contagem.items():
+            if R < 2:
+                continue
+            K = len(sub)
+            ub_net = (R - 1) * (K * n_est_ub + (K - 1) - n_tam_min)
+            if ub_net <= best_net:
+                continue
+            virtual_count = sum(1 for y in sub if y < 0)
+            if virtual_count > 1:
+                continue
+            if virtual_count == 1:
+                virt_pos = next(j for j, y in enumerate(sub) if y < 0)
+                if virt_pos > 0:
+                    virt_alias = -sub[virt_pos]
+                    if alias_first_line.get(virt_alias,
+                                            float('inf')) >= sub_first_line[sub]:
+                        continue
+            baseline = self._estimate_baseline_chars(sub, atom_count, comp_acc_k)
+            n_tam = len(str(atom_count + comp_acc_k + K - 1))
+            if baseline <= n_tam:
+                continue
+            net = (R - 1) * (baseline - n_tam)
+            candidates.append((net, sub, R, baseline, n_tam))
+            if net > best_net:
+                best_net = net
+                best = (sub, R)
+
+        iter_info = {
+            'n_pairs': sum(1 for v in contagem.values() if v >= 2),
+            'n_candidates': len(candidates),
+            'candidates_sorted': sorted(candidates, reverse=True,
+                                        key=lambda c: c[0]),
+            'picked': best,
+            'iter_num': len(iter_traces) + 1,
+        }
+
+        if best is None:
+            iter_info['stopped'] = True
+            iter_traces.append(iter_info)
+            break
+
+        sub, R = best
+        alias_temp = next_alias
+        next_alias += 1
+        comp_acc_k += len(sub) - 1
+        alias_to_sub[alias_temp] = list(sub)
+        virtual_id = -alias_temp
+        iter_info['alias_temp'] = alias_temp
+        iter_info['lines_affected'] = []
+        iter_info['n_substituicoes'] = 0
+
+        K = len(sub)
+        for li in range(len(pieces_per_line)):
+            pieces = pieces_per_line[li]
+            if pieces is None:
+                continue
+            novos = []
+            line_had_sub = False
+            for p in pieces:
+                if p[0] != 'refs':
+                    novos.append(p)
+                    continue
+                refs = p[1]
+                new_refs = []
+                i = 0
+                n_refs = len(refs)
+                while i < n_refs:
+                    if (i + K <= n_refs and tuple(refs[i:i + K]) == sub):
+                        new_refs.append(virtual_id)
+                        i += K
+                        iter_info['n_substituicoes'] += 1
+                        line_had_sub = True
+                    else:
+                        new_refs.append(refs[i])
+                        i += 1
+                if new_refs:
+                    novos.append(('refs', new_refs))
+            if line_had_sub:
+                iter_info['lines_affected'].append(li + 1)
+            pieces_per_line[li] = novos
+
+        iter_traces.append(iter_info)
+        if len(iter_traces) >= 99:
+            break
+
+    return alias_to_sub, iter_traces

tcf/auto_cadence.py

@@ -0,0 +1,99 @@
+"""Detect cadence — heuristica pre-pass pra OBAT shape-preserve hint.
+
+CAMADA-0 pre-pass (ordem do pipeline: pre-pass -> OBAT em core/ -> HCC em
+composicional/). Irmaos pre-pass: column_features, auto_min_len, obat_shape.
+
+Welded canonical 2026-05-22 (T-CODE-PACOTE1-WELD-CANONICAL).
+Origem: `experiments/lab/clean/EXP-010-tcf-delta-aware-prototype/auto_pre.py`
+(welded 2026-05-17, refino real-world 2026-05-19 ADR-0008).
+
+Heuristica 2-regras:
+- Regra 1 (wrapper+counter): lengths uniformes nas primeiras N strings
+  + LCP+LCS / length >= threshold em pares consecutivos
+- Regra 2 (numeric high-cardinality, ADR-0008): todas primeiras 20
+  strings sao numericas + cardinalidade > 0.5
+
+Quando dispara, encoder usa `processar_with_hint(unicas, min_len,
+prefer_shape_consistency=True)` em vez de `processar(unicas, min_len)`
+canonical.
+
+Refatorado pra usar ColumnFeatures (H-DA-11c). Versao canonical
+recebe `analyze_column(values)` ja' calculada — evita recomputar
+features basicas.
+"""
+
+from __future__ import annotations
+
+from tcf.column_features import ColumnFeatures
+from tcf.core.online import lcp_len, lcs_len
+
+
+def detect_cadence_from_features(
+    features: ColumnFeatures,
+    strings_unicas: list[str],
+    n_sample: int = 5,
+    threshold: float = 0.7,
+    numeric_card_threshold: float = 0.5,
+) -> tuple[bool, dict]:
+    """Detecta se coluna tem cadencia estrutural via 2 regras.
+
+    Args:
+        features: ColumnFeatures basico (avg_len, card, is_numeric, sample, ...)
+        strings_unicas: unicas pra calcular LCP/LCS em pares consecutivos
+        n_sample: tamanho sample regra 1 (lengths uniformes + LCP/LCS)
+        threshold: limiar LCP+LCS / length na regra 1 (default 0.7)
+        numeric_card_threshold: limiar cardinalidade regra 2 (default 0.5)
+
+    Returns:
+        (detectou, info) — info contem rule_hit + detalhes.
+    """
+    info: dict = {
+        "n_strings_total": len(strings_unicas),
+        "n_sample": min(n_sample, len(strings_unicas)),
+        "threshold": threshold,
+        "numeric_card_threshold": numeric_card_threshold,
+        "rule_hit": None,
+    }
+
+    if len(strings_unicas) < 2:
+        info["reason"] = "muito poucas strings (<2)"
+        return False, info
+
+    sample = strings_unicas[:min(n_sample, len(strings_unicas))]
+    lengths = [len(s) for s in sample]
+    info["lengths"] = lengths
+    uniform_length = (len(set(lengths)) == 1) and lengths[0] > 0
+
+    # ---- Regra 1: wrapper+counter ----
+    if uniform_length:
+        L = lengths[0]
+        ratios = []
+        for i in range(1, len(sample)):
+            a, b = sample[i - 1], sample[i]
+            lcp = lcp_len(a, b)
+            lcs = lcs_len(a, b)
+            ratio = (lcp + lcs) / L
+            ratios.append({"pair": i, "lcp": lcp, "lcs": lcs,
+                           "ratio": round(ratio, 3)})
+        info["lcp_lcs_ratios"] = ratios
+        if ratios and all(r["ratio"] >= threshold for r in ratios):
+            avg = sum(r["ratio"] for r in ratios) / len(ratios)
+            info["rule_hit"] = "1-uniform-length-high-lcp-lcs"
+            info["reason"] = f"L={L}, all ratios >= {threshold} (avg={avg:.2f})"
+            return True, info
+
+    # ---- Regra 2: numeric high-cardinality (ADR-0008) ----
+    # Cardinalidade computada sobre values (rows) em ColumnFeatures
+    info["cardinality"] = round(features.cardinality, 3)
+    info["is_numeric"] = features.is_numeric
+
+    if features.is_numeric and features.cardinality > numeric_card_threshold:
+        info["rule_hit"] = "2-numeric-high-cardinality"
+        info["reason"] = (
+            f"numeric, cardinality={features.cardinality:.3f} "
+            f"> {numeric_card_threshold}"
+        )
+        return True, info
+
+    info["reason"] = "nenhuma regra acionou"
+    return False, info

tcf/auto_min_len.py

@@ -0,0 +1,89 @@
+"""Auto-detect min_len por coluna (canonical, ADR-0010, H-DA-11).
+
+CAMADA-0 pre-pass (ordem do pipeline: pre-pass -> OBAT em core/ -> HCC em
+composicional/). Irmaos pre-pass: column_features, auto_cadence, obat_shape.
+
+Heuristica v3 (decision tree shallow em avg_len + cardinality + is_numeric)
+capturou 99.5% do oracle real-world em Adult+TPC-H (sub-exp
+`experiments/lab/dirty/2026-05-21-h-da-11-auto-min-len/02-heuristica-v1/`).
+A heuristica so' dispara n>=100 (real-world); seu efeito em bytes e' guardado
+por `tests/test_real_world_snapshots.py` (GATE). O 99.5% e' derivado do
+sub-exp acima, nao pinado como literal.
+
+Gating `n >= 100`: datasets pequenos (D1-D9, exemplos sinteticos) usam
+default ml=3 — preserva M9 baseline EXATO (1615B). O caminho do gating
+(D1-D9 byte-canonical) e' guardado por `tests/test_core_rt.py`.
+
+API:
+- `detect_min_len_from_features(features, n_threshold=100)` — heuristica
+  pura que recebe `ColumnFeatures` ja' calculada
+- `detect_min_len(values, n_threshold=100)` — wrapper backward compat
+  que chama `analyze_column(values)` internamente
+
+Welded canonical 2026-05-22 (T-EXP-H-DA-11). Refatorado pra usar
+ColumnFeatures unificado em 2026-05-22 (T-CODE-H-DA-11c).
+"""
+
+from __future__ import annotations
+
+from tcf.column_features import ColumnFeatures, analyze_column
+
+
+def detect_min_len_from_features(
+    features: ColumnFeatures, n_threshold: int = 100
+) -> int:
+    """Detecta min_len otimo a partir de ColumnFeatures.
+
+    Args:
+        features: ColumnFeatures ja' calculada via analyze_column
+        n_threshold: limite inferior de rows pra aplicar heuristica
+            (default 100 — datasets menores usam ml=3 default, preserva
+            M9 baseline)
+
+    Returns:
+        int em {3, 4, 5, 6}.
+
+    Heuristica v3 (decision tree shallow):
+        - n < n_threshold: 3 (gating)
+        - card < 0.2: 3 (baixa-card seguro)
+        - avg >= 25: 6 (long-form)
+        - avg >= 8 + card >= 0.4: 6 (dates, mid-len high-card)
+        - avg >= 5 + is_num + card >= 0.8: 6 (numeric high-card)
+        - avg >= 12 + card >= 0.7: 5 (c_phone)
+        - avg >= 3 + card >= 0.2: 4 (IDs sequenciais)
+        - else: 3
+    """
+    if features.n_rows < n_threshold:
+        return 3
+
+    avg_len = features.avg_len
+    card = features.cardinality
+    is_num = features.is_numeric
+
+    if card < 0.2:
+        return 3
+    if avg_len >= 25:
+        return 6
+    if avg_len >= 8 and card >= 0.4:
+        return 6
+    if avg_len >= 5 and is_num and card >= 0.8:
+        return 6
+    if avg_len >= 12 and card >= 0.7:
+        return 5
+    if avg_len >= 3 and card >= 0.2:
+        return 4
+    return 3
+
+
+def detect_min_len(values: list[str], n_threshold: int = 100) -> int:
+    """Backward-compat wrapper: analisa values e retorna min_len.
+
+    Equivale a:
+        detect_min_len_from_features(analyze_column(values), n_threshold)
+
+    Mantido para callers que nao tem ColumnFeatures pre-computado.
+    Para pipelines novos com multiplas heuristicas, preferir chamar
+    `analyze_column(values)` uma vez e passar para cada
+    `detect_X_from_features`.
+    """
+    return detect_min_len_from_features(analyze_column(values), n_threshold)

tcf/column_features.py

@@ -0,0 +1,88 @@
+"""ColumnFeatures — pre-pass unificado de features de coluna.
+
+CAMADA-0 pre-pass (ordem do pipeline: pre-pass -> OBAT em core/ -> HCC em
+composicional/). E' o produtor de features que os irmaos pre-pass
+(auto_cadence, auto_min_len, obat_shape) consomem.
+
+Calcula features basicas em 1 passada O(N) sobre values:
+- n_rows, n_unicas
+- avg_len, cardinality
+- is_numeric (sample check)
+- sample (primeiras N strings)
+
+Heuristicas downstream (detect_min_len, detect_cadence, futuras
+detect_X de naturezas pre-tx) recebem ColumnFeatures imutavel e
+escolhem comportamento. Reduz duplicacao + permite reuso.
+
+Conexoes:
+- ADR-0010 (auto-detect min_len) — usa ColumnFeatures
+- ADR-0008 (detect_cadence) — quando weldar canonical, usara' ColumnFeatures
+- H-DA-11c — motivacao desta unificacao
+
+Welded 2026-05-22 (T-CODE-H-DA-11c).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+def _is_numeric_string(v: str) -> bool:
+    """Aceita int, float, negativos. Rejeita empty."""
+    if not v:
+        return False
+    try:
+        float(v)
+        return True
+    except (ValueError, TypeError):
+        return False
+
+
+@dataclass(frozen=True)
+class ColumnFeatures:
+    """Features imutaveis extraidas de uma coluna de strings.
+
+    Computado por `analyze_column(values)` em 1 passada O(N).
+    """
+    n_rows: int
+    n_unicas: int
+    avg_len: float
+    cardinality: float
+    is_numeric: bool
+    sample: tuple[str, ...]  # tuple pra ser hashable (dataclass frozen)
+
+
+def analyze_column(values: list[str], sample_size: int = 20) -> ColumnFeatures:
+    """Calcula features basicas de uma coluna em 1 passada.
+
+    Args:
+        values: lista de strings da coluna (com possiveis duplicatas)
+        sample_size: tamanho do sample pra check is_numeric (default 20)
+
+    Returns:
+        ColumnFeatures imutavel.
+
+    Edge cases:
+        - values vazio: retorna features zerados, is_numeric=False
+        - is_numeric: True so' se TODAS as strings do sample parsam float
+    """
+    n = len(values)
+    if n == 0:
+        return ColumnFeatures(
+            n_rows=0, n_unicas=0, avg_len=0.0,
+            cardinality=0.0, is_numeric=False, sample=(),
+        )
+
+    n_unicas = len(set(values))
+    avg_len = sum(len(v) for v in values) / n
+    sample = tuple(values[:min(sample_size, n)])
+    is_num = all(_is_numeric_string(v) for v in sample) if sample else False
+
+    return ColumnFeatures(
+        n_rows=n,
+        n_unicas=n_unicas,
+        avg_len=avg_len,
+        cardinality=n_unicas / n,
+        is_numeric=is_num,
+        sample=sample,
+    )

tcf/composicional/__init__.py

@@ -0,0 +1,27 @@
+"""HCC — Hierarchical Compositional Coding.
+
+Camada 2 do TCF. Consome tokens raiz de OBAT e produz texto TCF
+compacto via:
+- Detector iterativo greedy de sub-tuplas reusaveis
+- Emit com operadores `~` (cria ref auto-nomeado) e `,` (concat efemero)
+- Pairwise left-assoc binarization
+- Range `a..b` como caso particular de composicao por sequencia
+- Output: sem brackets, LF only
+
+`syntax.py` e' adaptacao byte-exata em logica de
+`experiments/lab/dirty/old/2026-05-16-M8-virtual-refs-clean-output/M8-A-detector-unificado/syntax.py`
+(codnome de origem: `M8.A`).
+
+Adaptacoes vs original (welding step 2, 2026-05-17):
+- `from online import ...` → `from tcf.core.online import ...`
+- `from syntax_base import ...` → `from tcf.core.syntax_base import ...`
+- removido `sys.path.insert(...)` (Python package resolve naturalmente)
+
+Logica de encode/decode permanece byte-exata. Validado por M12 + M13 + M14.
+
+Ver `docs/algorithms/HCC.md` para detalhamento (estrutura,
+sub-linguagem matematica, body-order constraint, diferencial vs
+Re-Pair / Sequitur / LZW).
+
+Para uso via API publica: `from tcf import encode, decode`.
+"""