tejtm 0.1.0__py3-none-any.whl

tejtm/__init__.py

@@ -0,0 +1,30 @@
+from tejtm.coercion import coerce_record, coerce_value
+from tejtm.engine import MappingEngine
+from tejtm.models import (
+    DEFAULT_WEIGHTS,
+    FieldMapping,
+    MappingConfig,
+    MappingResult,
+    ScorerResult,
+    SourceColumn,
+    SourceProfile,
+    TargetField,
+    TargetSchema,
+    TypeGuess,
+)
+
+__all__ = [
+    "coerce_record",
+    "coerce_value",
+    "DEFAULT_WEIGHTS",
+    "FieldMapping",
+    "MappingConfig",
+    "MappingEngine",
+    "MappingResult",
+    "ScorerResult",
+    "SourceColumn",
+    "SourceProfile",
+    "TargetField",
+    "TargetSchema",
+    "TypeGuess",
+]

tejtm/assignment.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+
+def assign_one_to_one(
+    scores: list[list[float]],
+    min_confidence: float,
+) -> list[tuple[int, int] | None]:
+    """Return (row, col) per row index, or None when unassigned."""
+    if not scores:
+        return []
+    n_rows = len(scores)
+    n_cols = len(scores[0]) if scores else 0
+    if n_cols == 0:
+        return [None] * n_rows
+
+    pairs = _hungarian_maximize(scores)
+    result: list[tuple[int, int] | None] = [None] * n_rows
+    for row, col in pairs:
+        if scores[row][col] >= min_confidence:
+            result[row] = (row, col)
+    return result
+
+
+def _hungarian_maximize(scores: list[list[float]]) -> list[tuple[int, int]]:
+    n_rows = len(scores)
+    n_cols = len(scores[0])
+    n = max(n_rows, n_cols)
+    cost = [[1.0 - scores[i][j] if i < n_rows and j < n_cols else 1.0 for j in range(n)] for i in range(n)]
+    assignment = _linear_sum_assignment(cost)
+    out: list[tuple[int, int]] = []
+    for i, j in assignment:
+        if i < n_rows and j < n_cols:
+            out.append((i, j))
+    return out
+
+
+def _linear_sum_assignment(cost: list[list[float]]) -> list[tuple[int, int]]:
+    try:
+        from scipy.optimize import linear_sum_assignment  # type: ignore import-not-found
+
+        import numpy as np
+
+        row_ind, col_ind = linear_sum_assignment(np.array(cost))
+        return [(int(r), int(c)) for r, c in zip(row_ind, col_ind, strict=True)]
+    except ImportError:
+        return _greedy_maximize(cost)
+
+
+def _greedy_maximize(cost: list[list[float]]) -> list[tuple[int, int]]:
+    n = len(cost)
+    candidates: list[tuple[float, int, int]] = []
+    for i in range(n):
+        for j in range(n):
+            candidates.append((cost[i][j], i, j))
+    candidates.sort(key=lambda t: t[0])
+    used_rows: set[int] = set()
+    used_cols: set[int] = set()
+    pairs: list[tuple[int, int]] = []
+    for _, i, j in candidates:
+        if i in used_rows or j in used_cols:
+            continue
+        used_rows.add(i)
+        used_cols.add(j)
+        pairs.append((i, j))
+    return pairs

tejtm/coercion.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from datetime import date, datetime
+from typing import Any, Mapping
+
+from tejtm.typing.detectors import detect_semantic_type
+
+
+def coerce_value(value: Any, semantic_type: str) -> Any:
+    if value is None or value == "":
+        return None
+    text = str(value).strip()
+    if semantic_type == "boolean":
+        low = text.casefold()
+        if low in ("true", "yes", "y", "1"):
+            return True
+        if low in ("false", "no", "n", "0"):
+            return False
+        return text
+    if semantic_type == "integer":
+        return int(text.replace(",", ""))
+    if semantic_type == "number" or semantic_type == "currency":
+        return float(text.replace(",", "").replace("$", "").replace("€", "").replace("£", ""))
+    if semantic_type == "date":
+        return date.fromisoformat(text[:10])
+    if semantic_type == "datetime":
+        return datetime.fromisoformat(text.replace("Z", "+00:00")[:19])
+    return text
+
+
+def coerce_record(record: Mapping[str, Any], field_types: Mapping[str, str]) -> dict[str, Any]:
+    out: dict[str, Any] = {}
+    for key, value in record.items():
+        stype = field_types.get(key)
+        if stype is None:
+            out[key] = value
+            continue
+        try:
+            out[key] = coerce_value(value, stype)
+        except (ValueError, TypeError):
+            out[key] = value
+    return out
+
+
+def infer_field_types_from_samples(
+    records: list[Mapping[str, Any]],
+    column: str,
+) -> str:
+    samples = [str(r.get(column, "")) for r in records if r.get(column) is not None]
+    return detect_semantic_type(samples).label

tejtm/engine.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+from tejtm.assignment import assign_one_to_one
+from tejtm.models import (
+    FieldMapping,
+    MappingConfig,
+    MappingResult,
+    SourceColumn,
+    SourceProfile,
+    TargetField,
+    TargetSchema,
+)
+from tejtm.scorers import DEFAULT_SCORERS
+from tejtm.scorers.base import Scorer, ScoringContext, aggregate_scores
+from tejtm.scorers.header_embedding import HeaderEmbeddingScorer, _get_embedder
+from tejtm.typing.detectors import detect_semantic_type
+
+
+class MappingEngine:
+    def __init__(self, scorers: tuple[Scorer, ...] | None = None) -> None:
+        self._scorers = scorers if scorers is not None else DEFAULT_SCORERS
+
+    def propose_mappings(
+        self,
+        source: SourceProfile,
+        target: TargetSchema,
+        *,
+        config: MappingConfig | None = None,
+    ) -> MappingResult:
+        cfg = config or MappingConfig()
+        column_types = {col.name: detect_semantic_type(col.samples) for col in source.columns}
+        field_embeddings: dict[str, tuple[float, ...]] | None = {} if _get_embedder() else None
+        context = ScoringContext(
+            config=cfg,
+            column_types=column_types,
+            field_embeddings=field_embeddings,
+        )
+
+        fields = list(target.fields)
+        columns = list(source.columns)
+        field_ids = [f.id for f in fields]
+
+        override_mappings: list[FieldMapping] = []
+        override_sources: set[str] = set()
+        override_targets: set[str] = set()
+        for src_name, tgt_id in cfg.overrides.items():
+            if any(c.name == src_name for c in columns) and tgt_id in field_ids:
+                override_mappings.append(
+                    FieldMapping(
+                        source_column=src_name,
+                        target_field=tgt_id,
+                        confidence=1.0,
+                        breakdown=(),
+                    )
+                )
+                override_sources.add(src_name)
+                override_targets.add(tgt_id)
+
+        score_matrix: list[list[float]] = []
+        breakdowns: list[list[tuple]] = []
+        for col in columns:
+            row_scores: list[float] = []
+            row_breakdowns: list[tuple] = []
+            for fld in fields:
+                parts = self._score_pair(col, fld, context)
+                conf = aggregate_scores(parts)
+                row_scores.append(conf)
+                row_breakdowns.append(tuple(parts))
+            score_matrix.append(row_scores)
+            breakdowns.append(row_breakdowns)
+
+        assignments = assign_one_to_one(score_matrix, cfg.min_confidence)
+        mappings: list[FieldMapping] = list(override_mappings)
+        mapped_sources = set(override_sources)
+        mapped_targets = set(override_targets)
+
+        for row_idx, pair in enumerate(assignments):
+            if pair is None:
+                continue
+            col = columns[row_idx]
+            if col.name in mapped_sources:
+                continue
+            _, col_idx = pair
+            fld = fields[col_idx]
+            if fld.id in mapped_targets:
+                continue
+            conf = score_matrix[row_idx][col_idx]
+            mappings.append(
+                FieldMapping(
+                    source_column=col.name,
+                    target_field=fld.id,
+                    confidence=conf,
+                    breakdown=breakdowns[row_idx][col_idx],
+                )
+            )
+            mapped_sources.add(col.name)
+            mapped_targets.add(fld.id)
+
+        unmapped = tuple(c.name for c in columns if c.name not in mapped_sources)
+        unmatched = tuple(f.id for f in fields if f.id not in mapped_targets)
+        mappings.sort(key=lambda m: (-m.confidence, m.source_column))
+        return MappingResult(
+            mappings=tuple(mappings),
+            unmapped_columns=unmapped,
+            unmatched_fields=unmatched,
+        )
+
+    def _score_pair(self, column: SourceColumn, field: TargetField, context: ScoringContext) -> list:
+        parts = []
+        for scorer in self._scorers:
+            if isinstance(scorer, HeaderEmbeddingScorer) and context.field_embeddings is None:
+                continue
+            result = scorer.score(column, field, context)
+            if result is not None:
+                parts.append(result)
+        return parts

tejtm/models.py

@@ -0,0 +1,138 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Mapping, Sequence
+
+
+@dataclass(frozen=True, slots=True)
+class SourceColumn:
+    name: str
+    samples: Sequence[str] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class SourceProfile:
+    columns: Sequence[SourceColumn]
+
+    def __post_init__(self) -> None:
+        names = [c.name for c in self.columns]
+        if len(names) != len(set(names)):
+            raise ValueError("SourceProfile column names must be unique")
+
+
+@dataclass(frozen=True, slots=True)
+class TargetField:
+    id: str
+    labels: Sequence[str] = ()
+    aliases: Sequence[str] = ()
+    expected_semantic_type: str | None = None
+    description: str | None = None
+    exemplar_samples: Sequence[str] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class TargetSchema:
+    fields: Sequence[TargetField]
+
+    def __post_init__(self) -> None:
+        ids = [f.id for f in self.fields]
+        if len(ids) != len(set(ids)):
+            raise ValueError("TargetSchema field ids must be unique")
+
+
+@dataclass(frozen=True, slots=True)
+class TypeGuess:
+    label: str
+    confidence: float
+    evidence: str = ""
+
+
+@dataclass(frozen=True, slots=True)
+class ScorerResult:
+    scorer: str
+    score: float
+    weight: float
+    reason: str = ""
+
+
+@dataclass(frozen=True, slots=True)
+class FieldMapping:
+    source_column: str
+    target_field: str
+    confidence: float
+    breakdown: tuple[ScorerResult, ...] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class MappingResult:
+    mappings: tuple[FieldMapping, ...]
+    unmapped_columns: tuple[str, ...]
+    unmatched_fields: tuple[str, ...]
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "mappings": [
+                {
+                    "source_column": m.source_column,
+                    "target_field": m.target_field,
+                    "confidence": m.confidence,
+                    "breakdown": [
+                        {
+                            "scorer": b.scorer,
+                            "score": b.score,
+                            "weight": b.weight,
+                            "reason": b.reason,
+                        }
+                        for b in m.breakdown
+                    ],
+                }
+                for m in self.mappings
+            ],
+            "unmapped_columns": list(self.unmapped_columns),
+            "unmatched_fields": list(self.unmatched_fields),
+        }
+
+    @classmethod
+    def from_dict(cls, data: Mapping[str, Any]) -> MappingResult:
+        mappings = tuple(
+            FieldMapping(
+                source_column=m["source_column"],
+                target_field=m["target_field"],
+                confidence=float(m["confidence"]),
+                breakdown=tuple(
+                    ScorerResult(
+                        scorer=b["scorer"],
+                        score=float(b["score"]),
+                        weight=float(b["weight"]),
+                        reason=b.get("reason", ""),
+                    )
+                    for b in m.get("breakdown", [])
+                ),
+            )
+            for m in data.get("mappings", [])
+        )
+        return cls(
+            mappings=mappings,
+            unmapped_columns=tuple(data.get("unmapped_columns", [])),
+            unmatched_fields=tuple(data.get("unmatched_fields", [])),
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class MappingConfig:
+    weights: Mapping[str, float] = field(default_factory=dict)
+    min_confidence: float = 0.55
+    intent_margin: float = 0.08
+    overrides: Mapping[str, str] = field(default_factory=dict)
+
+    def weight(self, scorer: str, default: float) -> float:
+        return self.weights.get(scorer, default)
+
+
+DEFAULT_WEIGHTS: dict[str, float] = {
+    "exact_name": 1.0,
+    "fuzzy_name": 0.45,
+    "semantic_type": 0.25,
+    "header_embedding": 0.20,
+    "value_intent": 0.30,
+}

tejtm/normalize.py

@@ -0,0 +1,44 @@
+import re
+import unicodedata
+from typing import Sequence
+
+_CAMEL_BOUNDARY = re.compile(r"(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])")
+_NON_ALNUM = re.compile(r"[^\w\s]+", re.UNICODE)
+_WS = re.compile(r"\s+")
+
+
+def normalize_identifier(text: str) -> str:
+    if not text:
+        return ""
+    text = unicodedata.normalize("NFKC", text)
+    text = text.strip().casefold()
+    text = _NON_ALNUM.sub(" ", text)
+    text = _WS.sub(" ", text).strip()
+    return text
+
+
+def tokenize_identifier(text: str) -> tuple[str, ...]:
+    if not text:
+        return ()
+    raw = unicodedata.normalize("NFKC", text.strip())
+    raw = _NON_ALNUM.sub(" ", raw)
+    parts: list[str] = []
+    for chunk in _WS.split(raw):
+        if not chunk:
+            continue
+        for piece in _CAMEL_BOUNDARY.sub(" ", chunk.replace("_", " ")).split():
+            piece = normalize_identifier(piece)
+            if piece:
+                parts.append(piece)
+    return tuple(parts)
+
+
+def candidate_names(field_id: str, labels: Sequence[str], aliases: Sequence[str]) -> tuple[str, ...]:
+    seen: set[str] = set()
+    out: list[str] = []
+    for raw in (field_id, *labels, *aliases):
+        norm = normalize_identifier(raw)
+        if norm and norm not in seen:
+            seen.add(norm)
+            out.append(norm)
+    return tuple(out)

tejtm/pydantic_adapter.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Mapping, Sequence, TypeVar
+
+from tejtm.coercion import coerce_record
+from tejtm.engine import MappingEngine
+from tejtm.models import MappingConfig, MappingResult, SourceColumn, SourceProfile, TargetField, TargetSchema
+
+T = TypeVar("T")
+
+
+@dataclass(frozen=True, slots=True)
+class ImportResult:
+    valid_rows: list[Any]
+    errors: list[dict[str, Any]]
+    mapping: MappingResult
+    column_rename: dict[str, str]
+
+
+def _pydantic_field_names(model: type) -> list[tuple[str, list[str], str | None, list[str]]]:
+    try:
+        from pydantic import BaseModel
+    except ImportError as exc:
+        raise ImportError("Install tejtm with pydantic extra: pip install tejtm[pydantic]") from exc
+    if not isinstance(model, type) or not issubclass(model, BaseModel):
+        raise TypeError("model must be a Pydantic BaseModel subclass")
+
+    out: list[tuple[str, list[str], str | None, list[str]]] = []
+    for name, field_info in model.model_fields.items():
+        aliases: list[str] = []
+        extra = field_info.json_schema_extra
+        if isinstance(extra, dict):
+            raw = extra.get("aliases") or extra.get("tejtm_aliases")
+            if isinstance(raw, str):
+                aliases.append(raw)
+            elif isinstance(raw, (list, tuple)):
+                aliases.extend(str(a) for a in raw)
+        title = field_info.title or name
+        expected = None
+        exemplars: list[str] = []
+        if isinstance(extra, dict):
+            if extra.get("semantic_type"):
+                expected = str(extra["semantic_type"])
+            raw_ex = extra.get("tejtm_exemplars") or extra.get("exemplar_samples")
+            if isinstance(raw_ex, str):
+                exemplars.append(raw_ex)
+            elif isinstance(raw_ex, (list, tuple)):
+                exemplars.extend(str(x) for x in raw_ex)
+        out.append((name, [title, *aliases], expected, exemplars))
+    return out
+
+
+def target_schema_from_model(model: type) -> TargetSchema:
+    fields = [
+        TargetField(
+            id=name,
+            labels=(labels[0],) if labels else (),
+            aliases=tuple(labels[1:]),
+            expected_semantic_type=expected,
+            exemplar_samples=tuple(exemplars),
+        )
+        for name, labels, expected, exemplars in _pydantic_field_names(model)
+    ]
+    return TargetSchema(fields=fields)
+
+
+def propose_and_apply(
+    model: type[T],
+    rows: Sequence[Mapping[str, Any]],
+    *,
+    config: MappingConfig | None = None,
+    engine: MappingEngine | None = None,
+) -> ImportResult:
+    if not rows:
+        return ImportResult(valid_rows=[], errors=[], mapping=MappingResult((), (), ()), column_rename={})
+
+    columns = list(rows[0].keys())
+    samples_by_col: dict[str, list[str]] = {c: [] for c in columns}
+    for row in rows[:50]:
+        for c in columns:
+            val = row.get(c)
+            if val is not None and str(val).strip():
+                samples_by_col[c].append(str(val))
+
+    source = SourceProfile(
+        columns=[SourceColumn(name=c, samples=tuple(samples_by_col[c][:20])) for c in columns]
+    )
+    target = target_schema_from_model(model)
+    eng = engine or MappingEngine()
+    mapping = eng.propose_mappings(source, target, config=config)
+    rename = {m.source_column: m.target_field for m in mapping.mappings}
+
+    valid: list[Any] = []
+    errors: list[dict[str, Any]] = []
+    field_types = {f.id: f.expected_semantic_type or "freetext" for f in target.fields if f.expected_semantic_type}
+
+    for idx, row in enumerate(rows):
+        renamed = {rename.get(k, k): v for k, v in row.items()}
+        coerced = coerce_record(renamed, field_types)
+        try:
+            valid.append(model.model_validate(coerced))
+        except Exception as exc:
+            errors.append({"row": idx + 1, "message": str(exc)})
+
+    return ImportResult(valid_rows=valid, errors=errors, mapping=mapping, column_rename=rename)

tejtm/scorers/__init__.py

@@ -0,0 +1,22 @@
+from tejtm.scorers.exact import ExactNameScorer
+from tejtm.scorers.fuzzy import FuzzyNameScorer
+from tejtm.scorers.header_embedding import HeaderEmbeddingScorer
+from tejtm.scorers.semantic_type import SemanticTypeScorer
+from tejtm.scorers.value_intent import ValueIntentScorer
+
+DEFAULT_SCORERS = (
+    ExactNameScorer(),
+    FuzzyNameScorer(),
+    SemanticTypeScorer(),
+    HeaderEmbeddingScorer(),
+    ValueIntentScorer(),
+)
+
+__all__ = [
+    "DEFAULT_SCORERS",
+    "ExactNameScorer",
+    "FuzzyNameScorer",
+    "HeaderEmbeddingScorer",
+    "SemanticTypeScorer",
+    "ValueIntentScorer",
+]

tejtm/scorers/base.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Protocol
+
+from tejtm.models import MappingConfig, ScorerResult, SourceColumn, TargetField, TypeGuess
+
+
+@dataclass
+class ScoringContext:
+    config: MappingConfig
+    column_types: dict[str, TypeGuess] = field(default_factory=dict)
+    field_embeddings: dict[str, tuple[float, ...]] | None = None
+
+
+class Scorer(Protocol):
+    name: str
+
+    def score(
+        self,
+        column: SourceColumn,
+        field: TargetField,
+        context: ScoringContext,
+    ) -> ScorerResult | None: ...
+
+
+def aggregate_scores(parts: list[ScorerResult]) -> float:
+    if not parts:
+        return 0.0
+    num = sum(p.score * p.weight for p in parts)
+    den = sum(p.weight for p in parts)
+    if den <= 0:
+        return 0.0
+    return max(0.0, min(1.0, num / den))

tejtm/scorers/exact.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from tejtm.models import DEFAULT_WEIGHTS, ScorerResult, SourceColumn, TargetField
+from tejtm.normalize import candidate_names, normalize_identifier
+from tejtm.scorers.base import ScoringContext
+
+
+class ExactNameScorer:
+    name = "exact_name"
+
+    def score(self, column: SourceColumn, field: TargetField, context: ScoringContext) -> ScorerResult | None:
+        col = normalize_identifier(column.name)
+        if not col:
+            return None
+        names = candidate_names(field.id, field.labels, field.aliases)
+        if col in names:
+            w = context.config.weight(self.name, DEFAULT_WEIGHTS[self.name])
+            return ScorerResult(scorer=self.name, score=1.0, weight=w, reason="exact normalized match")
+        return None

tejtm/scorers/fuzzy.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from rapidfuzz import fuzz
+
+from tejtm.models import DEFAULT_WEIGHTS, ScorerResult, SourceColumn, TargetField
+from tejtm.normalize import candidate_names, normalize_identifier, tokenize_identifier
+from tejtm.scorers.base import ScoringContext
+
+
+class FuzzyNameScorer:
+    name = "fuzzy_name"
+
+    def score(self, column: SourceColumn, field: TargetField, context: ScoringContext) -> ScorerResult | None:
+        col_norm = normalize_identifier(column.name)
+        col_tokens = " ".join(tokenize_identifier(column.name))
+        if not col_norm:
+            return None
+        best = 0.0
+        for cand in candidate_names(field.id, field.labels, field.aliases):
+            scores = [
+                fuzz.ratio(col_norm, cand) / 100.0,
+                fuzz.WRatio(col_norm, cand) / 100.0,
+                fuzz.token_sort_ratio(col_tokens or col_norm, cand) / 100.0,
+                fuzz.token_set_ratio(col_tokens or col_norm, cand) / 100.0,
+            ]
+            best = max(best, max(scores))
+        w = context.config.weight(self.name, DEFAULT_WEIGHTS[self.name])
+        return ScorerResult(scorer=self.name, score=best, weight=w, reason=f"fuzzy best={best:.2f}")

tejtm/scorers/header_embedding.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import math
+
+from tejtm.models import DEFAULT_WEIGHTS, ScorerResult, SourceColumn, TargetField
+from tejtm.normalize import candidate_names
+from tejtm.scorers.base import ScoringContext
+
+_embedder = None
+
+
+def _get_embedder():
+    global _embedder
+    if _embedder is False:
+        return None
+    if _embedder is not None and _embedder is not False:
+        return _embedder
+    try:
+        from fastembed import TextEmbedding
+
+        _embedder = TextEmbedding(
+            model_name="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
+        )
+    except ImportError:
+        _embedder = False
+    return _embedder if _embedder is not False else None
+
+
+def _cosine(a: tuple[float, ...], b: tuple[float, ...]) -> float:
+    dot = sum(x * y for x, y in zip(a, b, strict=True))
+    na = math.sqrt(sum(x * x for x in a))
+    nb = math.sqrt(sum(y * y for y in b))
+    if na == 0 or nb == 0:
+        return 0.0
+    return max(0.0, min(1.0, dot / (na * nb)))
+
+
+class HeaderEmbeddingScorer:
+    name = "header_embedding"
+
+    def score(self, column: SourceColumn, field: TargetField, context: ScoringContext) -> ScorerResult | None:
+        model = _get_embedder()
+        if model is None:
+            return None
+        if context.field_embeddings is None:
+            return None
+        col_vec = _embed_text(model, column.name)
+        if col_vec is None:
+            return None
+        best = 0.0
+        for cand in candidate_names(field.id, field.labels, field.aliases):
+            fvec = context.field_embeddings.get(cand)
+            if fvec is None:
+                fvec = _embed_text(model, cand)
+                if fvec:
+                    context.field_embeddings[cand] = fvec
+            if fvec:
+                best = max(best, _cosine(col_vec, fvec))
+        w = context.config.weight(self.name, DEFAULT_WEIGHTS[self.name])
+        return ScorerResult(scorer=self.name, score=best, weight=w, reason=f"embedding cosine={best:.2f}")
+
+
+def _embed_text(model, text: str) -> tuple[float, ...] | None:
+    if not text.strip():
+        return None
+    vectors = list(model.embed([text]))
+    if not vectors:
+        return None
+    return tuple(float(x) for x in vectors[0])

tejtm/scorers/semantic_type.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from tejtm.models import DEFAULT_WEIGHTS, ScorerResult, SourceColumn, TargetField
+from tejtm.scorers.base import ScoringContext
+from tejtm.typing.detectors import type_compatibility
+
+
+class SemanticTypeScorer:
+    name = "semantic_type"
+
+    def score(self, column: SourceColumn, field: TargetField, context: ScoringContext) -> ScorerResult | None:
+        if field.expected_semantic_type is None:
+            return None
+        guess = context.column_types.get(column.name)
+        if guess is None:
+            return None
+        score = type_compatibility(guess, field.expected_semantic_type)
+        if score <= 0:
+            return ScorerResult(
+                scorer=self.name,
+                score=0.0,
+                weight=context.config.weight(self.name, DEFAULT_WEIGHTS[self.name]),
+                reason=f"source={guess.label} expected={field.expected_semantic_type}",
+            )
+        w = context.config.weight(self.name, DEFAULT_WEIGHTS[self.name])
+        return ScorerResult(
+            scorer=self.name,
+            score=score,
+            weight=w,
+            reason=f"source={guess.label} expected={field.expected_semantic_type}",
+        )

tejtm/scorers/value_intent.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from tejtm.models import DEFAULT_WEIGHTS, ScorerResult, SourceColumn, TargetField
+from tejtm.scorers.base import ScoringContext
+from tejtm.value_profile import expectation_from_type, signature_similarity, value_signature
+
+
+class ValueIntentScorer:
+    name = "value_intent"
+
+    def score(self, column: SourceColumn, field: TargetField, context: ScoringContext) -> ScorerResult | None:
+        column_sig = value_signature(column.samples)
+        if column_sig is None:
+            return None
+
+        scores: list[tuple[float, str]] = []
+
+        if field.expected_semantic_type:
+            expected_sig = expectation_from_type(field.expected_semantic_type)
+            sim = signature_similarity(column_sig, expected_sig)
+            scores.append((sim, f"type_profile:{field.expected_semantic_type}:{sim:.2f}"))
+
+        if field.exemplar_samples:
+            exemplar_sig = value_signature(field.exemplar_samples)
+            if exemplar_sig is not None:
+                sim = signature_similarity(column_sig, exemplar_sig)
+                scores.append((sim, f"exemplar:{sim:.2f}"))
+
+        embed_score = _description_value_embedding_score(column, field, context)
+        if embed_score is not None:
+            scores.append((embed_score, f"desc_embed:{embed_score:.2f}"))
+
+        if not scores:
+            return None
+
+        best_score, reason = max(scores, key=lambda t: t[0])
+        if best_score <= 0:
+            return None
+        w = context.config.weight(self.name, DEFAULT_WEIGHTS[self.name])
+        return ScorerResult(scorer=self.name, score=best_score, weight=w, reason=reason)
+
+
+def _description_value_embedding_score(
+    column: SourceColumn,
+    field: TargetField,
+    context: ScoringContext,
+) -> float | None:
+    if not field.description:
+        return None
+    if context.field_embeddings is None:
+        return None
+    from tejtm.scorers.header_embedding import _cosine, _embed_text, _get_embedder
+
+    model = _get_embedder()
+    if model is None:
+        return None
+    samples = [s for s in column.samples if s and str(s).strip()][:5]
+    if not samples:
+        return None
+    col_text = " | ".join(samples)
+    field_text = f"{field.id} {field.description}"
+    col_vec = _embed_text(model, col_text)
+    field_vec = context.field_embeddings.get(f"desc:{field.id}")
+    if field_vec is None:
+        field_vec = _embed_text(model, field_text)
+        if field_vec:
+            context.field_embeddings[f"desc:{field.id}"] = field_vec
+    if col_vec is None or field_vec is None:
+        return None
+    return _cosine(col_vec, field_vec)

tejtm/serialization.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from typing import Any
+
+from tejtm.models import MappingResult
+
+
+def mapping_to_yaml(result: MappingResult) -> str:
+    try:
+        import yaml
+    except ImportError as exc:
+        raise ImportError("Install tejtm with yaml extra: pip install tejtm[yaml]") from exc
+    return yaml.safe_dump(result.to_dict(), sort_keys=False)
+
+
+def mapping_from_yaml(text: str) -> MappingResult:
+    try:
+        import yaml
+    except ImportError as exc:
+        raise ImportError("Install tejtm with yaml extra: pip install tejtm[yaml]") from exc
+    data: Any = yaml.safe_load(text)
+    return MappingResult.from_dict(data or {})

tejtm/typing/__init__.py

@@ -0,0 +1,3 @@
+from tejtm.typing.detectors import detect_semantic_type, type_compatibility
+
+__all__ = ["detect_semantic_type", "type_compatibility"]

tejtm/typing/detectors.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import re
+from datetime import datetime
+from typing import Callable, Sequence
+
+from tejtm.models import TypeGuess
+
+_EMAIL = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$", re.IGNORECASE)
+_URL = re.compile(r"^https?://\S+$", re.IGNORECASE)
+_UUID = re.compile(
+    r"^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$",
+    re.IGNORECASE,
+)
+_PHONE = re.compile(r"^\+?[\d\s().-]{7,}$")
+_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
+_DATETIME = re.compile(r"^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}")
+_TIME = re.compile(r"^\d{1,2}:\d{2}(:\d{2})?$")
+_CURRENCY = re.compile(
+    r"^[\$€£]\s*-?\d{1,3}(?:,\d{3})*(?:\.\d+)?$|"
+    r"^-?\d{1,3}(?:,\d{3})+(?:\.\d+)?$"
+)
+_BOOL = {"true", "false", "yes", "no", "0", "1", "y", "n"}
+
+
+def _match_rate(samples: Sequence[str], predicate: Callable[[str], bool]) -> tuple[float, str]:
+    values = [s.strip() for s in samples if s and str(s).strip()]
+    if not values:
+        return 0.0, "no samples"
+    hits = sum(1 for v in values if predicate(v))
+    rate = hits / len(values)
+    return rate, f"{hits}/{len(values)} matched"
+
+
+def detect_semantic_type(samples: Sequence[str]) -> TypeGuess:
+    checks: list[tuple[str, Callable[[str], bool]]] = [
+        ("email", lambda v: bool(_EMAIL.match(v))),
+        ("url", lambda v: bool(_URL.match(v))),
+        ("uuid", lambda v: bool(_UUID.match(v))),
+        ("phone", _is_phone),
+        ("datetime", lambda v: bool(_DATETIME.match(v))),
+        ("date", lambda v: bool(_DATE.match(v)) and _parse_date(v)),
+        ("time", lambda v: bool(_TIME.match(v))),
+        ("boolean", lambda v: v.casefold() in _BOOL),
+        ("currency", lambda v: bool(_CURRENCY.match(v.replace(" ", "")))),
+        ("integer", _is_int),
+        ("number", _is_number),
+    ]
+    best_label = "freetext"
+    best_rate = 0.0
+    best_evidence = ""
+    for label, pred in checks:
+        rate, evidence = _match_rate(samples, pred)
+        if rate > best_rate:
+            best_rate = rate
+            best_label = label
+            best_evidence = evidence
+    if best_rate < 0.5:
+        return TypeGuess(label="freetext", confidence=best_rate, evidence=best_evidence or "weak signal")
+    return TypeGuess(label=best_label, confidence=best_rate, evidence=best_evidence)
+
+
+def _is_phone(value: str) -> bool:
+    if _DATE.match(value) or _DATETIME.match(value) or _TIME.match(value):
+        return False
+    if not _PHONE.match(value):
+        return False
+    digits = sum(c.isdigit() for c in value)
+    if digits < 7:
+        return False
+    return not value.isdigit()
+
+
+def _is_int(value: str) -> bool:
+    try:
+        int(value.replace(",", ""))
+        return "." not in value
+    except ValueError:
+        return False
+
+
+def _is_number(value: str) -> bool:
+    try:
+        float(value.replace(",", "").replace("$", "").replace("€", "").replace("£", ""))
+        return True
+    except ValueError:
+        return False
+
+
+def _parse_date(value: str) -> bool:
+    try:
+        datetime.strptime(value, "%Y-%m-%d")
+        return True
+    except ValueError:
+        return False
+
+
+def type_compatibility(source: TypeGuess, expected: str | None) -> float:
+    if expected is None:
+        return 0.0
+    if source.label == expected:
+        return min(1.0, 0.5 + 0.5 * source.confidence)
+    compatible = {
+        ("integer", "number"),
+        ("number", "integer"),
+        ("datetime", "date"),
+        ("date", "datetime"),
+        ("number", "currency"),
+        ("currency", "number"),
+    }
+    if (source.label, expected) in compatible:
+        return 0.6 * source.confidence
+    return 0.0

tejtm/value_profile.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Sequence
+
+from tejtm.models import TypeGuess
+from tejtm.typing.detectors import detect_semantic_type, type_compatibility
+
+_WS = re.compile(r"\s")
+
+
+@dataclass(frozen=True, slots=True)
+class ValueSignature:
+    semantic_type: str
+    type_confidence: float
+    avg_length: float
+    digit_ratio: float
+    space_ratio: float
+    distinct_ratio: float
+    uppercase_ratio: float
+    token_prefix: str = ""
+
+
+def value_signature(samples: Sequence[str]) -> ValueSignature | None:
+    values = [str(s).strip() for s in samples if s is not None and str(s).strip()]
+    if not values:
+        return None
+    guess = detect_semantic_type(values)
+    total_chars = sum(len(v) for v in values)
+    digits = sum(sum(c.isdigit() for c in v) for v in values)
+    spaces = sum(len(_WS.findall(v)) for v in values)
+    distinct = len(set(values)) / len(values)
+    upper = sum(1 for v in values if v.isupper()) / len(values)
+    return ValueSignature(
+        semantic_type=guess.label,
+        type_confidence=guess.confidence,
+        avg_length=total_chars / len(values),
+        digit_ratio=digits / total_chars if total_chars else 0.0,
+        space_ratio=spaces / total_chars if total_chars else 0.0,
+        distinct_ratio=distinct,
+        uppercase_ratio=upper,
+        token_prefix=_dominant_prefix(values),
+    )
+
+
+def signature_similarity(a: ValueSignature, b: ValueSignature) -> float:
+    type_score = type_compatibility(
+        TypeGuess(a.semantic_type, a.type_confidence),
+        b.semantic_type,
+    )
+    if type_score <= 0 and a.semantic_type != b.semantic_type:
+        type_part = 0.15 if _types_loosely_compatible(a.semantic_type, b.semantic_type) else 0.0
+    else:
+        type_part = type_score
+
+    prefix_part = 1.0 if a.token_prefix and a.token_prefix == b.token_prefix else 0.0
+    shape = 1.0 - min(
+        1.0,
+        (
+            abs(a.digit_ratio - b.digit_ratio)
+            + abs(a.space_ratio - b.space_ratio)
+            + abs(a.distinct_ratio - b.distinct_ratio) * 0.5
+            + abs(a.uppercase_ratio - b.uppercase_ratio) * 0.5
+            + min(1.0, abs(a.avg_length - b.avg_length) / max(a.avg_length, b.avg_length, 1.0))
+        )
+        / 3.0,
+    )
+    blended = 0.45 * type_part + 0.35 * shape + 0.20 * prefix_part
+    return max(0.0, min(1.0, blended))
+
+
+def expectation_from_type(expected_semantic_type: str) -> ValueSignature:
+    profiles: dict[str, ValueSignature] = {
+        "email": ValueSignature("email", 1.0, 22.0, 0.0, 0.0, 0.9, 0.0),
+        "phone": ValueSignature("phone", 1.0, 12.0, 0.5, 0.05, 0.9, 0.0),
+        "uuid": ValueSignature("uuid", 1.0, 36.0, 0.0, 0.0, 1.0, 0.0),
+        "url": ValueSignature("url", 1.0, 30.0, 0.05, 0.0, 0.9, 0.0),
+        "date": ValueSignature("date", 1.0, 10.0, 0.3, 0.0, 0.8, 0.0),
+        "datetime": ValueSignature("datetime", 1.0, 19.0, 0.35, 0.05, 0.8, 0.0),
+        "integer": ValueSignature("integer", 1.0, 4.0, 0.9, 0.0, 0.7, 0.0),
+        "number": ValueSignature("number", 1.0, 6.0, 0.4, 0.0, 0.7, 0.0),
+        "currency": ValueSignature("currency", 1.0, 8.0, 0.5, 0.0, 0.6, 0.0),
+        "boolean": ValueSignature("boolean", 1.0, 4.0, 0.0, 0.0, 0.3, 0.0),
+        "freetext": ValueSignature("freetext", 0.5, 16.0, 0.1, 0.12, 0.85, 0.1),
+    }
+    return profiles.get(expected_semantic_type, profiles["freetext"])
+
+
+def _dominant_prefix(values: Sequence[str]) -> str:
+    buckets: dict[str, int] = {}
+    for v in values:
+        token = v.strip()
+        if "-" in token:
+            key = token.split("-", 1)[0].casefold()
+        elif len(token) >= 3:
+            key = token[:3].casefold()
+        else:
+            key = token.casefold()
+        buckets[key] = buckets.get(key, 0) + 1
+    if not buckets:
+        return ""
+    return max(buckets, key=buckets.get)
+
+
+def _types_loosely_compatible(a: str, b: str) -> bool:
+    pairs = {("integer", "number"), ("date", "datetime"), ("freetext", "uuid")}
+    return (a, b) in pairs or (b, a) in pairs

tejtm-0.1.0.dist-info/METADATA

@@ -0,0 +1,113 @@
+Metadata-Version: 2.5
+Name: tejtm
+Version: 0.1.0
+Summary: Schema mapping with composite confidence for tabular imports
+Requires-Python: >=3.11
+Requires-Dist: rapidfuzz>=3.0
+Provides-Extra: dev
+Requires-Dist: email-validator>=2.0; extra == 'dev'
+Requires-Dist: openpyxl>=3.1; extra == 'dev'
+Requires-Dist: pydantic>=2.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Provides-Extra: phone
+Requires-Dist: phonenumbers>=8.13; extra == 'phone'
+Provides-Extra: pydantic
+Requires-Dist: email-validator>=2.0; extra == 'pydantic'
+Requires-Dist: pydantic>=2.0; extra == 'pydantic'
+Provides-Extra: semantic
+Requires-Dist: fastembed>=0.4; extra == 'semantic'
+Provides-Extra: yaml
+Requires-Dist: pyyaml>=6; extra == 'yaml'
+Description-Content-Type: text/markdown
+
+# tejtm
+
+tejtm maps messy import columns to a target schema you define. It returns one-to-one pairs with a composite confidence and a per-scorer breakdown. It does not read files, join records, or validate rows.
+
+## Install
+
+```bash
+pip install -e ".[dev]"
+```
+
+Optional extras:
+
+```bash
+pip install -e ".[semantic,pydantic,yaml]"
+```
+
+`semantic` adds FastEmbed header similarity. `pydantic` builds a target schema from a model. `yaml` saves mapping results.
+
+## Map columns to a target schema
+
+The library takes column names plus sample values. You keep file I/O in your app.
+
+```python
+from tejtm import MappingEngine, MappingConfig, SourceColumn, SourceProfile, TargetField, TargetSchema
+
+source = SourceProfile(columns=[
+    SourceColumn("phone", samples=("+1 555-0100",)),
+    SourceColumn("contact", samples=("user@example.com",)),
+    SourceColumn("Cust Name", samples=("Ada Lovelace",)),
+])
+target = TargetSchema(fields=[
+    TargetField("phone_number", aliases=("phone",), expected_semantic_type="phone"),
+    TargetField("email", expected_semantic_type="email"),
+    TargetField("full_name", aliases=("name", "customer name")),
+])
+result = MappingEngine().propose_mappings(
+    source,
+    target,
+    config=MappingConfig(min_confidence=0.5),
+)
+for mapping in result.mappings:
+    print(mapping.source_column, "->", mapping.target_field, mapping.confidence)
+print("unmapped", result.unmapped_columns)
+print("unmatched", result.unmatched_fields)
+```
+
+You should see `phone -> phone_number`, `contact -> email`, and `Cust Name -> full_name`. Weak pairs stay out of `mappings` and land in `unmapped_columns` or `unmatched_fields`.
+
+Default `min_confidence` is `0.55`. Lower it for messy vendor headers, then auto-apply only the pairs above a threshold you pick.
+
+## Apply the mapping
+
+tejtm does not rewrite rows. Use `source_column` and `target_field` to rename keys in your own loop. For cell casting after rename, call `coerce_value` or `coerce_record`. For a Pydantic model, see [How to customize mapping](docs/customization.md).
+
+## Fixture results
+
+The six directories under `tests/fixtures/scenarios/` are the only sector samples in this repo. They are small synthetic tables, not a customer corpus. Treat a fixture as passing when every `expect_mappings` pair lands in `result.mappings` at that fixture's `min_confidence`.
+
+Measured 29 Oct 2026 in this checkout, FastEmbed absent, two identical runs. `header_embedding` abstained. Scores come from `exact_name`, `fuzzy_name`, `semantic_type`, and `value_intent` only. The limiter is fixture size (2 or 3 columns) and those lowered thresholds, not production files.
+
+| Fixture | Format | min_confidence | Mapped / expected | Lowest mapped pair |
+| --- | --- | --- | --- | --- |
+| `crm_contacts` | csv | 0.45 | 3 / 3 | `contact` -> `email` (0.549) |
+| `edge_duplicate_generic_headers` | csv | 0.35 | 2 / 2 | `ref_a` -> `order_reference` (0.601) |
+| `finance_ledger` | xlsx | 0.4 | 3 / 3 | `GL Code` -> `ledger_account` (0.752) |
+| `healthcare_appointments` | csv | 0.4 | 3 / 3 | `Clinic` -> `facility_name` (0.480) |
+| `hr_payroll` | csv | 0.4 | 3 / 3 | `Gross Pay` -> `gross_amount` (0.569) |
+| `warehouse_inventory` | csv | 0.4 | 3 / 3 | `Bin Loc` -> `storage_location` (0.807) |
+
+All 17 expected pairs mapped at those fixture thresholds. At the library default of `0.55`, 15 of 17 still map. The two that drop are `contact` -> `email` in `crm_contacts` (0.549) and `Clinic` -> `facility_name` in `healthcare_appointments` (0.480). See [About mapping limits](docs/limitations.md) for why demos sit below the default.
+
+`.venv/bin/python -m pytest -q` reported 52 passed in this checkout. That count is the test suite, not a mapping-accuracy figure.
+
+Regenerate the table:
+
+```bash
+.venv/bin/python tests/support/fixture_stats.py
+```
+
+## What to read next
+
+- [How to customize mapping](docs/customization.md) covers weights, overrides, custom scorers, exemplars, Pydantic, and YAML.
+- [About mapping limits](docs/limitations.md) covers joins, duplicate headers, optional extras, and unused config.
+- [Product spec](docs/specs/schema-mapping-spec.md) is the v1 contract.
+- [Glossary](docs/GLOSSARY.md) defines source column, target field, and value intent.
+
+Run tests with the project virtualenv:
+
+```bash
+.venv/bin/python -m pytest -q
+```

tejtm-0.1.0.dist-info/RECORD

@@ -0,0 +1,21 @@
+tejtm/__init__.py,sha256=VAnT4aWGRuKJd0zL3Ghx3rCoSsaFDCuCafcn4r0RLTM,584
+tejtm/assignment.py,sha256=kKLhc_R30G6XbUJ9BpHUzT0s9MUP083vb595-KUz-JU,2063
+tejtm/coercion.py,sha256=1w2UMNMWizkxD3aFyIgni-CGuHVY60ZGkaJcL_J3B18,1632
+tejtm/engine.py,sha256=ARrqP056Fp11eFPP2e9SF4dPPO_rVCUZ9fy0vh0E_IM,4321
+tejtm/models.py,sha256=rIehZReUv6qIlzVFUOFJ-lvtAmJrjrWr01jgeKhfjTc,3892
+tejtm/normalize.py,sha256=ze54_b9_2NYixbVTvDZVckHupooMUonPWojMsaVoGmo,1315
+tejtm/pydantic_adapter.py,sha256=jYb0itc51lsPjkYYmx80HbJjG1bT6faMG0xqYuzxLLs,3982
+tejtm/serialization.py,sha256=PVZ_BgA0qOiCESbBa9-yccZYfEiIr7Ok97UctcX458Y,656
+tejtm/value_profile.py,sha256=2tr8lTPtLW-lyknDtHuL7rE7M9TofCT8GjP6yVoWjM4,4053
+tejtm/scorers/__init__.py,sha256=3cnOd2QnmqJtoa0JMLNOhWnx891TkXli4EP-Y_ARx1I,590
+tejtm/scorers/base.py,sha256=R93JjO45TjGLtMqdG9-gVBrjPBNHEx_xiskDeYkUJ1o,849
+tejtm/scorers/exact.py,sha256=-NesDzCycBX5gYbbWJYqJTv7fmbR0SbuV8NppIon9Mc,780
+tejtm/scorers/fuzzy.py,sha256=It7H_ME19WKDd_lX5JY0MRvy-PSiWaFff_n58a4Ztao,1212
+tejtm/scorers/header_embedding.py,sha256=FuhSZUMrOo20q7yIrB1_oLV-1jk4tYe7b13adZcvkaU,2234
+tejtm/scorers/semantic_type.py,sha256=A_o6lw6WoMhuuIksRXx5Lln9mO5IA851w2vRpp6JkJ4,1211
+tejtm/scorers/value_intent.py,sha256=wld6P2n2KSpVs5aoFDxEZ7RKgmK5AfK202LiT3kwSuc,2623
+tejtm/typing/__init__.py,sha256=lp8D-9bz25HKpp3i13dMdBFr-CB11L8Bzr81C7vd4Qk,134
+tejtm/typing/detectors.py,sha256=mgBj6HHHMC_iijnivtZH0YIgJbIT1M3jrGI-qJ1gIlM,3619
+tejtm-0.1.0.dist-info/METADATA,sha256=xVoAyLiNfZz2IVhEiV3k9332tLEShEyd0LPCiEMX_Eo,5067
+tejtm-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+tejtm-0.1.0.dist-info/RECORD,,

tejtm-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any