jdatamunch-mcp 0.1.0__tar.gz

jdatamunch_mcp-0.1.0/PKG-INFO

@@ -0,0 +1,24 @@
+Metadata-Version: 2.4
+Name: jdatamunch-mcp
+Version: 0.1.0
+Summary: Token-efficient MCP server for tabular data retrieval via CSV/Excel indexing
+Project-URL: Homepage, https://github.com/jgravelle/jdatamunch-mcp
+Project-URL: Repository, https://github.com/jgravelle/jdatamunch-mcp
+Project-URL: Bug Tracker, https://github.com/jgravelle/jdatamunch-mcp/issues
+Author-email: "J. Gravelle" <j@gravelle.us>
+Requires-Python: >=3.10
+Requires-Dist: charset-normalizer>=3.0.0
+Requires-Dist: mcp<2.0.0,>=1.10.0
+Provides-Extra: all
+Requires-Dist: anthropic>=0.40.0; extra == 'all'
+Requires-Dist: google-generativeai>=0.8.0; extra == 'all'
+Requires-Dist: openpyxl>=3.1.0; extra == 'all'
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.40.0; extra == 'anthropic'
+Provides-Extra: excel
+Requires-Dist: openpyxl>=3.1.0; extra == 'excel'
+Provides-Extra: gemini
+Requires-Dist: google-generativeai>=0.8.0; extra == 'gemini'
+Description-Content-Type: text/markdown
+
+# jdatamunch-mcp

jdatamunch_mcp-0.1.0/README.md

@@ -0,0 +1 @@
+# jdatamunch-mcp

jdatamunch_mcp-0.1.0/pyproject.toml

@@ -0,0 +1,45 @@
+[project]
+name = "jdatamunch-mcp"
+version = "0.1.0"
+description = "Token-efficient MCP server for tabular data retrieval via CSV/Excel indexing"
+readme = "README.md"
+requires-python = ">=3.10"
+authors = [
+    { name = "J. Gravelle", email = "j@gravelle.us" },
+]
+dependencies = [
+    "mcp>=1.10.0,<2.0.0",
+    "charset-normalizer>=3.0.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/jgravelle/jdatamunch-mcp"
+Repository = "https://github.com/jgravelle/jdatamunch-mcp"
+"Bug Tracker" = "https://github.com/jgravelle/jdatamunch-mcp/issues"
+
+[project.optional-dependencies]
+excel = ["openpyxl>=3.1.0"]
+anthropic = ["anthropic>=0.40.0"]
+gemini = ["google-generativeai>=0.8.0"]
+all = ["openpyxl>=3.1.0", "anthropic>=0.40.0", "google-generativeai>=0.8.0"]
+
+[project.scripts]
+jdatamunch-mcp = "jdatamunch_mcp.server:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/jdatamunch_mcp"]
+
+[dependency-groups]
+dev = [
+    "pytest>=9.0.2",
+    "pytest-asyncio>=1.3.0",
+    "pytest-cov>=7.0.0",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+asyncio_mode = "auto"

jdatamunch_mcp-0.1.0/server.json

@@ -0,0 +1,21 @@
+{
+  "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
+  "name": "io.github.jgravelle/jdatamunch-mcp",
+  "title": "jDataMunch MCP",
+  "description": "Token-efficient MCP server for tabular data retrieval. Index CSV/Excel files, query rows, aggregate — 99%+ token savings vs raw file reads.",
+  "version": "0.1.0",
+  "packages": [
+    {
+      "registryType": "pypi",
+      "identifier": "jdatamunch-mcp",
+      "version": "0.1.0",
+      "transport": {
+        "type": "stdio"
+      }
+    }
+  ],
+  "repository": {
+    "url": "https://github.com/jgravelle/jdatamunch-mcp",
+    "source": "github"
+  }
+}

jdatamunch_mcp-0.1.0/src/jdatamunch_mcp/__init__.py

@@ -0,0 +1,3 @@
+"""jdatamunch-mcp: Token-efficient MCP server for tabular data retrieval."""
+
+__version__ = "0.1.0"

jdatamunch_mcp-0.1.0/src/jdatamunch_mcp/config.py

@@ -0,0 +1,25 @@
+"""Environment variable handling and defaults for jdatamunch-mcp."""
+
+import os
+from pathlib import Path
+from typing import Optional
+
+
+def get_index_path(override: Optional[str] = None) -> Path:
+    """Return the base index storage path."""
+    if override:
+        return Path(override)
+    return Path(os.environ.get("DATA_INDEX_PATH", str(Path.home() / ".data-index")))
+
+
+def get_max_rows() -> int:
+    return int(os.environ.get("JDATAMUNCH_MAX_ROWS", "5000000"))
+
+
+def get_share_savings() -> bool:
+    return os.environ.get("JDATAMUNCH_SHARE_SAVINGS", "1") != "0"
+
+
+def get_use_ai_summaries() -> bool:
+    v = os.environ.get("JDATAMUNCH_USE_AI_SUMMARIES", "true").lower()
+    return v not in ("false", "0", "no", "off")

jdatamunch_mcp-0.1.0/src/jdatamunch_mcp/parser/__init__.py

@@ -0,0 +1,34 @@
+"""Parser registry for tabular file formats."""
+
+from pathlib import Path
+from typing import Optional
+
+from .types import ParsedDataset
+from .csv_parser import parse_csv
+
+
+def parse_file(
+    path: str,
+    encoding: Optional[str] = None,
+    delimiter: Optional[str] = None,
+    header_row: int = 0,
+    sheet: Optional[str] = None,
+) -> ParsedDataset:
+    """Parse a tabular file and return a streaming ParsedDataset."""
+    p = Path(path)
+    suffix = p.suffix.lower()
+
+    if suffix in (".csv", ".tsv", ".txt"):
+        return parse_csv(path, encoding=encoding, delimiter=delimiter, header_row=header_row)
+    elif suffix in (".xlsx", ".xls"):
+        try:
+            from .excel_parser import parse_excel
+            return parse_excel(path, sheet=sheet, header_row=header_row)
+        except ImportError:
+            raise ValueError(
+                "Excel support requires openpyxl: pip install 'jdatamunch-mcp[excel]'"
+            )
+    else:
+        raise ValueError(
+            f"Unsupported file format: {suffix!r}. Supported: .csv, .tsv, .xlsx, .xls"
+        )

jdatamunch_mcp-0.1.0/src/jdatamunch_mcp/parser/csv_parser.py

@@ -0,0 +1,107 @@
+"""Streaming CSV parser with auto-encoding detection."""
+
+import csv
+import os
+from pathlib import Path
+from typing import Generator, Optional
+
+from .types import ColumnInfo, ParsedDataset
+
+
+def _detect_encoding(path: str) -> str:
+    """Detect file encoding using charset-normalizer."""
+    try:
+        from charset_normalizer import from_path
+        result = from_path(path, cp_isolation=["utf-8", "latin-1", "cp1252"])
+        if result.best():
+            return str(result.best().encoding)
+    except Exception:
+        pass
+    return "utf-8"
+
+
+def _detect_delimiter(sample: str) -> str:
+    """Detect CSV delimiter using csv.Sniffer."""
+    try:
+        dialect = csv.Sniffer().sniff(sample, delimiters=",\t|;")
+        return dialect.delimiter
+    except Exception:
+        return ","
+
+
+def _row_generator(
+    path: str,
+    encoding: str,
+    delimiter: str,
+    header_row: int,
+) -> Generator:
+    """Yield data rows as lists of strings, skipping the header row."""
+    with open(path, newline="", encoding=encoding, errors="replace") as f:
+        reader = csv.reader(f, delimiter=delimiter)
+        for i, row in enumerate(reader):
+            if i == header_row:
+                continue  # skip header
+            yield row
+
+
+def parse_csv(
+    path: str,
+    encoding: Optional[str] = None,
+    delimiter: Optional[str] = None,
+    header_row: int = 0,
+) -> ParsedDataset:
+    """Parse a CSV file and return a streaming ParsedDataset.
+
+    The row_iterator is a fresh generator each time this is called.
+    Column names come from the header_row.
+    """
+    path = str(Path(path).resolve())
+    file_size = os.path.getsize(path)
+
+    # Detect encoding from file bytes
+    if not encoding:
+        encoding = _detect_encoding(path)
+
+    # Read a sample for delimiter sniffing
+    with open(path, newline="", encoding=encoding, errors="replace") as f:
+        sample = f.read(65536)
+
+    # Detect delimiter
+    if not delimiter:
+        if path.lower().endswith(".tsv"):
+            delimiter = "\t"
+        else:
+            delimiter = _detect_delimiter(sample)
+
+    # Parse header row to get column names
+    header: list = []
+    with open(path, newline="", encoding=encoding, errors="replace") as f:
+        reader = csv.reader(f, delimiter=delimiter)
+        for i, row in enumerate(reader):
+            if i == header_row:
+                header = row
+                break
+
+    columns = [ColumnInfo(name=name.strip(), position=i) for i, name in enumerate(header)]
+
+    # Estimate total row count from file size + sample density
+    sample_lines = sample.count("\n")
+    if sample_lines > 1:
+        bytes_per_row = len(sample.encode(encoding, errors="replace")) / sample_lines
+        estimated_rows = max(0, int(file_size / bytes_per_row) - 1)
+    else:
+        estimated_rows = 0
+
+    metadata = {
+        "encoding": encoding,
+        "delimiter": delimiter,
+        "header_row": header_row,
+        "estimated_rows": estimated_rows,
+        "file_size": file_size,
+    }
+
+    return ParsedDataset(
+        columns=columns,
+        row_iterator=_row_generator(path, encoding, delimiter, header_row),
+        metadata=metadata,
+    )

jdatamunch_mcp-0.1.0/src/jdatamunch_mcp/parser/types.py

@@ -0,0 +1,19 @@
+"""Core dataclasses for parsed tabular datasets."""
+
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass
+class ColumnInfo:
+    """Metadata about a single column parsed from the file header."""
+    name: str
+    position: int
+
+
+@dataclass
+class ParsedDataset:
+    """Result of parsing a tabular file. Row iterator is lazy/streaming."""
+    columns: list        # list[ColumnInfo]
+    row_iterator: Any    # Generator[list[str], None, None] — yields lists of raw strings
+    metadata: dict       # encoding, delimiter, header_row, estimated_rows, file_size

jdatamunch_mcp-0.1.0/src/jdatamunch_mcp/profiler/__init__.py

@@ -0,0 +1 @@
+"""Column profiling for tabular datasets."""

jdatamunch_mcp-0.1.0/src/jdatamunch_mcp/profiler/column_profiler.py

@@ -0,0 +1,299 @@
+"""Single-pass streaming column profiler.
+
+Processes rows one at a time using per-column accumulators.
+Designed to work with index_local.py's main loop where profiling
+and SQLite loading happen in the same pass over the data.
+"""
+
+import re
+from dataclasses import dataclass, field
+from typing import Any, Optional
+
+_NULL_VALUES = frozenset([
+    "", "null", "NULL", "none", "None", "N/A", "n/a", "NA", "na",
+    "NaN", "nan", "-", ".", "#N/A", "#NA", "#NULL!", "n.a.", "N.A.",
+])
+
+# Type rank: lower = more specific
+_TYPE_RANK = {"integer": 0, "float": 1, "datetime": 2, "string": 3}
+_TYPE_FROM_RANK = {0: "integer", 1: "float", 2: "datetime", 3: "string"}
+
+MAX_CARDINALITY_TRACK = 5_000   # stop adding new keys to value_counts after this
+SAMPLE_SIZE = 10                 # distinct non-null samples to collect
+RESERVOIR_SIZE = 10_000          # numeric values for approximate median
+
+VALUE_INDEX_CARDINALITY_LIMIT = 1_000  # full value map stored if cardinality <= this
+TOP_VALUES_LIMIT = 50                   # top values stored for high-cardinality columns
+
+# Common datetime patterns (regex → strptime format string)
+_DATETIME_PATTERNS = [
+    (re.compile(r"^\d{4}-\d{2}-\d{2}$"), "%Y-%m-%d"),
+    (re.compile(r"^\d{2}/\d{2}/\d{4}$"), "%m/%d/%Y"),
+    (re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}"), "%Y-%m-%dT%H:%M:%S"),
+    (re.compile(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}"), "%Y-%m-%d %H:%M:%S"),
+    (re.compile(r"^\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}"), "%m/%d/%Y %H:%M:%S"),
+    # US date + 12h time with AM/PM (e.g. "01/15/2020 12:00:00 AM")
+    (re.compile(r"^\d{1,2}/\d{1,2}/\d{4} \d{1,2}:\d{2}:\d{2} [AP]M$"), "%m/%d/%Y %I:%M:%S %p"),
+]
+
+
+def _is_datetime_str(value: str) -> bool:
+    for rx, _ in _DATETIME_PATTERNS:
+        if rx.match(value):
+            return True
+    return False
+
+
+def _get_datetime_format(value: str) -> Optional[str]:
+    for rx, fmt in _DATETIME_PATTERNS:
+        if rx.match(value):
+            return fmt
+    return None
+
+
+@dataclass
+class _ColAcc:
+    """Per-column accumulator updated once per row."""
+    name: str
+    position: int
+    # Type tracking (rank only advances upward)
+    type_rank: int = 0
+    # Row counts
+    count: int = 0        # non-null rows
+    null_count: int = 0
+    # Numeric stats (valid when type_rank <= 1)
+    num_min: float = field(default_factory=lambda: float("inf"))
+    num_max: float = field(default_factory=lambda: float("-inf"))
+    num_sum: float = 0.0
+    # Reservoir for approximate median (first RESERVOIR_SIZE numeric values)
+    reservoir: list = field(default_factory=list)
+    # Cardinality / value frequency
+    value_counts: dict = field(default_factory=dict)
+    cardinality_overflow: bool = False
+    # Samples: first SAMPLE_SIZE distinct non-null values seen
+    samples: list = field(default_factory=list)
+    _samples_set: set = field(default_factory=set)
+    # Datetime range (valid when type_rank == 2)
+    dt_min: Optional[str] = None
+    dt_max: Optional[str] = None
+    dt_format: Optional[str] = None
+
+
+def update_acc(acc: _ColAcc, raw_value: str) -> None:
+    """Update accumulator with one raw string value from the CSV."""
+    stripped = raw_value.strip() if raw_value else ""
+
+    if stripped in _NULL_VALUES:
+        acc.null_count += 1
+        return
+
+    acc.count += 1
+
+    # --- Type detection & promotion ---
+    if acc.type_rank == 0:  # currently integer
+        try:
+            int(stripped)
+        except ValueError:
+            acc.type_rank = 1  # promote to float
+
+    if acc.type_rank == 1:  # currently float
+        try:
+            float(stripped)
+        except ValueError:
+            # Check datetime before falling to string
+            if _is_datetime_str(stripped):
+                acc.type_rank = 2
+            else:
+                acc.type_rank = 3  # string
+
+    if acc.type_rank == 2:  # currently datetime
+        if not _is_datetime_str(stripped):
+            acc.type_rank = 3  # string
+        elif acc.dt_format is None:
+            acc.dt_format = _get_datetime_format(stripped)
+
+    # --- Numeric stats ---
+    if acc.type_rank <= 1:
+        try:
+            num = float(stripped)
+            if num < acc.num_min:
+                acc.num_min = num
+            if num > acc.num_max:
+                acc.num_max = num
+            acc.num_sum += num
+            if len(acc.reservoir) < RESERVOIR_SIZE:
+                acc.reservoir.append(num)
+        except ValueError:
+            pass
+
+    # --- Datetime min/max ---
+    if acc.type_rank == 2:
+        if acc.dt_min is None or stripped < acc.dt_min:
+            acc.dt_min = stripped
+        if acc.dt_max is None or stripped > acc.dt_max:
+            acc.dt_max = stripped
+
+    # --- Cardinality / value counts ---
+    if stripped in acc.value_counts:
+        acc.value_counts[stripped] += 1
+    elif not acc.cardinality_overflow:
+        if len(acc.value_counts) < MAX_CARDINALITY_TRACK:
+            acc.value_counts[stripped] = 1
+        else:
+            acc.cardinality_overflow = True
+            acc.value_counts[stripped] = 1
+
+    # --- Samples ---
+    if len(acc.samples) < SAMPLE_SIZE and stripped not in acc._samples_set:
+        acc.samples.append(stripped)
+        acc._samples_set.add(stripped)
+
+
+def _compute_median(reservoir: list) -> Optional[float]:
+    if not reservoir:
+        return None
+    sorted_vals = sorted(reservoir)
+    n = len(sorted_vals)
+    mid = n // 2
+    if n % 2 == 0:
+        return (sorted_vals[mid - 1] + sorted_vals[mid]) / 2.0
+    return float(sorted_vals[mid])
+
+
+@dataclass
+class ColumnProfile:
+    """Fully computed profile for a single column."""
+    name: str
+    position: int
+    type: str              # "integer", "float", "datetime", "string"
+    count: int             # non-null row count
+    null_count: int
+    null_pct: float
+    cardinality: int
+    cardinality_is_exact: bool
+    is_unique: bool
+    is_primary_key_candidate: bool
+    min: Optional[Any]
+    max: Optional[Any]
+    mean: Optional[float]
+    median: Optional[float]
+    sample_values: list
+    value_index: Optional[dict]   # full {value: count} for cardinality <= 1000
+    top_values: Optional[list]    # [{"value": ..., "count": ...}] for high-cardinality
+    datetime_min: Optional[str] = None
+    datetime_max: Optional[str] = None
+    datetime_format: Optional[str] = None
+    ai_summary: Optional[str] = None
+
+
+def finalize_profile(acc: _ColAcc) -> ColumnProfile:
+    """Build a ColumnProfile from a completed _ColAcc."""
+    total = acc.count + acc.null_count
+    null_pct = round(acc.null_count / total * 100, 1) if total > 0 else 0.0
+    col_type = _TYPE_FROM_RANK[acc.type_rank]
+
+    cardinality = len(acc.value_counts)
+    cardinality_is_exact = not acc.cardinality_overflow
+
+    is_unique = (cardinality_is_exact and cardinality == acc.count and acc.count > 0)
+    is_pk_candidate = (
+        is_unique
+        and acc.null_count == 0
+        and col_type in ("integer", "string")
+    )
+
+    # Numeric stats
+    if col_type in ("integer", "float") and acc.count > 0:
+        raw_min = acc.num_min
+        raw_max = acc.num_max
+        mean_val = round(acc.num_sum / acc.count, 4)
+        median_val = _compute_median(acc.reservoir)
+        if col_type == "integer":
+            min_val = int(raw_min) if raw_min != float("inf") else None
+            max_val = int(raw_max) if raw_max != float("-inf") else None
+            median_val = round(median_val, 1) if median_val is not None else None
+        else:
+            min_val = raw_min if raw_min != float("inf") else None
+            max_val = raw_max if raw_max != float("-inf") else None
+    else:
+        min_val = max_val = mean_val = median_val = None
+
+    # Convert sample values to their native type
+    samples: list = []
+    for s in acc.samples:
+        if col_type == "integer":
+            try:
+                samples.append(int(s))
+                continue
+            except ValueError:
+                pass
+        elif col_type == "float":
+            try:
+                samples.append(float(s))
+                continue
+            except ValueError:
+                pass
+        samples.append(s)
+
+    # Value index / top values
+    if cardinality <= VALUE_INDEX_CARDINALITY_LIMIT:
+        value_index: Optional[dict] = {}
+        for val_str, cnt in acc.value_counts.items():
+            if col_type == "integer":
+                try:
+                    key: Any = int(val_str)
+                except ValueError:
+                    key = val_str
+            elif col_type == "float":
+                try:
+                    key = float(val_str)
+                except ValueError:
+                    key = val_str
+            else:
+                key = val_str
+            value_index[str(key)] = cnt
+        top_values = None
+    else:
+        value_index = None
+        sorted_vals = sorted(acc.value_counts.items(), key=lambda x: x[1], reverse=True)
+        top_values = [{"value": v, "count": c} for v, c in sorted_vals[:TOP_VALUES_LIMIT]]
+
+    return ColumnProfile(
+        name=acc.name,
+        position=acc.position,
+        type=col_type,
+        count=acc.count,
+        null_count=acc.null_count,
+        null_pct=null_pct,
+        cardinality=cardinality,
+        cardinality_is_exact=cardinality_is_exact,
+        is_unique=is_unique,
+        is_primary_key_candidate=is_pk_candidate,
+        min=min_val,
+        max=max_val,
+        mean=mean_val,
+        median=median_val,
+        sample_values=samples,
+        value_index=value_index,
+        top_values=top_values,
+        datetime_min=acc.dt_min if col_type == "datetime" else None,
+        datetime_max=acc.dt_max if col_type == "datetime" else None,
+        datetime_format=acc.dt_format if col_type == "datetime" else None,
+    )
+
+
+def infer_types_from_sample(
+    accs: list,  # list[_ColAcc]
+    sample_rows: list,
+) -> list:
+    """Run a subset of rows through the accumulators to detect preliminary types.
+
+    Used to determine column types before creating the SQLite schema.
+    Returns the modified accs list (same objects, updated in-place).
+    """
+    n_cols = len(accs)
+    for row in sample_rows:
+        for i, acc in enumerate(accs):
+            raw = row[i] if i < len(row) else ""
+            update_acc(acc, raw)
+    return accs

jdatamunch_mcp-0.1.0/src/jdatamunch_mcp/profiler/histogram.py

@@ -0,0 +1,36 @@
+"""Numeric histogram computation from a column profile's reservoir."""
+
+from typing import Optional
+
+
+def compute_histogram(
+    reservoir: list,
+    bins: int = 10,
+    col_min: Optional[float] = None,
+    col_max: Optional[float] = None,
+) -> Optional[dict]:
+    """Compute a histogram from a reservoir of numeric values.
+
+    Returns a dict with 'bins' (count per bin) and 'edges' (bin boundaries),
+    or None if the reservoir is empty.
+    """
+    if not reservoir:
+        return None
+
+    lo = col_min if col_min is not None else min(reservoir)
+    hi = col_max if col_max is not None else max(reservoir)
+
+    if lo == hi:
+        return {"bins": [len(reservoir)], "edges": [lo, hi], "bin_count": 1}
+
+    bin_width = (hi - lo) / bins
+    counts = [0] * bins
+    edges = [round(lo + i * bin_width, 6) for i in range(bins + 1)]
+
+    for val in reservoir:
+        idx = int((val - lo) / bin_width)
+        if idx >= bins:
+            idx = bins - 1
+        counts[idx] += 1
+
+    return {"bins": counts, "edges": edges, "bin_count": bins}

jdatamunch_mcp-0.1.0/src/jdatamunch_mcp/profiler/value_indexer.py

@@ -0,0 +1,32 @@
+"""Value index builder for low-cardinality columns.
+
+Low-cardinality columns (cardinality <= VALUE_INDEX_CARDINALITY_LIMIT) get a
+full inverted index: {value_str: count}. This index is stored in index.json
+and used by search_data for fast value matching without touching SQLite.
+"""
+
+# Re-export the constant so callers can import from this module
+from .column_profiler import VALUE_INDEX_CARDINALITY_LIMIT
+
+
+def build_value_search_index(profiles: list) -> dict:
+    """Build a search-optimised view of all value indexes.
+
+    Returns:
+        {column_name: {"low_cardinality": bool, "values": [str, ...], "top_values": [...]}}
+
+    Used by search_data to quickly find columns whose values match a query.
+    """
+    result = {}
+    for p in profiles:
+        if p.value_index is not None:
+            result[p.name] = {
+                "low_cardinality": True,
+                "values": list(p.value_index.keys()),
+            }
+        elif p.top_values is not None:
+            result[p.name] = {
+                "low_cardinality": False,
+                "values": [tv["value"] for tv in p.top_values],
+            }
+    return result