mammoth-cli 2.0.78__tar.gz → 2.0.80__tar.gz

{mammoth_cli-2.0.78 → mammoth_cli-2.0.80}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mammoth-cli
-Version: 2.0.78
+Version: 2.0.80
 Summary: Command-line interface for the Mammoth Analytics platform
 Project-URL: homepage, https://mammoth.io
 Project-URL: repository, https://github.com/EdgeMetric/mammothsdk

{mammoth_cli-2.0.78 → mammoth_cli-2.0.80}/mammoth_cli/__init__.py

@@ -6,7 +6,7 @@ not implement a second HTTP client and does not call private SDK members.
 
 from __future__ import annotations
 
-__version__ = "2.0.78"
+__version__ = "2.0.80"
 
 # The versioned machine output envelope contract.
 SCHEMA_VERSION = 1

{mammoth_cli-2.0.78 → mammoth_cli-2.0.80}/mammoth_cli/bundled_skill/mammoth-cli/SKILL.md

@@ -42,12 +42,16 @@ never read one from a file/env var, never run `auth login`
 yourself. If any envelope carries `meta.update_available`, run its `command`
 before the next step.
 
+In a shell: piped stdout is compact JSON and prompts are off; in a session
+that is not piped, `export MAMMOTH_OUTPUT=json MAMMOTH_NO_INPUT=1` once.
+`project ensure` saves the active project (`--project`, `MAMMOTH_PROFILE`
+and `MAMMOTH_PROJECT` override it). `mammoth skill show --input '{"file":
+"references/recipes/transforms.md"}'` prints any file of this skill;
+`mammoth skill agents-md install` writes a steering block into a
+repository's AGENTS.md so later sessions start here.
+
 ## Defaults you do not repeat
 
-- Piped stdout is compact JSON and prompts are off. Only in a session that
-  is not piped, `export MAMMOTH_OUTPUT=json MAMMOTH_NO_INPUT=1` once.
-- `project ensure` saves the active project; `--project` only overrides it.
-  `MAMMOTH_PROFILE` / `MAMMOTH_PROJECT` also work as session defaults.
 - The CLI remembers which dataset owns each view from any read (`view list
   DATASET_ID`, `view get VIEW_ID`, an upload). After that, no `dataset_id`
   on transforms, exports or deletes. If a command asks for the parent
@@ -96,10 +100,10 @@ with the command tried.
 Users often upload related files without saying how they relate ("make a
 dashboard from t_a and t_b"). Work it out from the data before building:
 
-1. Upload every file. Each result carries a `view` preview of what Mammoth
-   made of it (`view_id`, column types, sample rows, `column_warnings`,
-   `before_dashboard`). Do not open the local files (`cat`, `head`) — what
-   Mammoth made of them is what counts. Run `view data get VIEW_ID` per view;
+1. Upload every file, or find the ones already uploaded. An upload result
+   carries a `view` preview of what Mammoth made of it (`view_id`, column
+   types, sample rows, `column_warnings`, `before_dashboard`). What Mammoth
+   made of the files is what counts, not the raw files. Run `view data get VIEW_ID` per view;
    warnings list numbers/dates stored as text, and blanks, each with its fix.
 2. Find the keys. A column in one view whose values appear in a column of
    the other (`customer_id` and `id`, `order_ref` and `order_no`) is a
@@ -114,8 +118,8 @@ dashboard from t_a and t_b"). Work it out from the data before building:
    is a miss.
 5. `view transform join` the lookup view into the main view (`LEFT`). The
    result's `join_check` gives `match_rate`, `unmatched_rows` and
-   `unmatched_keys`; put them in your report, and stop to compare the keys
-   if more than a few rows found no match.
+   `unmatched_keys`; put them in your report. When the result's
+   `verify.needs_user` is set, stop and tell the user before building on it.
 6. If the data has money, add it before you make the dashboard: `revenue`
    (`math`, `qty * price`, `new_column`), the `fix` in `before_dashboard`.
    A dashboard sees only the columns the view had when it was made.
@@ -134,9 +138,6 @@ your report. If no column links the files, ask before you combine them.
 
 - Ids are positionals; request fields are one `--input '{...}'` document;
   no per-field flags (`project create NAME`, not `--name`).
-- `mammoth skill show --input '{"file": "references/recipes/transforms.md"}'`
-  prints any file of this skill; `mammoth skill agents-md install` writes a
-  steering block into a repository's AGENTS.md so later sessions start here.
 - `COMMAND --help` lists input fields; `schema find WORDS` inlines
   `accepted_fields`/`agent_example` for top matches, else `mammoth schema
   get COMMAND_ID` (`--input '{"full": true}'` for the schema); `schema
@@ -161,11 +162,10 @@ your report. If no column links the files, ask before you combine them.
 
 ## Verify before you report
 
-- A success envelope proves the call, not the outcome. After a row-scoped
-  change (set-values, filter, replace, join, fill) run `view data get
-  VIEW_ID` (50 rows by default, `limit` to change) and check rows the
-  condition should and should not have touched. Sample the target column
-  before the change.
+- A success envelope proves the call; the write's `state` (the data,
+  object or job status read back after it settled) shows the outcome.
+  Check it against your plan: rows the change should and should not have
+  touched. `unreadable` or a partial sample: read it with `view data get`.
 - A uniform result (every amount 0, every region "Unknown", most join rows
   unmatched) means the previous step went wrong; stop and re-inspect it.
 - `column_warnings` on a data read and `join_check` on a join are findings,

{mammoth_cli-2.0.78 → mammoth_cli-2.0.80}/mammoth_cli/services/data_quality.py

@@ -11,6 +11,7 @@ from __future__ import annotations
 
 import json
 import re
+from collections import Counter
 from collections.abc import Iterable, Mapping
 from datetime import datetime
 from typing import Any
@@ -20,8 +21,14 @@ _MOSTLY = 0.8
 #: Fewer non-blank values than this are too few to call a pattern.
 _MIN_VALUES = 3
 _MAX_EXAMPLES = 3
+#: Most groups of variant spellings to name in one warning.
+_MAX_VARIANT_GROUPS = 5
 
 _NUMBER = re.compile(r"^[-+]?[$€£]?\s*\d[\d,]*(\.\d+)?\s*%?$|^[-+]?[$€£]?\s*\.\d+\s*%?$")
+#: Punctuation is normalised to a space, not dropped, so "Pepsi,Cola" and
+#: "Pepsi Cola" collapse the same way as "Pepsi Cola" itself.
+_PUNCTUATION = re.compile(r"[^\w\s]", re.UNICODE)
+_WHITESPACE = re.compile(r"\s+")
 _DATE_FORMATS = (
     "%Y-%m-%d",
     "%Y/%m/%d",
@@ -61,6 +68,82 @@ def _convert_hint(view_id: int | None, column: str, to: str) -> str:
     return f"mammoth view transform convert-type {target} --input '{spec}'"
 
 
+def _bulk_replace_hint(view_id: int | None, column: str, mapping: list[dict[str, Any]]) -> str:
+    target = str(view_id) if view_id is not None else "VIEW_ID"
+    spec = json.dumps({"columns": [column], "mapping": mapping})
+    return f"mammoth view transform bulk-replace {target} --input '{spec}'"
+
+
+def _spelling_key(text: str) -> str:
+    """Case-, whitespace- and punctuation-insensitive grouping key.
+
+    Punctuation is replaced with a space (not dropped) before whitespace is
+    collapsed, so inner punctuation ("Pepsi, Co.") normalises the same way
+    as a plain space ("Pepsi Co"). This is normalisation, not fuzzy
+    matching: different words ("pepsi cola") get a different key than
+    ("pepsi") and are never grouped.
+    """
+    despaced = _WHITESPACE.sub(" ", _PUNCTUATION.sub(" ", text))
+    return despaced.strip().lower()
+
+
+def _variant_spelling_groups(texts: list[str]) -> list[tuple[str, list[str]]]:
+    """Group distinct values that share a spelling key, keys sorted for stable output.
+
+    Only keys with more than one distinct raw value are a "group" worth
+    reporting; a single spelling of a value is not a finding.
+    """
+    by_key: dict[str, list[str]] = {}
+    for text in dict.fromkeys(texts):  # distinct values, first-seen order
+        key = _spelling_key(text)
+        if not key:
+            continue
+        by_key.setdefault(key, []).append(text)
+    groups = [(key, values) for key, values in by_key.items() if len(values) > 1]
+    return sorted(groups, key=lambda item: item[0])
+
+
+def _most_used_spelling(values: list[str], counts: Counter[str]) -> str:
+    """The group's most frequent real spelling (first seen on a tie), trimmed.
+
+    Never a synthesised form such as ``str.title()``, which would rewrite
+    "McDonald's" as "Mcdonald'S".
+    """
+    return max(values, key=lambda value: counts[value]).strip()
+
+
+def _variant_spellings_warning(
+    column: str,
+    texts: list[str],
+    view_id: int | None,
+    checked: int,
+) -> dict[str, Any] | None:
+    groups = _variant_spelling_groups(texts)
+    if not groups:
+        return None
+    shown = groups[:_MAX_VARIANT_GROUPS]
+    counts = Counter(texts)
+    canonical = {key: _most_used_spelling(values, counts) for key, values in shown}
+    mapping = [{"search": sorted(values), "replace": canonical[key]} for key, values in shown]
+    examples = "; ".join(
+        "{" + ", ".join(repr(v) for v in sorted(values)) + "}" + f" -> {canonical[key]!r}"
+        for key, values in shown
+    )
+    detail = (
+        f"{len(groups)} group(s) of values in this TEXT column differ only in case, "
+        f"whitespace or punctuation and will be counted separately unless unified: {examples}."
+    )
+    if len(groups) > _MAX_VARIANT_GROUPS:
+        detail += f" ({len(groups) - _MAX_VARIANT_GROUPS} more not shown)."
+    return {
+        "column": column,
+        "issue": "variant_spellings",
+        "detail": detail,
+        "fix": _bulk_replace_hint(view_id, column, mapping),
+        "rows_checked": checked,
+    }
+
+
 def _duplicate_rows_warning(
     rows: list[Mapping[str, Any]],
     view_id: int | None,
@@ -111,9 +194,10 @@ def column_warnings(
     Returns:
         One record per finding: ``column``, ``issue``
         (``numbers_stored_as_text``, ``dates_stored_as_text``,
-        ``blank_values`` or the table-level ``duplicate_rows``), ``detail``
-        and, where one command fixes it, ``fix``. Counts are over the rows
-        given (``rows_checked`` on each record).
+        ``variant_spellings``, ``blank_values`` or the table-level
+        ``duplicate_rows``), ``detail`` and, where one command fixes it,
+        ``fix``. Counts are over the rows given (``rows_checked`` on each
+        record).
     """
     materialised = [row for row in rows if isinstance(row, Mapping)]
     if not materialised:
@@ -166,6 +250,9 @@ def column_warnings(
                     }
                 )
                 break
+            variant_warning = _variant_spellings_warning(column, texts, view_id, checked)
+            if variant_warning is not None:
+                warnings.append(variant_warning)
         if blanks:
             warnings.append(
                 {

{mammoth_cli-2.0.78 → mammoth_cli-2.0.80}/mammoth_cli/services/sdk_service.py

@@ -489,6 +489,7 @@ class SdkMammothService:
         local_columns = getattr(view, "columns", {}) or {}
         local_internal = {value for value in local_columns.values() if isinstance(value, str)}
         local_display = set(local_columns)
+        local_types = getattr(view, "column_types", {}) or {}
 
         def check(value: Any, *, scope: str, columns: dict[str, str], display: set[str]) -> None:
             internal = {item for item in columns.values() if isinstance(item, str)}
@@ -583,6 +584,8 @@ class SdkMammothService:
                     columns=local_columns,
                     display=local_display,
                 )
+                if not spec.get("value_is_column") and "value" in spec:
+                    SdkMammothService._coerce_numeric_condition_value(spec, local_types)
             if spec.get("value_is_column") and "value" in spec:
                 check(
                     spec["value"],
@@ -639,6 +642,54 @@ class SdkMammothService:
                 display=foreign_display,
             )
 
+    @staticmethod
+    def _coerce_numeric_condition_value(spec: dict[str, Any], column_types: dict[str, str]) -> None:
+        """Convert a numeric-string condition value against a NUMERIC column.
+
+        The backend accepts a filter/set-values/pivot condition whose value is
+        a string even when the column is NUMERIC, then puts the whole pipeline
+        into ref_error (7003 "type mismatch") on the next read. The CLI already
+        knows the column's type, so it converts what it can and fails locally,
+        before any request, for what it can't. DATE columns are left as is.
+        """
+        column = spec.get("column")
+        if not isinstance(column, str) or column_types.get(column) != "NUMERIC":
+            return
+
+        def convert(item: Any) -> Any:
+            if not isinstance(item, str):
+                return item
+            try:
+                return int(item)
+            except ValueError:
+                pass
+            try:
+                number = float(item)
+            except ValueError:
+                raise SdkMammothService._condition_value_type_error(
+                    column, "NUMERIC", item
+                ) from None
+            return int(number) if number.is_integer() else number
+
+        value = spec["value"]
+        spec["value"] = (
+            [convert(item) for item in value] if isinstance(value, list) else convert(value)
+        )
+
+    @staticmethod
+    def _condition_value_type_error(column: str, column_type: str, value: str) -> CliError:
+        """Build a stable pre-mutation condition-value type error."""
+        return CliError(
+            code="invalid_condition_value",
+            message=(
+                f"Condition value {value!r} for column '{column}' ({column_type}) "
+                "is not a valid number."
+            ),
+            exit_status=EXIT_USAGE,
+            hint="Use a numeric value for a NUMERIC column.",
+            details={"column": column, "column_type": column_type, "value": value},
+        )
+
     @staticmethod
     def column_input_error(
         reference: str,

{mammoth_cli-2.0.78 → mammoth_cli-2.0.80}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mammoth-cli"
-version = "2.0.78"
+version = "2.0.80"
 description = "Command-line interface for the Mammoth Analytics platform"
 authors = [{name = "Mammoth Analytics", email = "support@mammoth.io"}]
 readme = "README.md"