mammoth-cli 2.2.63__tar.gz → 2.2.65__tar.gz

{mammoth_cli-2.2.63 → mammoth_cli-2.2.65}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mammoth-cli
-Version: 2.2.63
+Version: 2.2.65
 Summary: Command-line interface for the Mammoth Analytics platform
 Project-URL: homepage, https://mammoth.io
 Project-URL: repository, https://github.com/EdgeMetric/mammothsdk

{mammoth_cli-2.2.63 → mammoth_cli-2.2.65}/mammoth_cli/__init__.py

@@ -6,7 +6,7 @@ not implement a second HTTP client and does not call private SDK members.
 
 from __future__ import annotations
 
-__version__ = "2.2.63"
+__version__ = "2.2.65"
 
 # The versioned machine output envelope contract.
 SCHEMA_VERSION = 1

{mammoth_cli-2.2.63 → mammoth_cli-2.2.65}/mammoth_cli/commands/dataset.py

@@ -15,10 +15,12 @@ from typing import Any, Protocol
 
 from mammoth_cli.errors.envelope import (
     CODE_INVALID_ARGUMENT,
+    CODE_JOB_FAILED,
     CODE_MISSING_ARGUMENT,
     CODE_MISSING_FIELD,
     CODE_SDK_SYMBOL_UNRESOLVED,
     CODE_UNSUPPORTED_CONTRACT,
+    EXIT_API,
     EXIT_USAGE,
     CliError,
 )
@@ -169,52 +171,62 @@ def _tokens_in_order(needle: str, name: str) -> bool:
     return True
 
 
-def _find_in_projects(
-    service: Any,
-    needle: str,
-    projects: list[dict[str, Any]],
-    skipped: list[dict[str, Any]] | None = None,
-) -> list[dict[str, Any]]:
-    """Datasets whose name has every token of ``needle`` in order, in each of ``projects``.
+def _search_term(needle: str) -> str:
+    """The longest word of ``needle``: the server searches a substring, the words are
+    matched in order here."""
+    return max(needle.split(), key=len, default="")
+
+
+def _hit_record(hit: dict[str, Any]) -> dict[str, Any]:
+    """A resource-search hit in the shape of a dataset record, for ``name_hit``/``source_of``."""
+    info = hit.get("additional_info")
+    original = info.get("ORIGINAL_FILE_PROPS") if isinstance(info, dict) else None
+    file_name = original.get("FILE_NAME") if isinstance(original, dict) else None
+    return {
+        "id": hit.get("object_id"),
+        "name": hit.get("name"),
+        "status": hit.get("status"),
+        "created_at": hit.get("created_at"),
+        "updated_at": hit.get("last_updated_at"),
+        "stats": {"row_count": hit.get("row_count"), "column_count": hit.get("column_count")},
+        "additional_info": info,
+        "sources": [
+            {
+                "type": hit.get("source_type"),
+                "details": {"file_name": file_name, "connector_key": hit.get("integration_key")},
+            }
+        ],
+    }
+
 
-    A project that cannot be read does not end the search: with ``skipped`` it is
-    recorded there (id, name, why) and the rest are still searched.
+def _find_by_name(
+    service: Any, needle: str, projects: list[dict[str, Any]]
+) -> tuple[list[dict[str, Any]], bool]:
+    """Datasets in ``projects`` whose name has every token of ``needle`` in order, and whether cut.
+
+    One workspace search returns each hit with its size, times and source, so no
+    project's dataset list is read: that route costs the server a few queries per
+    dataset, and a project of a few hundred datasets took 20+ s to list.
     """
-    matches: list[dict[str, Any]] = []
-    for project in projects:
-        project_id = project.get("id")
-        if project_id is None:
+    names = {p.get("id"): p.get("name") for p in projects}
+    hits, cut = search_hits(service, "datasource", _search_term(needle), fields="standard")
+    matches = []
+    for hit in hits:
+        name = hit.get("name")
+        if hit.get("project_id") not in names or not isinstance(name, str):
             continue
-        try:
-            response = service.call(
-                "mammoth.api.datasets.DatasetsAPI.list_all",
-                project_id=project_id,
-                fields=DATASET_ROW_FIELDS,
-            )
-        except CliError as error:
-            if skipped is None:
-                raise
-            skipped.append(
-                {
-                    "project_id": project_id,
-                    "project_name": project.get("name"),
-                    "error": error.message,
-                }
-            )
+        if not _tokens_in_order(needle, name):
             continue
-        datasets = response.get("datasets", []) if isinstance(response, dict) else []
-        for dataset in datasets:
-            name = dataset.get("name") if isinstance(dataset, dict) else None
-            if isinstance(name, str) and _tokens_in_order(needle, name):
-                matches.append(
-                    {
-                        "project_id": project_id,
-                        "project_name": project.get("name"),
-                        **name_hit(dataset),
-                        "source": source_of(dataset),
-                    }
-                )
-    return matches
+        record = _hit_record(hit)
+        matches.append(
+            {
+                "project_id": hit["project_id"],
+                "project_name": names[hit["project_id"]],
+                **name_hit(record),
+                "source": source_of(record),
+            }
+        )
+    return matches, cut
 
 
 #: Pages of resource search read for one name (100 hits each); past it the result says so.
@@ -269,15 +281,6 @@ def search_cut_note(kind: str) -> str:
     )
 
 
-def _projects_holding(
-    service: Any, needle: str, visible: list[dict[str, Any]]
-) -> tuple[list[dict[str, Any]], bool]:
-    """The visible projects that hold a dataset whose name contains ``needle``."""
-    rows, cut = search_hits(service, "datasource", needle)
-    holding = {r.get("project_id") for r in rows}
-    return [p for p in visible if p.get("id") in holding], cut
-
-
 def _other_projects(visible: list[dict[str, Any]], project_id: int) -> list[dict[str, Any]]:
     """Every visible project except ``project_id``."""
     return [p for p in visible if p.get("id") != project_id]
@@ -402,22 +405,20 @@ def _find_scoped(
     needle: str,
     visible: list[dict[str, Any]],
     project_id: int,
-    skipped: list[dict[str, Any]],
 ) -> tuple[list[dict[str, Any]], list[dict[str, Any]], bool]:
     """``(matches, projects searched, cut)``: the project's own matches, then every other one's."""
-    home = named_project(service, project_id, visible)
-    own = _find_in_projects(service, needle, [home], skipped)
-    holding, cut = _projects_holding(service, needle, visible)
-    outside = _find_in_projects(service, needle, _other_projects(holding, project_id), skipped)
-    searched = [home] + _other_projects(visible, project_id)
-    return _tag_scope(own + outside, project_id), searched, cut
+    searched = [named_project(service, project_id, visible), *_other_projects(visible, project_id)]
+    found, cut = _find_by_name(service, needle, searched)
+    matches = sorted(_tag_scope(found, project_id), key=lambda m: not m["in_project"])
+    return matches, searched, cut
 
 
 def dataset_find(invocation: Invocation) -> HandlerResult:
     """Search dataset names for a substring across every visible project.
 
     Read-only local composite: lists the projects the credential can see, then
-    lists datasets in each and keeps a case-insensitive substring match. With
+    reads one workspace search for the name and keeps the datasets whose name has
+    every word of the substring, in order, ignoring case. With
     ``--project`` the project's own matches come first and the matches in every
     other visible project are returned too, each row marked ``in_project`` and
     carrying its ``project_id`` and ``project_name``: the in-product agent runs
@@ -429,23 +430,19 @@ def dataset_find(invocation: Invocation) -> HandlerResult:
     """
     name_substring = _require_string_positional(invocation, "name substring")
     needle = name_substring.lower()
-    skipped: list[dict[str, Any]] = []
     dropped = 0
     with open_service(invocation) as (service, auth):
         visible = _visible_projects(service)
         cut = False
         if invocation.project is not None:
-            matches, projects, cut = _find_scoped(
-                service, needle, visible, invocation.project, skipped
-            )
+            matches, projects, cut = _find_scoped(service, needle, visible, invocation.project)
             if not all_projects_requested(invocation):
                 matches, dropped = narrow_to_project(
                     matches, lambda m: str(m.get("name", "")).lower() == needle
                 )
         else:
             projects = visible
-            holding, cut = _projects_holding(service, needle, visible)
-            matches = _find_in_projects(service, needle, holding, skipped)
+            matches, cut = _find_by_name(service, needle, visible)
         meta = {
             "profile": invocation.profile,
             "workspace_id": auth.workspace_id,
@@ -453,10 +450,8 @@ def dataset_find(invocation: Invocation) -> HandlerResult:
         }
     result: dict[str, Any] = {
         "matches": matches,
-        "projects_searched": len(projects) - len(skipped),
+        "projects_searched": len(projects),
     }
-    if skipped:
-        result["projects_skipped"] = skipped
     notes = [ambiguity_note(len(matches), name_substring)]
     if dropped:
         result["elsewhere"] = dropped
@@ -561,10 +556,8 @@ def _dataset_name_search(
         records = data.get("datasets", []) if isinstance(data, dict) else []
         page = search_page(records, name, int(document.get("offset", 0)), document.get("limit"))
         if page["matched"] == 0:
-            holding, cut = _projects_holding(service, name.lower(), _visible_projects(service))
-            elsewhere = _find_in_projects(
-                service, name.lower(), _other_projects(holding, project_id)
-            )
+            found, cut = _find_by_name(service, name.lower(), _visible_projects(service))
+            elsewhere = [m for m in found if m["project_id"] != project_id]
             if elsewhere:
                 page["in_other_projects"] = elsewhere
                 page["note"] = (
@@ -877,6 +870,7 @@ def dataset_create(invocation: Invocation) -> HandlerResult:
     # A clone can land in another project (``dataset_spec.target_project_id``): the
     # name check and the result's scope are that project's, not the active one's.
     landing_project_id = _clone_target_project(ds_creation_type, dataset_spec) or project_id
+    _default_data_pull_file(ds_creation_type, dataset_spec)
     kwargs: dict[str, Any] = {
         "dataset_spec": dataset_spec,
         "ds_creation_type": ds_creation_type,
@@ -927,29 +921,92 @@ def _require_clone_views(ds_creation_type: object, dataset_spec: object) -> None
         )
 
 
+#: The server's ``data_pull_file`` values for a cloud file import; the first is the app default.
+_DATA_PULL_FILE_VALUES = ("Pull same file", "Pull next file based on name pattern")
+
+
+def _default_data_pull_file(ds_creation_type: object, dataset_spec: object) -> None:
+    """Default a cloud file import's ``data_pull_file`` to "Pull same file", as the web app does.
+
+    Raises:
+        CliError: ``invalid_argument`` listing the allowed values when it is not one of them.
+    """
+    if ds_creation_type != "cloud" or not isinstance(dataset_spec, dict):
+        return
+    properties = dataset_spec.get("query_properties")
+    if not isinstance(properties, dict) or not properties.get("file_path"):
+        return
+    pull = properties.setdefault("data_pull_file", _DATA_PULL_FILE_VALUES[0])
+    if pull not in _DATA_PULL_FILE_VALUES:
+        raise CliError(
+            code=CODE_INVALID_ARGUMENT,
+            message="query_properties.data_pull_file must be one of "
+            f"{list(_DATA_PULL_FILE_VALUES)}, got {pull!r}.",
+            exit_status=EXIT_USAGE,
+        )
+
+
 def _created_dataset(handle: Any, settled: Any) -> dict[str, Any]:
     """Shape a create job (handle + settled result) into a labeled result.
 
-    ``wait_if_job`` returns the completed job's inner response, where the new
-    dataset id lives under ``ds_id``. Falls back to the raw job handle when the
-    server returned something unrecognized, so no information is lost.
+    What each creation type settles with: weburl, clone and a single cloud file or
+    database import carry the new dataset id (``ds_id``, ``dataset_id`` or
+    ``datasource_id``); a cloud import deferred to a later start carries
+    ``datasource_config_id`` and reports ``scheduled``; a bulk cloud file import
+    carries ``files_found``/``created``/``failed`` and is returned as the server
+    sent it; a sketch is not a job and answers with the dataset itself. A job that
+    settled with none of these made nothing, so it raises with the server's reason
+    rather than report success.
+
+    Raises:
+        CliError: ``job_failed`` when the settled job made no dataset.
     """
-    job_id = handle.get("job_id") if isinstance(handle, dict) else None
+    if not isinstance(handle, dict) or "job_id" not in handle:
+        return settled if isinstance(settled, dict) else {"job_id": None}
+    job_id = handle["job_id"]
     ds_id = None
+    config_id = None
     if isinstance(settled, dict):
-        ds_id = settled.get("ds_id") or settled.get("dataset_id")
-        nested = settled.get("response")
-        if ds_id is None and isinstance(nested, dict):
-            ds_id = nested.get("ds_id") or nested.get("dataset_id")
+        for source in (settled, settled.get("response")):
+            if not isinstance(source, dict):
+                continue
+            ds_id = ds_id or _first_id(source, _DATASET_ID_KEYS)
+            config_id = config_id or source.get("datasource_config_id")
         job_id = settled.get("job_id", job_id)
+    if ds_id is None and config_id is not None:
+        return {"status": "scheduled", "datasource_config_id": config_id, "job_id": job_id}
+    if ds_id is None and isinstance(settled, dict) and "files_found" in settled:
+        return settled
     if ds_id is None:
-        # No recognizable dataset id: surface the settled payload untouched
-        # rather than claim a readiness we cannot confirm.
-        return settled if isinstance(settled, dict) else {"job_id": job_id}
-    result: dict[str, Any] = {"status": "ready", "dataset_id": ds_id}
-    if job_id is not None:
-        result["job_id"] = job_id
-    return result
+        reason = _job_reason(settled)
+        raise CliError(
+            code=CODE_JOB_FAILED,
+            message="The dataset was not created"
+            + (f": {reason}" if reason else f" (job {job_id} finished without a dataset id)."),
+            exit_status=EXIT_API,
+        )
+    return {"status": "ready", "dataset_id": ds_id, "job_id": job_id}
+
+
+#: Keys a settled create job may carry its dataset id under.
+_DATASET_ID_KEYS = ("ds_id", "dataset_id", "datasource_id")
+
+
+def _first_id(source: dict[str, Any], keys: tuple[str, ...]) -> Any:
+    """The first non-empty value of ``keys`` in ``source``, or None."""
+    return next((source[key] for key in keys if source.get(key)), None)
+
+
+def _job_reason(settled: Any) -> str | None:
+    """The failure reason a settled job carries (``failure_reason``, ``message``, ``error``)."""
+    if not isinstance(settled, dict):
+        return None
+    nested = settled.get("response")
+    for source in (settled, nested if isinstance(nested, dict) else {}):
+        for key in ("failure_reason", "message", "error"):
+            if source.get(key):
+                return str(source[key])
+    return None
 
 
 #: Pages (of 100) read to predict a final name; past it the answer is "a suffix may be added".

{mammoth_cli-2.2.63 → mammoth_cli-2.2.65}/mammoth_cli/commands/view.py

@@ -357,7 +357,9 @@ def _read_meta(
     """Envelope metadata for a data read, naming the dataset and view it read.
 
     A number in an answer needs its source; without the names an agent that
-    silently used the wrong one of two look-alike datasets never says so.
+    silently used the wrong one of two look-alike datasets never says so. Both
+    carry their row count: a view's steps can filter rows, so a total read from
+    it is not the dataset's total, and the two counts are how an agent sees that.
     """
     meta = _meta(invocation, workspace_id, project_id)
     view = service.call(
@@ -365,17 +367,26 @@ def _read_meta(
         dataset_id=dataset_id,
         dataview_id=view_id,
         project_id=project_id,
-        fields="__min",
+        fields="id,name,row_count",
     )
     # The read just proved this dataset holds the view; keep the pair so the
     # next command on it (a transform) needs no ``dataset_id`` of its own.
     parents.remember(_profile_name(invocation), workspace_id, {view_id: dataset_id})
     dataset = service.call(
-        _DATASET_GET_SYMBOL, dataset_id=dataset_id, project_id=project_id, fields="id,name"
+        _DATASET_GET_SYMBOL,
+        dataset_id=dataset_id,
+        project_id=project_id,
+        fields="id,name,stats",
     )
     dataset = dataset.get("dataset", dataset) if isinstance(dataset, dict) else {}
-    meta["dataset"] = {"id": dataset_id, "name": dataset.get("name")}
-    meta["view"] = {"id": view_id, "name": view.get("name") if isinstance(view, dict) else None}
+    view = view if isinstance(view, dict) else {}
+    stats = dataset.get("stats")
+    meta["dataset"] = {
+        "id": dataset_id,
+        "name": dataset.get("name"),
+        "row_count": stats.get("row_count") if isinstance(stats, dict) else None,
+    }
+    meta["view"] = {"id": view_id, "name": view.get("name"), "row_count": view.get("row_count")}
     return meta
 
 

{mammoth_cli-2.2.63 → mammoth_cli-2.2.65}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mammoth-cli"
-version = "2.2.63"
+version = "2.2.65"
 description = "Command-line interface for the Mammoth Analytics platform"
 authors = [{name = "Mammoth Analytics", email = "support@mammoth.io"}]
 readme = "README.md"

{mammoth_cli-2.2.63 → mammoth_cli-2.2.65}/spec/manifests/commands/dataset.yaml

@@ -247,7 +247,7 @@ commands:
   - CT-DATASET-CREATE-ERROR
   draft_test: null
   undo_test: null
-  known_restrictions: 'ds_creation_type is freeform and dataset_spec is arbitrary, so variants beyond the documented path are not qualified. Retained live evidence supports ds_creation_type=weburl with dataset_spec.url; the CLI command waits for the raw SDK job and returned ready for five OWID URL imports. Re-read returned dataset/view IDs and delete only IDs created by this task. A weburl dataset is a one-time copy that keeps no source details to refresh from, so automation''s run_data_retrieval task cannot refresh it (only a cloud-connected dataset can be refreshed); say so rather than scheduling a refresh for it. The result''s name is the dataset''s real name; when the requested name was taken the server adds a suffix, and requested_name and note say so. Use name, not the name you asked for. To copy a dataset as a new independent one: {"ds_creation_type": "clone", "dataset_spec": {"ds_name": NEW_NAME, "source_ds_id": ID, "clone_dataview_ids": [DEFAULT_VIEW_ID], "exclude_data": false}}, DEFAULT_VIEW_ID being the id in dataset''s `default_view` from view list (other views only when asked); an empty clone_dataview_ids is refused here because it makes a blank view. A clone waits and returns the new dataset_id; never use view export dataset for a plain copy, which leaves a live export on the source. A weburl import names the dataset after the downloaded file: follow it with dataset rename to the name the user picked. To import a file from an SFTP server (or another file connector) as a new dataset: connector connection create sftp, then {"ds_creation_type": "cloud", "dataset_spec": {"connector_key": "sftp", "connection_key": KEY, "query_properties": {"ds_name": NAME, "file_path": PATH_ON_SERVER}}}; the recipe is recipes/file-import.md.'
+  known_restrictions: 'ds_creation_type is freeform and dataset_spec is arbitrary, so variants beyond the documented path are not qualified. Retained live evidence supports ds_creation_type=weburl with dataset_spec.url; the CLI command waits for the raw SDK job and returned ready for five OWID URL imports. Re-read returned dataset/view IDs and delete only IDs created by this task. A weburl dataset is a one-time copy that keeps no source details to refresh from, so automation''s run_data_retrieval task cannot refresh it (only a cloud-connected dataset can be refreshed); say so rather than scheduling a refresh for it. The result''s name is the dataset''s real name; when the requested name was taken the server adds a suffix, and requested_name and note say so. Use name, not the name you asked for. To copy a dataset as a new independent one: {"ds_creation_type": "clone", "dataset_spec": {"ds_name": NEW_NAME, "source_ds_id": ID, "clone_dataview_ids": [DEFAULT_VIEW_ID], "exclude_data": false}}, DEFAULT_VIEW_ID being the id in dataset''s `default_view` from view list (other views only when asked); an empty clone_dataview_ids is refused here because it makes a blank view. A clone waits and returns the new dataset_id; never use view export dataset for a plain copy, which leaves a live export on the source. A weburl import names the dataset after the downloaded file: follow it with dataset rename to the name the user picked. To import a file from an SFTP server (or another file connector) as a new dataset: connector connection create sftp, then {"ds_creation_type": "cloud", "dataset_spec": {"connector_key": "sftp", "connection_key": KEY, "query_properties": {"ds_name": NAME, "file_path": PATH_ON_SERVER, "data_pull_file": "Pull same file"}}}. data_pull_file is a string and may be left out, which means "Pull same file" (what the web app does); its only other value is "Pull next file based on name pattern", which also needs folder_path and name_pattern. A pull that finds no file at the path fails the create with the server''s reason; the recipe is recipes/file-import.md.'
   reviewed_by: primary
 - command_id: dataset.create-from-pdf
   command_path: dataset create-from-pdf