memorysync-cli 1.2.1__tar.gz → 1.4.0__tar.gz

{memorysync_cli-1.2.1 → memorysync_cli-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: memorysync-cli
-Version: 1.2.1
+Version: 1.4.0
 Summary: MemorySync from your terminal. Zero dependencies.
 Project-URL: Documentation, https://docs.memorysync.io/cli
 Project-URL: Homepage, https://memorysync.io/cli

memorysync_cli-1.4.0/src/memorysync_cli/_version.py

@@ -0,0 +1 @@
+__version__ = "1.4.0"

{memorysync_cli-1.2.1 → memorysync_cli-1.4.0}/src/memorysync_cli/commands/memory.py

@@ -512,6 +512,8 @@ def import_memories(ctx: dict) -> dict:
     records: list[dict] = []
     errors: list[str] = []
 
+    ref_field = flags.get("ref_field") or None
+
     def _validate(number: int, entry: Any) -> None:
         """Record one candidate as either usable or an error, in line order."""
         if not isinstance(entry, dict):
@@ -528,6 +530,7 @@ def import_memories(ctx: dict) -> dict:
             "text": text.strip(),
             "source": entry.get("source") or "cli-import",
             "line": number,
+            "ref": _pick_ref(entry, ref_field),
         }
         if isinstance(entry.get("metadata"), dict):
             record["metadata"] = entry["metadata"]
@@ -564,12 +567,14 @@ def import_memories(ctx: dict) -> dict:
     if flags.get("dry_run"):
         preview = list(errors[:10])
         trailer = len(errors) - len(preview)
+        with_ref = sum(1 for record in records if record.get("ref"))
         return {
             "data": {
                 "file": source_path,
                 "valid": len(records),
                 "invalid": len(errors),
                 "errors": errors,
+                "identified": with_ref,
             },
             "text": lambda: "\n".join(
                 line
@@ -577,6 +582,10 @@ def import_memories(ctx: dict) -> dict:
                     style.yellow("Dry run. Nothing was sent."),
                     f"{style.dim('valid   ')} {len(records)}",
                     f"{style.dim('invalid ')} {len(errors)}",
+                    # Shown always, not only under --resume, because knowing the
+                    # file has usable ids is what tells someone --resume is
+                    # available to them.
+                    f"{style.dim('with id ')} {with_ref}",
                     *[style.dim(f"  {message}") for message in preview],
                     style.dim(f"  ... and {trailer} more") if trailer > 0 else "",
                     style.yellow(
@@ -599,6 +608,44 @@ def import_memories(ctx: dict) -> dict:
     if not records:
         raise usage_error(f"{source_path} contained no records.")
 
+    # --resume is only honest if every record has a stable identity. Falling back
+    # to the line number would break the moment the file is re-exported in a
+    # different order, and falling back to a hash of the text would make two
+    # genuinely identical records collide, so the second would be reported as
+    # already stored and silently dropped. Refusing names the problem instead.
+    if flags.get("resume"):
+        missing = [record for record in records if not record.get("ref")]
+        if missing:
+            shown = ", ".join(f"line {record['line']}" for record in missing[:5])
+            looked_for = f'"{ref_field}"' if ref_field else "id, _id, uuid"
+            extra = "" if ref_field else " and metadata.mem0_id"
+            raise usage_error(
+                f"--resume needs an id on every record, and {len(missing)} "
+                f"{'has' if len(missing) == 1 else 'have'} none.",
+                f"First: {shown}. Looked for {looked_for}{extra}. "
+                "Pass --ref-field <name> to use a different field, or run without "
+                "--resume.",
+            )
+
+    # --async hands the whole file to the server and returns a job id. The
+    # synchronous path below is capped at 50 records per request by the route, so a
+    # large file is thousands of round trips held open by one command; this is one
+    # upload and a poll.
+    if flags.get("async"):
+        job = api.create_import(
+            filename=Path(source_path).name,
+            content=raw.encode("utf-8"),
+            end_user_id=user,
+            resume=bool(flags.get("resume")),
+            continue_on_error=bool(flags.get("continue_on_error")),
+        )
+        waited = bool(flags.get("wait"))
+        finished = _poll_import(api, job["id"]) if waited else job
+        return {
+            "data": {"file": source_path, "user": user, **finished},
+            "text": lambda: _render_import_job(finished, waited=waited),
+        }
+
     batch_size = _resolve_batch_size(flags)
     batches = [records[i : i + batch_size] for i in range(0, len(records), batch_size)]
     totals = {
@@ -606,23 +653,31 @@ def import_memories(ctx: dict) -> dict:
         "imported": 0,
         "memory_ids": [],
         "skipped": 0,
+        # Rows the server had already stored under the same identifier. Counted
+        # apart from ``skipped`` because they mean something different to whoever
+        # is resuming: nothing was lost, this row is simply already in.
+        "already_imported": 0,
         "failed": 0,
         "unreported": 0,
         "failures": [],
         "batches": len(batches),
         "batch_size": batch_size,
+        "resume": bool(flags.get("resume")),
     }
 
+    resume = bool(flags.get("resume"))
     for batch in batches:
         before = totals["failed"]
-        items = [
-            {
+        items = []
+        for record in batch:
+            item = {
                 key: value
                 for key, value in record.items()
                 if key in {"text", "source", "metadata"}
             }
-            for record in batch
-        ]
+            if resume and record.get("ref"):
+                item["client_ref"] = record["ref"]
+            items.append(item)
         try:
             response = api.bulk_add_memories({"items": items}) or {}
         except Exception as error:  # noqa: BLE001 — reported, not raised
@@ -653,6 +708,11 @@ def import_memories(ctx: dict) -> dict:
             if status == "created":
                 totals["imported"] += 1
                 totals["memory_ids"].extend(item.get("memory_ids") or [])
+            elif status == "skipped" and item.get("reason") == "already_ingested":
+                totals["already_imported"] += 1
+                # The ids the first run produced, so a resumed import still ends
+                # up knowing every memory its file corresponds to.
+                totals["memory_ids"].extend(item.get("memory_ids") or [])
             elif status == "skipped":
                 totals["skipped"] += 1
             else:
@@ -668,6 +728,11 @@ def import_memories(ctx: dict) -> dict:
             line
             for line in [
                 f"{style.green('Imported')} {totals['imported']} of {totals['total']}",
+                style.dim(
+                    f"  {totals['already_imported']} already imported on an earlier run"
+                )
+                if totals["already_imported"]
+                else "",
                 style.dim(f"  {totals['skipped']} skipped as duplicate or low value")
                 if totals["skipped"]
                 else "",
@@ -692,6 +757,135 @@ def import_memories(ctx: dict) -> dict:
 # higher ``--batch-size`` is clamped rather than sent and refused.
 SERVER_BULK_MAX = 50
 
+#: Terminal job states. Polling stops at any of these.
+IMPORT_TERMINAL = frozenset({"completed", "failed", "cancelled"})
+
+
+def _poll_import(api, job_id: str) -> dict:
+    """Poll a job until it finishes.
+
+    Backs off from one second to five. A tight loop on a job that takes an hour is
+    thousands of pointless requests, and a fixed long interval makes a ten-second
+    job feel broken.
+    """
+    import time
+
+    wait_s = 1.0
+    while True:
+        job = api.get_import(job_id) or {}
+        if job.get("status") in IMPORT_TERMINAL:
+            return job
+        time.sleep(wait_s)
+        wait_s = min(5.0, wait_s * 1.5)
+
+
+def _render_import_job(job: dict, *, waited: bool = False) -> str:
+    """One job as a human-readable block.
+
+    Shared by ``import --async`` and ``import-status``.
+    """
+    status = job.get("status") or "unknown"
+    if status == "completed":
+        label = style.green("Completed")
+    elif status == "failed":
+        label = style.red("Failed")
+    elif status == "cancelled":
+        label = style.yellow("Cancelled")
+    else:
+        label = style.dim(status)
+
+    memories = job.get("memories_created") or 0
+    lines = [f"{label} {style.dim(job.get('id') or '')}"]
+    if job.get("total_rows"):
+        lines.append(
+            f"{style.dim('progress')} {job.get('processed_rows')}/{job.get('total_rows')}"
+            f" ({job.get('progress_percentage')}%)"
+        )
+    lines.append(
+        f"{style.dim('imported')} {job.get('created') or 0} record(s), "
+        f"{memories} memor{'y' if memories == 1 else 'ies'}"
+    )
+    if job.get("already_imported"):
+        lines.append(
+            style.dim(f"  {job['already_imported']} already imported on an earlier run")
+        )
+    if job.get("skipped"):
+        lines.append(style.dim(f"  {job['skipped']} skipped as duplicate or low value"))
+    if job.get("failed"):
+        lines.append(style.red(f"  {job['failed']} failed"))
+    for problem in (job.get("invalid_rows") or [])[:5]:
+        lines.append(style.dim(f"    {problem}"))
+    if job.get("error_message"):
+        lines.append(style.red(job["error_message"]))
+    if not waited and status not in IMPORT_TERMINAL:
+        lines.append(style.dim(f"Check again with: memorysync import-status {job.get('id')}"))
+    return "\n".join(lines)
+
+
+def import_status(ctx: dict) -> dict:
+    """Poll, list, or cancel a background import."""
+    flags, positionals, api = ctx["flags"], ctx["positionals"], ctx["api"]
+    job_id = positionals[0] if positionals else None
+
+    if not job_id:
+        response = api.list_imports(flags.get("limit")) or {}
+        jobs = response.get("jobs") or []
+        return {
+            "data": {"jobs": jobs, "count": len(jobs)},
+            "text": lambda: (
+                style.dim("No imports yet.")
+                if not jobs
+                else "\n\n".join(_render_import_job(job) for job in jobs)
+            ),
+        }
+
+    if flags.get("cancel"):
+        job = api.cancel_import(job_id) or {}
+        return {
+            "data": job,
+            "text": lambda: "\n".join(
+                [
+                    style.yellow("Cancellation requested."),
+                    style.dim("Records already imported stay imported."),
+                    _render_import_job(job),
+                ]
+            ),
+        }
+
+    waited = bool(flags.get("wait"))
+    job = _poll_import(api, job_id) if waited else (api.get_import(job_id) or {})
+    return {"data": job, "text": lambda: _render_import_job(job, waited=waited)}
+
+
+def _pick_ref(entry: dict, ref_field: str | None) -> str | None:
+    """The record's own id, for ``--resume``.
+
+    Checked in the order an export is likely to carry one. ``metadata.mem0_id``
+    is included because the Mem0 migration guide tells people to put it there, so
+    a file built by following our own documentation resumes without extra flags.
+    Numbers are accepted and stringified; a caller-supplied id is frequently an
+    integer and refusing it would be pedantry.
+    """
+    metadata = entry.get("metadata") if isinstance(entry.get("metadata"), dict) else {}
+    if ref_field:
+        candidates = [entry.get(ref_field), metadata.get(ref_field)]
+    else:
+        candidates = [
+            entry.get("id"),
+            entry.get("_id"),
+            entry.get("uuid"),
+            metadata.get("mem0_id"),
+        ]
+    for value in candidates:
+        if isinstance(value, str) and value.strip():
+            return value.strip()
+        # bool is an int subclass and is never a usable identifier.
+        if isinstance(value, int) and not isinstance(value, bool):
+            return str(value)
+        if isinstance(value, float) and value == int(value):
+            return str(int(value))
+    return None
+
 
 def _describe_read_failure(error: OSError) -> str:
     """Why a file could not be read, worded identically to the Node CLI.

{memorysync_cli-1.2.1 → memorysync_cli-1.4.0}/src/memorysync_cli/http.py

@@ -26,6 +26,8 @@ import time
 import urllib.error
 import urllib.parse
 import urllib.request
+import uuid
+from dataclasses import dataclass, field
 from typing import Any
 
 from ._version import __version__
@@ -42,6 +44,50 @@ from .errors import (
 END_USER_HEADER = "X-End-User-ID"
 
 
+@dataclass
+class MultipartBody:
+    """One file plus simple text fields, for an upload."""
+
+    filename: str
+    content: bytes
+    fields: dict[str, str] = field(default_factory=dict)
+
+
+def _encode_multipart(part: MultipartBody) -> tuple[bytes, str]:
+    """Build a ``multipart/form-data`` body by hand.
+
+    Written out rather than pulled from a library because this package has zero
+    dependencies, and the Node CLI's built-in ``FormData`` produces the same wire
+    format, so the two uploads are interchangeable.
+
+    The boundary is random per request. A fixed one would break the moment a
+    payload happened to contain it.
+    """
+    boundary = f"----MemorySync{uuid.uuid4().hex}"
+    lines: list[bytes] = []
+
+    for name, value in part.fields.items():
+        lines.append(f"--{boundary}\r\n".encode())
+        lines.append(
+            f'Content-Disposition: form-data; name="{name}"\r\n\r\n'.encode()
+        )
+        lines.append(f"{value}\r\n".encode())
+
+    # The filename is quoted and any quote inside it escaped, so a file called
+    # `my"file.jsonl` cannot break out of the header.
+    safe_name = part.filename.replace('"', '\\"')
+    lines.append(f"--{boundary}\r\n".encode())
+    lines.append(
+        f'Content-Disposition: form-data; name="file"; filename="{safe_name}"\r\n'.encode()
+    )
+    lines.append(b"Content-Type: application/octet-stream\r\n\r\n")
+    lines.append(part.content)
+    lines.append(b"\r\n")
+    lines.append(f"--{boundary}--\r\n".encode())
+
+    return b"".join(lines), f"multipart/form-data; boundary={boundary}"
+
+
 class ApiClient:
     """A thin, synchronous client over the MemorySync API."""
 
@@ -100,6 +146,7 @@ class ApiClient:
         body: Any = None,
         query: dict[str, Any] | None = None,
         timeout: int | None = None,
+        multipart: "MultipartBody | None" = None,
     ) -> Any:
         url = self.base_url + path
         if query:
@@ -113,7 +160,10 @@ class ApiClient:
 
         payload = None
         headers = self.headers()
-        if body is not None:
+        if multipart is not None:
+            payload, content_type = _encode_multipart(multipart)
+            headers["Content-Type"] = content_type
+        elif body is not None:
             payload = json.dumps(body).encode("utf-8")
             headers["Content-Type"] = "application/json"
 
@@ -368,6 +418,52 @@ class ApiClient:
         """
         return self.request("GET", "/evaluation/usage")
 
+    # ── Asynchronous bulk import ──────────────────────────────────────
+
+    def create_import(
+        self,
+        *,
+        filename: str,
+        content: bytes,
+        end_user_id: str | None,
+        resume: bool,
+        continue_on_error: bool,
+    ) -> Any:
+        """Upload a payload and get a job id back.
+
+        Sent as multipart rather than JSON because the payload can be tens of
+        megabytes and the server streams it out of storage rather than holding it.
+        """
+        fields: dict[str, str] = {
+            "resume": "true" if resume else "false",
+            "continue_on_error": "true" if continue_on_error else "false",
+        }
+        if end_user_id:
+            fields["end_user_id"] = end_user_id
+        return self.request(
+            "POST",
+            "/imports",
+            multipart=MultipartBody(
+                fields=fields,
+                filename=filename,
+                content=content,
+            ),
+            # An upload is not a 30-second request. The server validates the whole
+            # payload before answering.
+            timeout=300_000,
+        )
+
+    def get_import(self, job_id: str) -> Any:
+        return self.request("GET", f"/imports/{urllib.parse.quote(job_id, safe='')}")
+
+    def list_imports(self, limit: int | None = None) -> Any:
+        return self.request("GET", "/imports", query={"limit": limit})
+
+    def cancel_import(self, job_id: str) -> Any:
+        return self.request(
+            "POST", f"/imports/{urllib.parse.quote(job_id, safe='')}/cancel"
+        )
+
     def bulk_add_memories(self, body: dict) -> Any:
         """Ingest many records in one request.
 

{memorysync_cli-1.2.1 → memorysync_cli-1.4.0}/src/memorysync_cli/main.py

@@ -63,6 +63,7 @@ _HANDLERS: dict[str, Handler] = {
     "update": memory.update,
     "delete": memory.delete_memories,
     "import": memory.import_memories,
+    "import-status": memory.import_status,
     "export": memory.export_memories,
     "quota": admin.quota,
     "status": admin.status,

{memorysync_cli-1.2.1 → memorysync_cli-1.4.0}/src/memorysync_cli/registry.json

@@ -372,8 +372,8 @@
     {
       "name": "import",
       "summary": "Bulk load memories from a JSON or JSONL file.",
-      "description": "Each record needs a text field, and may carry its own user and metadata. Validates the whole file before sending anything, and reports per-row errors with line numbers rather than failing on the first bad row.",
-      "usage": "memorysync import <file> [--user <id>] [--batch-size <n>]",
+      "description": "Each record needs a text field, and may carry its own user and metadata. Validates the whole file before sending anything, and reports per-row errors with line numbers rather than failing on the first bad row. Pass --resume to make re-running the same file safe: each record is sent with an identifier the server remembers, so rows that already landed are reported as skipped instead of being stored a second time. Pass --async for a large file: the file is uploaded once and processed in the background, and you get a job id to poll with `memorysync import-status`.",
+      "usage": "memorysync import <file> [--user <id>] [--batch-size <n>] [--resume] [--async]",
       "flags": [
         {
           "name": "--user",
@@ -386,6 +386,23 @@
           "value": "n",
           "description": "Records per request. Default 50."
         },
+        {
+          "name": "--resume",
+          "description": "Send a per-record identifier so re-running the same file skips rows already stored."
+        },
+        {
+          "name": "--ref-field",
+          "value": "name",
+          "description": "Field holding each record’s own id, used by --resume. Default: id, then _id, then uuid."
+        },
+        {
+          "name": "--async",
+          "description": "Upload the file and return a job id instead of waiting. Use for large files."
+        },
+        {
+          "name": "--wait",
+          "description": "With --async, poll until the job finishes."
+        },
         {
           "name": "--continue-on-error",
           "description": "Keep going after a failed batch."
@@ -396,11 +413,39 @@
         }
       ],
       "examples": [
-        "memorysync import memories.jsonl --user alice --dry-run"
+        "memorysync import memories.jsonl --user alice --dry-run",
+        "memorysync import memories.jsonl --user alice --resume",
+        "memorysync import memories.jsonl --user alice --resume --async --wait"
       ],
       "requires_auth": true,
       "consumes_quota": "add_requests"
     },
+    {
+      "name": "import-status",
+      "summary": "Check a background import, or list recent ones.",
+      "description": "With a job id, reports progress and per-outcome counts. Without one, lists recent imports newest first. Pass --wait to poll until the job finishes, and --cancel to ask the worker to stop between batches — records already imported stay imported.",
+      "usage": "memorysync import-status [<job-id>] [--wait] [--cancel]",
+      "flags": [
+        {
+          "name": "--wait",
+          "description": "Poll until the job reaches a final state."
+        },
+        {
+          "name": "--cancel",
+          "description": "Ask the worker to stop between batches."
+        },
+        {
+          "name": "--limit",
+          "value": "n",
+          "description": "How many recent jobs to list. Default 20."
+        }
+      ],
+      "examples": [
+        "memorysync import-status",
+        "memorysync import-status 3f9a1c7e-0b2d-4a51-9e88-1c2d3e4f5a6b --wait"
+      ],
+      "requires_auth": true
+    },
     {
       "name": "export",
       "summary": "Write a user's memories to a file or stdout.",

memorysync_cli-1.2.1/src/memorysync_cli/_version.py

@@ -1 +0,0 @@
-__version__ = "1.2.1"