memorysync-cli 1.3.0__tar.gz → 1.4.0__tar.gz

{memorysync_cli-1.3.0 → memorysync_cli-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: memorysync-cli
-Version: 1.3.0
+Version: 1.4.0
 Summary: MemorySync from your terminal. Zero dependencies.
 Project-URL: Documentation, https://docs.memorysync.io/cli
 Project-URL: Homepage, https://memorysync.io/cli

memorysync_cli-1.4.0/src/memorysync_cli/_version.py

@@ -0,0 +1 @@
+__version__ = "1.4.0"

{memorysync_cli-1.3.0 → memorysync_cli-1.4.0}/src/memorysync_cli/commands/memory.py

@@ -627,6 +627,25 @@ def import_memories(ctx: dict) -> dict:
                 "--resume.",
             )
 
+    # --async hands the whole file to the server and returns a job id. The
+    # synchronous path below is capped at 50 records per request by the route, so a
+    # large file is thousands of round trips held open by one command; this is one
+    # upload and a poll.
+    if flags.get("async"):
+        job = api.create_import(
+            filename=Path(source_path).name,
+            content=raw.encode("utf-8"),
+            end_user_id=user,
+            resume=bool(flags.get("resume")),
+            continue_on_error=bool(flags.get("continue_on_error")),
+        )
+        waited = bool(flags.get("wait"))
+        finished = _poll_import(api, job["id"]) if waited else job
+        return {
+            "data": {"file": source_path, "user": user, **finished},
+            "text": lambda: _render_import_job(finished, waited=waited),
+        }
+
     batch_size = _resolve_batch_size(flags)
     batches = [records[i : i + batch_size] for i in range(0, len(records), batch_size)]
     totals = {
@@ -738,6 +757,105 @@ def import_memories(ctx: dict) -> dict:
 # higher ``--batch-size`` is clamped rather than sent and refused.
 SERVER_BULK_MAX = 50
 
+#: Terminal job states. Polling stops at any of these.
+IMPORT_TERMINAL = frozenset({"completed", "failed", "cancelled"})
+
+
+def _poll_import(api, job_id: str) -> dict:
+    """Poll a job until it finishes.
+
+    Backs off from one second to five. A tight loop on a job that takes an hour is
+    thousands of pointless requests, and a fixed long interval makes a ten-second
+    job feel broken.
+    """
+    import time
+
+    wait_s = 1.0
+    while True:
+        job = api.get_import(job_id) or {}
+        if job.get("status") in IMPORT_TERMINAL:
+            return job
+        time.sleep(wait_s)
+        wait_s = min(5.0, wait_s * 1.5)
+
+
+def _render_import_job(job: dict, *, waited: bool = False) -> str:
+    """One job as a human-readable block.
+
+    Shared by ``import --async`` and ``import-status``.
+    """
+    status = job.get("status") or "unknown"
+    if status == "completed":
+        label = style.green("Completed")
+    elif status == "failed":
+        label = style.red("Failed")
+    elif status == "cancelled":
+        label = style.yellow("Cancelled")
+    else:
+        label = style.dim(status)
+
+    memories = job.get("memories_created") or 0
+    lines = [f"{label} {style.dim(job.get('id') or '')}"]
+    if job.get("total_rows"):
+        lines.append(
+            f"{style.dim('progress')} {job.get('processed_rows')}/{job.get('total_rows')}"
+            f" ({job.get('progress_percentage')}%)"
+        )
+    lines.append(
+        f"{style.dim('imported')} {job.get('created') or 0} record(s), "
+        f"{memories} memor{'y' if memories == 1 else 'ies'}"
+    )
+    if job.get("already_imported"):
+        lines.append(
+            style.dim(f"  {job['already_imported']} already imported on an earlier run")
+        )
+    if job.get("skipped"):
+        lines.append(style.dim(f"  {job['skipped']} skipped as duplicate or low value"))
+    if job.get("failed"):
+        lines.append(style.red(f"  {job['failed']} failed"))
+    for problem in (job.get("invalid_rows") or [])[:5]:
+        lines.append(style.dim(f"    {problem}"))
+    if job.get("error_message"):
+        lines.append(style.red(job["error_message"]))
+    if not waited and status not in IMPORT_TERMINAL:
+        lines.append(style.dim(f"Check again with: memorysync import-status {job.get('id')}"))
+    return "\n".join(lines)
+
+
+def import_status(ctx: dict) -> dict:
+    """Poll, list, or cancel a background import."""
+    flags, positionals, api = ctx["flags"], ctx["positionals"], ctx["api"]
+    job_id = positionals[0] if positionals else None
+
+    if not job_id:
+        response = api.list_imports(flags.get("limit")) or {}
+        jobs = response.get("jobs") or []
+        return {
+            "data": {"jobs": jobs, "count": len(jobs)},
+            "text": lambda: (
+                style.dim("No imports yet.")
+                if not jobs
+                else "\n\n".join(_render_import_job(job) for job in jobs)
+            ),
+        }
+
+    if flags.get("cancel"):
+        job = api.cancel_import(job_id) or {}
+        return {
+            "data": job,
+            "text": lambda: "\n".join(
+                [
+                    style.yellow("Cancellation requested."),
+                    style.dim("Records already imported stay imported."),
+                    _render_import_job(job),
+                ]
+            ),
+        }
+
+    waited = bool(flags.get("wait"))
+    job = _poll_import(api, job_id) if waited else (api.get_import(job_id) or {})
+    return {"data": job, "text": lambda: _render_import_job(job, waited=waited)}
+
 
 def _pick_ref(entry: dict, ref_field: str | None) -> str | None:
     """The record's own id, for ``--resume``.

{memorysync_cli-1.3.0 → memorysync_cli-1.4.0}/src/memorysync_cli/http.py

@@ -26,6 +26,8 @@ import time
 import urllib.error
 import urllib.parse
 import urllib.request
+import uuid
+from dataclasses import dataclass, field
 from typing import Any
 
 from ._version import __version__
@@ -42,6 +44,50 @@ from .errors import (
 END_USER_HEADER = "X-End-User-ID"
 
 
+@dataclass
+class MultipartBody:
+    """One file plus simple text fields, for an upload."""
+
+    filename: str
+    content: bytes
+    fields: dict[str, str] = field(default_factory=dict)
+
+
+def _encode_multipart(part: MultipartBody) -> tuple[bytes, str]:
+    """Build a ``multipart/form-data`` body by hand.
+
+    Written out rather than pulled from a library because this package has zero
+    dependencies, and the Node CLI's built-in ``FormData`` produces the same wire
+    format, so the two uploads are interchangeable.
+
+    The boundary is random per request. A fixed one would break the moment a
+    payload happened to contain it.
+    """
+    boundary = f"----MemorySync{uuid.uuid4().hex}"
+    lines: list[bytes] = []
+
+    for name, value in part.fields.items():
+        lines.append(f"--{boundary}\r\n".encode())
+        lines.append(
+            f'Content-Disposition: form-data; name="{name}"\r\n\r\n'.encode()
+        )
+        lines.append(f"{value}\r\n".encode())
+
+    # The filename is quoted and any quote inside it escaped, so a file called
+    # `my"file.jsonl` cannot break out of the header.
+    safe_name = part.filename.replace('"', '\\"')
+    lines.append(f"--{boundary}\r\n".encode())
+    lines.append(
+        f'Content-Disposition: form-data; name="file"; filename="{safe_name}"\r\n'.encode()
+    )
+    lines.append(b"Content-Type: application/octet-stream\r\n\r\n")
+    lines.append(part.content)
+    lines.append(b"\r\n")
+    lines.append(f"--{boundary}--\r\n".encode())
+
+    return b"".join(lines), f"multipart/form-data; boundary={boundary}"
+
+
 class ApiClient:
     """A thin, synchronous client over the MemorySync API."""
 
@@ -100,6 +146,7 @@ class ApiClient:
         body: Any = None,
         query: dict[str, Any] | None = None,
         timeout: int | None = None,
+        multipart: "MultipartBody | None" = None,
     ) -> Any:
         url = self.base_url + path
         if query:
@@ -113,7 +160,10 @@ class ApiClient:
 
         payload = None
         headers = self.headers()
-        if body is not None:
+        if multipart is not None:
+            payload, content_type = _encode_multipart(multipart)
+            headers["Content-Type"] = content_type
+        elif body is not None:
             payload = json.dumps(body).encode("utf-8")
             headers["Content-Type"] = "application/json"
 
@@ -368,6 +418,52 @@ class ApiClient:
         """
         return self.request("GET", "/evaluation/usage")
 
+    # ── Asynchronous bulk import ──────────────────────────────────────
+
+    def create_import(
+        self,
+        *,
+        filename: str,
+        content: bytes,
+        end_user_id: str | None,
+        resume: bool,
+        continue_on_error: bool,
+    ) -> Any:
+        """Upload a payload and get a job id back.
+
+        Sent as multipart rather than JSON because the payload can be tens of
+        megabytes and the server streams it out of storage rather than holding it.
+        """
+        fields: dict[str, str] = {
+            "resume": "true" if resume else "false",
+            "continue_on_error": "true" if continue_on_error else "false",
+        }
+        if end_user_id:
+            fields["end_user_id"] = end_user_id
+        return self.request(
+            "POST",
+            "/imports",
+            multipart=MultipartBody(
+                fields=fields,
+                filename=filename,
+                content=content,
+            ),
+            # An upload is not a 30-second request. The server validates the whole
+            # payload before answering.
+            timeout=300_000,
+        )
+
+    def get_import(self, job_id: str) -> Any:
+        return self.request("GET", f"/imports/{urllib.parse.quote(job_id, safe='')}")
+
+    def list_imports(self, limit: int | None = None) -> Any:
+        return self.request("GET", "/imports", query={"limit": limit})
+
+    def cancel_import(self, job_id: str) -> Any:
+        return self.request(
+            "POST", f"/imports/{urllib.parse.quote(job_id, safe='')}/cancel"
+        )
+
     def bulk_add_memories(self, body: dict) -> Any:
         """Ingest many records in one request.
 

{memorysync_cli-1.3.0 → memorysync_cli-1.4.0}/src/memorysync_cli/main.py

@@ -63,6 +63,7 @@ _HANDLERS: dict[str, Handler] = {
     "update": memory.update,
     "delete": memory.delete_memories,
     "import": memory.import_memories,
+    "import-status": memory.import_status,
     "export": memory.export_memories,
     "quota": admin.quota,
     "status": admin.status,

{memorysync_cli-1.3.0 → memorysync_cli-1.4.0}/src/memorysync_cli/registry.json

@@ -372,8 +372,8 @@
     {
       "name": "import",
       "summary": "Bulk load memories from a JSON or JSONL file.",
-      "description": "Each record needs a text field, and may carry its own user and metadata. Validates the whole file before sending anything, and reports per-row errors with line numbers rather than failing on the first bad row. Pass --resume to make re-running the same file safe: each record is sent with an identifier the server remembers, so rows that already landed are reported as skipped instead of being stored a second time.",
-      "usage": "memorysync import <file> [--user <id>] [--batch-size <n>] [--resume]",
+      "description": "Each record needs a text field, and may carry its own user and metadata. Validates the whole file before sending anything, and reports per-row errors with line numbers rather than failing on the first bad row. Pass --resume to make re-running the same file safe: each record is sent with an identifier the server remembers, so rows that already landed are reported as skipped instead of being stored a second time. Pass --async for a large file: the file is uploaded once and processed in the background, and you get a job id to poll with `memorysync import-status`.",
+      "usage": "memorysync import <file> [--user <id>] [--batch-size <n>] [--resume] [--async]",
       "flags": [
         {
           "name": "--user",
@@ -395,6 +395,14 @@
           "value": "name",
           "description": "Field holding each record’s own id, used by --resume. Default: id, then _id, then uuid."
         },
+        {
+          "name": "--async",
+          "description": "Upload the file and return a job id instead of waiting. Use for large files."
+        },
+        {
+          "name": "--wait",
+          "description": "With --async, poll until the job finishes."
+        },
         {
           "name": "--continue-on-error",
           "description": "Keep going after a failed batch."
@@ -406,11 +414,38 @@
       ],
       "examples": [
         "memorysync import memories.jsonl --user alice --dry-run",
-        "memorysync import memories.jsonl --user alice --resume"
+        "memorysync import memories.jsonl --user alice --resume",
+        "memorysync import memories.jsonl --user alice --resume --async --wait"
       ],
       "requires_auth": true,
       "consumes_quota": "add_requests"
     },
+    {
+      "name": "import-status",
+      "summary": "Check a background import, or list recent ones.",
+      "description": "With a job id, reports progress and per-outcome counts. Without one, lists recent imports newest first. Pass --wait to poll until the job finishes, and --cancel to ask the worker to stop between batches — records already imported stay imported.",
+      "usage": "memorysync import-status [<job-id>] [--wait] [--cancel]",
+      "flags": [
+        {
+          "name": "--wait",
+          "description": "Poll until the job reaches a final state."
+        },
+        {
+          "name": "--cancel",
+          "description": "Ask the worker to stop between batches."
+        },
+        {
+          "name": "--limit",
+          "value": "n",
+          "description": "How many recent jobs to list. Default 20."
+        }
+      ],
+      "examples": [
+        "memorysync import-status",
+        "memorysync import-status 3f9a1c7e-0b2d-4a51-9e88-1c2d3e4f5a6b --wait"
+      ],
+      "requires_auth": true
+    },
     {
       "name": "export",
       "summary": "Write a user's memories to a file or stdout.",

memorysync_cli-1.3.0/src/memorysync_cli/_version.py

@@ -1 +0,0 @@
-__version__ = "1.3.0"