polygres-cli 0.1.1__tar.gz → 0.1.2__tar.gz

{polygres_cli-0.1.1 → polygres_cli-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polygres-cli
-Version: 0.1.1
+Version: 0.1.2
 Summary: Command-line interface for Polygres
 Project-URL: Homepage, https://polygres.com
 Project-URL: Documentation, https://docs.evokoa.com/polygres

{polygres_cli-0.1.1 → polygres_cli-0.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "polygres-cli"
-version = "0.1.1"
+version = "0.1.2"
 description = "Command-line interface for Polygres"
 readme = "README.md"
 requires-python = ">=3.10"

{polygres_cli-0.1.1 → polygres_cli-0.1.2}/src/polygres_cli/cli_client.py

@@ -1,19 +1,21 @@
 from __future__ import annotations
 
+import base64
+import hashlib
 import json as jsonlib
 import time
 from collections.abc import Callable
 from email.utils import parsedate_to_datetime
 from pathlib import Path
-from typing import Any, BinaryIO
-from urllib.parse import urlsplit
+from typing import Any
+from urllib.parse import quote, urlsplit
 
 import httpx
 
 from polygres_cli.cli_errors import AUTH, UNAVAILABLE, CliError, api_error_from_response
 from polygres_cli.cli_secrets import redact_string
 
-VERSION = "0.1.1"
+VERSION = "0.1.2"
 RETRY_STATUSES = {408, 429, 500, 502, 503, 504}
 HEAVY_REQUEST_TIMEOUT = 120.0
 
@@ -111,13 +113,121 @@ class CliControlPlaneClient:
         return self._delete(f"/projects/{project_id}/api-keys/{key_id}")
 
     def csv_preview(self, project_id: str, file: Path, fields: dict[str, str]) -> dict[str, Any]:
+        file_size = file.stat().st_size
+        session = self._post(
+            f"/projects/{project_id}/imports/csv/upload-sessions",
+            {"original_filename": file.name, "file_size_bytes": file_size},
+        )
+        upload = session.get("upload")
+        if not isinstance(upload, dict):
+            raise CliError("IMPORT_INVALID", "Upload session response is invalid.")
+        job_id = upload.get("job_id")
+        upload_url = upload.get("upload_url")
+        block_size = upload.get("block_size_bytes")
+        if (
+            not isinstance(job_id, str)
+            or not isinstance(upload_url, str)
+            or isinstance(block_size, bool)
+            or not isinstance(block_size, int)
+            or block_size <= 0
+        ):
+            raise CliError("IMPORT_INVALID", "Upload session response is incomplete.")
+        sha256 = self._upload_csv_blocks(file, upload_url, block_size)
+        payload: dict[str, Any] = {
+            "job_id": job_id,
+            "original_filename": file.name,
+            "file_size_bytes": file_size,
+            "sha256": sha256,
+            "target_schema": fields.get("target_schema", "public"),
+            "target_table": fields.get("target_table"),
+            "mode": fields.get("mode", "create_table"),
+            "encoding": fields.get("encoding", "utf-8"),
+            "quote_char": fields.get("quote_char", '"'),
+            "has_header": fields.get("has_header", "true") == "true",
+            "sample_row_count": int(fields.get("sample_row_count", "50")),
+        }
+        for name in ("delimiter", "escape_char"):
+            if name in fields:
+                payload[name] = fields[name]
+        return self._post(
+            f"/projects/{project_id}/imports/csv/upload-sessions/{job_id}/complete",
+            payload,
+            timeout=HEAVY_REQUEST_TIMEOUT,
+        )
+
+    def _upload_csv_blocks(self, file: Path, upload_url: str, block_size: int) -> str:
+        digest = hashlib.sha256()
+        block_ids: list[str] = []
         with file.open("rb") as handle:
-            return self._multipart(
-                "POST",
-                f"/projects/{project_id}/imports/csv/preview",
-                handle,
-                file.name,
-                fields,
+            index = 0
+            while chunk := handle.read(block_size):
+                digest.update(chunk)
+                block_id = base64.b64encode(f"{index:08d}".encode()).decode()
+                block_ids.append(block_id)
+                block_url = f"{upload_url}&comp=block&blockid={quote(block_id, safe='')}"
+                self._blob_request(
+                    "PUT",
+                    block_url,
+                    content=chunk,
+                    headers={
+                        "Content-MD5": base64.b64encode(
+                            hashlib.md5(chunk, usedforsecurity=False).digest()
+                        ).decode(),
+                        "x-ms-version": "2023-11-03",
+                    },
+                )
+                index += 1
+        if block_ids:
+            block_list = "<?xml version=\"1.0\" encoding=\"utf-8\"?><BlockList>" + "".join(
+                f"<Latest>{block_id}</Latest>" for block_id in block_ids
+            ) + "</BlockList>"
+            self._blob_request(
+                "PUT",
+                f"{upload_url}&comp=blocklist",
+                content=block_list.encode(),
+                headers={"Content-Type": "application/xml", "x-ms-version": "2023-11-03"},
+            )
+        else:
+            self._blob_request(
+                "PUT",
+                upload_url,
+                content=b"",
+                headers={"x-ms-blob-type": "BlockBlob", "x-ms-version": "2023-11-03"},
+            )
+        return digest.hexdigest()
+
+    def _blob_request(
+        self, method: str, url: str, *, content: bytes, headers: dict[str, str]
+    ) -> None:
+        response: httpx.Response | None = None
+        for attempt in range(self._max_retries + 1):
+            try:
+                response = self._client.request(
+                    method,
+                    url,
+                    content=content,
+                    headers=headers,
+                    timeout=HEAVY_REQUEST_TIMEOUT,
+                )
+            except (httpx.TimeoutException, httpx.NetworkError) as exc:
+                if attempt < self._max_retries:
+                    _sleep_before_retry(attempt, None)
+                    continue
+                raise CliError(
+                    "IMPORT_UPLOAD_FAILED",
+                    "Direct CSV upload is unavailable.",
+                    exit_code=UNAVAILABLE,
+                ) from exc
+            if response.status_code in RETRY_STATUSES and attempt < self._max_retries:
+                _sleep_before_retry(attempt, response.headers.get("Retry-After"))
+                continue
+            break
+        assert response is not None
+        if response.is_error:
+            raise CliError(
+                "IMPORT_UPLOAD_FAILED",
+                f"Direct CSV upload failed with HTTP {response.status_code}.",
+                exit_code=UNAVAILABLE,
             )
 
     def start_csv_import(
@@ -243,30 +353,12 @@ class CliControlPlaneClient:
     def _delete(self, path: str) -> dict[str, Any]:
         return self._request("DELETE", path, retry=False)
 
-    def _multipart(
-        self,
-        method: str,
-        path: str,
-        file: BinaryIO,
-        filename: str,
-        fields: dict[str, str],
-    ) -> dict[str, Any]:
-        return self._request(
-            method,
-            path,
-            data=fields,
-            files={"file": (filename, file, "text/csv")},
-            retry=False,
-        )
-
     def _request(
         self,
         method: str,
         path: str,
         *,
         json: dict[str, Any] | None = None,
-        data: dict[str, str] | None = None,
-        files: dict[str, Any] | None = None,
         auth: bool = True,
         retry: bool = False,
         allow_refresh: bool = True,
@@ -289,8 +381,6 @@ class CliControlPlaneClient:
             try:
                 request_kwargs: dict[str, Any] = {
                     "headers": headers,
-                    "data": data,
-                    "files": files,
                 }
                 if json is not None:
                     headers["Content-Type"] = "application/json"
@@ -334,8 +424,6 @@ class CliControlPlaneClient:
                     method,
                     path,
                     json=json,
-                    data=data,
-                    files=files,
                     auth=auth,
                     retry=retry,
                     allow_refresh=False,

{polygres_cli-0.1.1 → polygres_cli-0.1.2}/tests/heavy/test_cli_installed_staging.py

@@ -31,7 +31,7 @@ def test_installed_cli_staging_whoami_contract(tmp_path: Path) -> None:
         [executable, "--version"], env=env, text=True, capture_output=True, check=False
     )
     assert version.returncode == 0
-    assert version.stdout.strip() == "polygres 0.1.1"
+    assert version.stdout.strip() == "polygres 0.1.2"
 
     whoami = subprocess.run(
         [executable, "--json", "whoami"],

{polygres_cli-0.1.1 → polygres_cli-0.1.2}/tests/test_cli.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
+import hashlib
 import json
 import os
+import re
 import stat
 import subprocess
 from datetime import datetime, timedelta, timezone
@@ -36,6 +38,46 @@ def _stub(route: object, **kwargs: object) -> object:
     return getattr(route, "mo" + "ck")(**kwargs)
 
 
+def _stub_csv_direct_preview(
+    preview: dict[str, object], *, status_code: int = 200
+) -> tuple[object, object, object, object]:
+    session_route = _stub(
+        respx.post(f"{API_BASE_URL}/projects/{PROJECT_ID}/imports/csv/upload-sessions"),
+        return_value=httpx.Response(
+            status_code if status_code != 200 else 200,
+            json=(
+                preview
+                if status_code != 200
+                else {
+                    "upload": {
+                        "job_id": IMPORT_ID,
+                        "upload_url": "https://blob.example.test/file.csv?sig=secret",
+                        "block_size_bytes": 8 * 1024 * 1024,
+                    }
+                }
+            ),
+        ),
+    )
+    block_route = _stub(
+        respx.put(
+            re.compile(r"https://blob\.example\.test/file\.csv\?.*comp=block&blockid=.*")
+        ),
+        return_value=httpx.Response(201),
+    )
+    commit_route = _stub(
+        respx.put(re.compile(r"https://blob\.example\.test/file\.csv\?.*comp=blocklist.*")),
+        return_value=httpx.Response(201),
+    )
+    complete_route = _stub(
+        respx.post(
+            f"{API_BASE_URL}/projects/{PROJECT_ID}/imports/csv/upload-sessions/"
+            f"{IMPORT_ID}/complete"
+        ),
+        return_value=httpx.Response(200, json=preview),
+    )
+    return session_route, block_route, commit_route, complete_route
+
+
 def run_cli(
     args: list[str],
     capsys: pytest.CaptureFixture[str],
@@ -93,7 +135,7 @@ def test_version_and_config_path_json(
 ) -> None:
     rc, out, _ = run_cli(["--version"], capsys, monkeypatch, tmp_path)
     assert rc == 0
-    assert out.strip() == "polygres 0.1.1"
+    assert out.strip() == "polygres 0.1.2"
 
     rc, out, _ = run_cli(["--json", "config", "path"], capsys, monkeypatch, tmp_path)
     assert rc == 0
@@ -430,7 +472,7 @@ def test_projects_list_uses_env_token_and_selected_project_json(
     assert err == ""
     assert route.called
     assert route.calls[0].request.headers["Authorization"] == f"Bearer {ACCESS_TOKEN}"
-    assert route.calls[0].request.headers["User-Agent"] == "polygres-cli/0.1.1"
+    assert route.calls[0].request.headers["User-Agent"] == "polygres-cli/0.1.2"
     assert json.loads(out) == {
         "projects": [{"id": PROJECT_ID, "name": "Support", "status": "ready"}],
         "selected_project_id": PROJECT_ID,
@@ -1066,18 +1108,14 @@ def test_import_csv_sends_sample_row_count_only_to_preview(
     write_config(tmp_path, {"version": 1, "selected_project_id": PROJECT_ID})
     csv_path = tmp_path / "documents.csv"
     csv_path.write_text("id,title\n1,Hello\n", encoding="utf-8")
-    preview_route = _stub(
-        respx.post(f"{API_BASE_URL}/projects/{PROJECT_ID}/imports/csv/preview"),
-        return_value=httpx.Response(
-            200,
-            json={
-                "request_id": "req_preview",
-                "preview": {
-                    "job_id": IMPORT_ID,
-                    "columns": [{"name": "id"}, {"name": "title"}],
-                },
+    session_route, block_route, commit_route, preview_route = _stub_csv_direct_preview(
+        {
+            "request_id": "req_preview",
+            "preview": {
+                "job_id": IMPORT_ID,
+                "columns": [{"name": "id"}, {"name": "title"}],
             },
-        ),
+        }
     )
     import_route = _stub(
         respx.post(f"{API_BASE_URL}/projects/{PROJECT_ID}/imports/csv"),
@@ -1110,9 +1148,15 @@ def test_import_csv_sends_sample_row_count_only_to_preview(
     assert err == ""
     assert preview_route.called
     assert import_route.called
-    preview_body = preview_route.calls[0].request.content
+    complete_payload = json.loads(preview_route.calls[0].request.content)
     import_payload = json.loads(import_route.calls[0].request.content)
-    assert b'name="sample_row_count"' in preview_body
+    assert session_route.called
+    assert block_route.called
+    assert commit_route.called
+    assert complete_payload["sample_row_count"] == 50
+    assert complete_payload["file_size_bytes"] == csv_path.stat().st_size
+    assert complete_payload["sha256"] == hashlib.sha256(csv_path.read_bytes()).hexdigest()
+    assert csv_path.read_bytes() not in preview_route.calls[0].request.content
     assert import_route.calls[0].request.headers["content-type"] == "application/json"
     assert import_payload["job_id"] == IMPORT_ID
     assert [column["name"] for column in import_payload["columns"]] == ["id", "title"]
@@ -1129,19 +1173,16 @@ def test_import_csv_preserves_tier_limit_error_and_skips_final_submission(
     write_config(tmp_path, {"version": 1, "selected_project_id": PROJECT_ID})
     csv_path = tmp_path / "documents.csv"
     csv_path.write_text("id,title\n1,Hello\n", encoding="utf-8")
-    preview_route = _stub(
-        respx.post(f"{API_BASE_URL}/projects/{PROJECT_ID}/imports/csv/preview"),
-        return_value=httpx.Response(
-            413,
-            json={
-                "request_id": "req_import_limit",
-                "error": {
-                    "code": "IMPORT_LIMIT_EXCEEDED",
-                    "message": "File exceeds the project tier storage limit.",
-                    "details": {"limit_bytes": 8, "source": "storage_bytes"},
-                },
+    preview_route, _, _, _ = _stub_csv_direct_preview(
+        {
+            "request_id": "req_import_limit",
+            "error": {
+                "code": "IMPORT_LIMIT_EXCEEDED",
+                "message": "File exceeds the project tier storage limit.",
+                "details": {"limit_bytes": 8, "source": "storage_bytes"},
             },
-        ),
+        },
+        status_code=413,
     )
     import_route = _stub(respx.post(f"{API_BASE_URL}/projects/{PROJECT_ID}/imports/csv"))
 
@@ -1605,16 +1646,12 @@ def test_csv_import_uses_preview_columns_and_wait_polling(
     csv_path = tmp_path / "documents.csv"
     csv_path.write_text("id,title\n1,Hello\n", encoding="utf-8")
     columns = [{"name": "id", "type": "text", "nullable": False}]
-    preview_route = _stub(
-        respx.post(f"{API_BASE_URL}/projects/{PROJECT_ID}/imports/csv/preview"),
-        return_value=httpx.Response(
-            200,
-            json={
-                "request_id": "req_preview",
-                "preview": {"job_id": IMPORT_ID, "columns": columns},
-            },
-        ),
-    )
+    preview_route = _stub_csv_direct_preview(
+        {
+            "request_id": "req_preview",
+            "preview": {"job_id": IMPORT_ID, "columns": columns},
+        }
+    )[3]
     import_route = _stub(
         respx.post(f"{API_BASE_URL}/projects/{PROJECT_ID}/imports/csv"),
         return_value=httpx.Response(
@@ -1846,7 +1883,7 @@ def test_login_browser_fallback_polls_stores_tokens_and_redacts_output(
     assert ACCESS_TOKEN not in out
     assert REFRESH_TOKEN not in out
     assert json.loads(start_route.calls[0].request.content) == {
-        "client": {"name": "polygres-cli", "version": "0.1.1"}
+        "client": {"name": "polygres-cli", "version": "0.1.2"}
     }
     assert json.loads(poll_route.calls[0].request.content) == {
         "login_session_id": "cls_abcdefghijklmnopqrstuvwxyz",
@@ -1914,22 +1951,18 @@ def test_csv_import_propagates_preview_effective_parser_settings(
     path = tmp_path / "documents.csv"
     path.write_text("id|title\n1|Hello\n", encoding="utf-8")
     columns = [{"name": "id", "type": "text", "nullable": False}]
-    _stub(
-        respx.post(f"{API_BASE_URL}/projects/{PROJECT_ID}/imports/csv/preview"),
-        return_value=httpx.Response(
-            200,
-            json={
-                "preview": {
-                    "job_id": IMPORT_ID,
-                    "encoding": "utf-8-sig",
-                    "delimiter": "|",
-                    "quote_char": "'",
-                    "escape_char": "\\",
-                    "has_header": False,
-                    "columns": columns,
-                }
-            },
-        ),
+    _stub_csv_direct_preview(
+        {
+            "preview": {
+                "job_id": IMPORT_ID,
+                "encoding": "utf-8-sig",
+                "delimiter": "|",
+                "quote_char": "'",
+                "escape_char": "\\",
+                "has_header": False,
+                "columns": columns,
+            }
+        }
     )
     import_route = _stub(
         respx.post(f"{API_BASE_URL}/projects/{PROJECT_ID}/imports/csv"),
@@ -2084,11 +2117,8 @@ def test_human_import_polling_writes_progress_only_to_stderr(
     path = tmp_path / "documents.csv"
     path.write_text("id\n1\n", encoding="utf-8")
     monkeypatch.setattr(cli.time, "sleep", lambda _seconds: None)
-    _stub(
-        respx.post(f"{API_BASE_URL}/projects/{PROJECT_ID}/imports/csv/preview"),
-        return_value=httpx.Response(
-            200, json={"preview": {"job_id": IMPORT_ID, "columns": [{"name": "id"}]}}
-        ),
+    _stub_csv_direct_preview(
+        {"preview": {"job_id": IMPORT_ID, "columns": [{"name": "id"}]}}
     )
     _stub(
         respx.post(f"{API_BASE_URL}/projects/{PROJECT_ID}/imports/csv"),