mainsequence 4.3.20__tar.gz → 4.3.24__tar.gz

{mainsequence-4.3.20/mainsequence.egg-info → mainsequence-4.3.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mainsequence
-Version: 4.3.20
+Version: 4.3.24
 Summary: Main Sequence SDK 
 Author-email: Main Sequence GmbH <dev@main-sequence.io>
 License: MainSequence GmbH SDK License Agreement

{mainsequence-4.3.20 → mainsequence-4.3.24}/mainsequence/client/metatables/core.py

@@ -500,9 +500,17 @@ class MetaTableOperationScope(BasePydanticModel):
         return self
 
 
+DEFAULT_META_TABLE_OPERATION_MAX_ROWS = 1_000
+DEFAULT_META_TABLE_OPERATION_STATEMENT_TIMEOUT_MS = 15_000
+
+
 class MetaTableOperationLimits(BasePydanticModel):
-    max_rows: int | None = Field(default=None, ge=1)
-    statement_timeout_ms: int | None = Field(default=None, ge=1)
+    max_rows: int = Field(default=DEFAULT_META_TABLE_OPERATION_MAX_ROWS, ge=1)
+    offset: int = Field(default=0, ge=0)
+    statement_timeout_ms: int = Field(
+        default=DEFAULT_META_TABLE_OPERATION_STATEMENT_TIMEOUT_MS,
+        ge=1,
+    )
 
 
 class MetaTableCompiledSQLOperation(BasePydanticModel):
@@ -511,7 +519,12 @@ class MetaTableCompiledSQLOperation(BasePydanticModel):
     dialect: MetaTableCompiledSQLDialect = "postgresql"
     statement: MetaTableStatementPayload
     scope: MetaTableOperationScope
-    limits: MetaTableOperationLimits | None = None
+    limits: MetaTableOperationLimits = Field(default_factory=MetaTableOperationLimits)
+
+    @field_validator("limits", mode="before")
+    @classmethod
+    def default_limits(cls, value):
+        return {} if value is None else value
 
 
 class MetaTableRequestFields(BasePydanticModel):
@@ -1704,12 +1717,68 @@ class MetaTable(BasePydanticModel, LabelableObjectMixin, ShareableObjectMixin, B
             if isinstance(operation, MetaTableCompiledSQLOperation)
             else MetaTableCompiledSQLOperation(**operation)
         )
-        return cls._post_action(
+        response = cls._post_action(
             "execute-operation",
             payload,
             timeout=timeout,
             expected_statuses=(200,),
         )
+        if payload.operation != "select":
+            return response
+
+        rows = response.get("rows")
+        if not isinstance(rows, list):
+            return response
+
+        requested_max_rows = payload.limits.max_rows
+        if len(rows) >= requested_max_rows:
+            return response
+
+        pagination = response.get("pagination") or {}
+        has_more = bool(pagination.get("has_more"))
+        next_offset = pagination.get("next_offset")
+        if not has_more or next_offset in (None, ""):
+            return response
+
+        accumulated_rows = list(rows)
+        last_pagination = pagination
+        while (
+            has_more
+            and next_offset not in (None, "")
+            and len(accumulated_rows) < requested_max_rows
+        ):
+            remaining_rows = requested_max_rows - len(accumulated_rows)
+            next_payload = payload.model_copy(deep=True)
+            next_payload.limits.offset = int(next_offset)
+            next_payload.limits.max_rows = remaining_rows
+            page_response = cls._post_action(
+                "execute-operation",
+                next_payload,
+                timeout=timeout,
+                expected_statuses=(200,),
+            )
+            page_rows = page_response.get("rows") or []
+            if not isinstance(page_rows, list) or not page_rows:
+                last_pagination = page_response.get("pagination") or {}
+                break
+
+            accumulated_rows.extend(page_rows[:remaining_rows])
+            last_pagination = page_response.get("pagination") or {}
+            has_more = bool(last_pagination.get("has_more"))
+            next_offset = last_pagination.get("next_offset")
+
+        response["rows"] = accumulated_rows
+        response["row_count"] = len(accumulated_rows)
+        response["max_rows"] = requested_max_rows
+        response["truncated"] = bool(last_pagination.get("has_more"))
+        response["pagination"] = {
+            "limit": requested_max_rows,
+            "offset": payload.limits.offset,
+            "returned_count": len(accumulated_rows),
+            "has_more": bool(last_pagination.get("has_more")),
+            "next_offset": last_pagination.get("next_offset"),
+        }
+        return response
 
 
 # Global executor (or you could define one on your class)

{mainsequence-4.3.20 → mainsequence-4.3.24/mainsequence.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mainsequence
-Version: 4.3.20
+Version: 4.3.24
 Summary: Main Sequence SDK 
 Author-email: Main Sequence GmbH <dev@main-sequence.io>
 License: MainSequence GmbH SDK License Agreement

{mainsequence-4.3.20 → mainsequence-4.3.24}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "mainsequence"
-version = "4.3.20"
+version = "4.3.24"
 description = "Main Sequence SDK "
 readme = "README.md"
 requires-python = ">=3.11"

{mainsequence-4.3.20 → mainsequence-4.3.24}/tests/test_meta_tables_client_models.py

@@ -410,6 +410,182 @@ def test_meta_table_execute_operation_serializes_scope_uid(monkeypatch):
     assert captured["payload"]["json"]["statement"]["parameters"] == {
         "symbol_1": "%BTC%",
     }
+    assert captured["payload"]["json"]["limits"]["offset"] == 0
+
+
+def test_meta_table_execute_operation_fetches_requested_rows_with_backend_pagination(
+    monkeypatch,
+):
+    calls = []
+
+    def fake_make_request(**kwargs):
+        payload = kwargs["payload"]["json"]
+        calls.append(payload)
+        offset = payload["limits"]["offset"]
+        if offset == 0:
+            return _Response(
+                {
+                    "ok": True,
+                    "operation": "select",
+                    "dialect": "postgresql",
+                    "row_count": 2,
+                    "rows": [{"value": 1}, {"value": 2}],
+                    "truncated": True,
+                    "max_rows": 2,
+                    "pagination": {
+                        "limit": 2,
+                        "offset": 0,
+                        "returned_count": 2,
+                        "has_more": True,
+                        "next_offset": 2,
+                    },
+                }
+            )
+        if offset == 2:
+            return _Response(
+                {
+                    "ok": True,
+                    "operation": "select",
+                    "dialect": "postgresql",
+                    "row_count": 2,
+                    "rows": [{"value": 3}, {"value": 4}],
+                    "truncated": True,
+                    "max_rows": 2,
+                    "pagination": {
+                        "limit": 2,
+                        "offset": 2,
+                        "returned_count": 2,
+                        "has_more": True,
+                        "next_offset": 4,
+                    },
+                }
+            )
+        return _Response(
+            {
+                "ok": True,
+                "operation": "select",
+                "dialect": "postgresql",
+                "row_count": 1,
+                "rows": [{"value": 5}],
+                "truncated": False,
+                "max_rows": 1,
+                "pagination": {
+                    "limit": 1,
+                    "offset": 4,
+                    "returned_count": 1,
+                    "has_more": False,
+                    "next_offset": None,
+                },
+            }
+        )
+
+    monkeypatch.setattr(meta_table_models, "make_request", fake_make_request)
+    monkeypatch.setattr(
+        meta_table_models.MetaTable,
+        "build_session",
+        classmethod(lambda cls: SimpleNamespace(headers={})),
+    )
+
+    result = meta_table_models.MetaTable.execute_operation(
+        {
+            "operation": "select",
+            "statement": {
+                "sql": "SELECT value FROM public.asset ORDER BY value",
+                "parameters": {},
+            },
+            "scope": {
+                "data_source_uid": "dddddddd-dddd-4ddd-8ddd-dddddddddddd",
+                "tables": [
+                    {
+                        "meta_table_uid": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
+                        "alias": "asset",
+                    }
+                ],
+            },
+            "limits": {
+                "max_rows": 5,
+                "statement_timeout_ms": 15000,
+            },
+        }
+    )
+
+    assert [row["value"] for row in result["rows"]] == [1, 2, 3, 4, 5]
+    assert result["row_count"] == 5
+    assert result["max_rows"] == 5
+    assert result["truncated"] is False
+    assert result["pagination"] == {
+        "limit": 5,
+        "offset": 0,
+        "returned_count": 5,
+        "has_more": False,
+        "next_offset": None,
+    }
+    assert [call["limits"]["offset"] for call in calls] == [0, 2, 4]
+    assert [call["limits"]["max_rows"] for call in calls] == [5, 3, 1]
+
+
+def test_meta_table_execute_operation_does_not_paginate_upsert(monkeypatch):
+    calls = []
+
+    def fake_make_request(**kwargs):
+        calls.append(kwargs["payload"]["json"])
+        return _Response(
+            {
+                "ok": True,
+                "operation": "upsert",
+                "dialect": "postgresql",
+                "row_count": 1,
+                "rows": [{"uid": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"}],
+                "truncated": True,
+                "max_rows": 1,
+                "pagination": {
+                    "limit": 1,
+                    "offset": 0,
+                    "returned_count": 1,
+                    "has_more": True,
+                    "next_offset": 1,
+                },
+            }
+        )
+
+    monkeypatch.setattr(meta_table_models, "make_request", fake_make_request)
+    monkeypatch.setattr(
+        meta_table_models.MetaTable,
+        "build_session",
+        classmethod(lambda cls: SimpleNamespace(headers={})),
+    )
+
+    result = meta_table_models.MetaTable.execute_operation(
+        {
+            "operation": "upsert",
+            "statement": {
+                "sql": (
+                    "INSERT INTO public.asset (uid) VALUES (%(uid)s::UUID) "
+                    "ON CONFLICT (uid) DO UPDATE SET uid = %(uid)s::UUID "
+                    "RETURNING uid"
+                ),
+                "parameters": {"uid": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"},
+            },
+            "scope": {
+                "data_source_uid": "dddddddd-dddd-4ddd-8ddd-dddddddddddd",
+                "tables": [
+                    {
+                        "meta_table_uid": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
+                        "alias": "asset",
+                        "access": "write",
+                    }
+                ],
+            },
+            "limits": {
+                "max_rows": 1,
+                "statement_timeout_ms": 15000,
+            },
+        }
+    )
+
+    assert result["operation"] == "upsert"
+    assert len(calls) == 1
+    assert calls[0]["limits"]["offset"] == 0
 
 
 def test_dynamic_table_data_source_issue_migration_connection_posts_scope(monkeypatch):
@@ -728,6 +904,31 @@ def test_compiled_sql_v1_protocol_is_validated_by_pydantic():
         )
 
 
+def test_compiled_sql_v1_operation_uses_backend_limit_defaults():
+    operation = build_operation(
+        operation="select",
+        sql="SELECT asset.symbol FROM public.asset AS asset",
+        scope={
+            "dataSourceUid": "dddddddd-dddd-4ddd-8ddd-dddddddddddd",
+            "tables": [
+                {
+                    "metaTableUid": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
+                    "alias": "asset",
+                }
+            ]
+        },
+    )
+
+    assert operation.limits.max_rows == 1_000
+    assert operation.limits.offset == 0
+    assert operation.limits.statement_timeout_ms == 15_000
+    assert meta_table_models._payload_json(operation)["limits"] == {
+        "max_rows": 1_000,
+        "offset": 0,
+        "statement_timeout_ms": 15_000,
+    }
+
+
 def test_compiled_sql_v1_scope_resolves_session_data_source(monkeypatch):
     monkeypatch.setattr(
         meta_table_models,
@@ -774,7 +975,7 @@ def test_compiled_sql_v1_serializes_typed_temporal_parameters():
                     "metaTableUid": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
                     "alias": "asset",
                 }
-            ]
+            ],
         },
     )