cloudquery-plugin-sdk 0.1.33__py2.py3-none-any.whl → 0.1.34__py2.py3-none-any.whl

cloudquery/sdk/internal/memdb/memdb.py

@@ -5,6 +5,7 @@ from cloudquery.sdk import message
 from cloudquery.sdk import schema
 from typing import List, Generator, Dict
 import pyarrow as pa
+from cloudquery.sdk.schema.table import Table
 from cloudquery.sdk.types import JSONType
 from dataclasses import dataclass, field
 
@@ -109,5 +110,9 @@ class MemDB(plugin.Plugin):
             else:
                 raise NotImplementedError(f"Unknown message type {type(msg)}")
 
+    def read(self, table: Table) -> Generator[message.ReadMessage, None, None]:
+        for table, record in self._db.items():
+            yield message.ReadMessage(record)
+
     def close(self) -> None:
         self._db = {}

cloudquery/sdk/internal/servers/plugin_v3/plugin.py

@@ -81,8 +81,14 @@ class PluginServicer(plugin_pb2_grpc.PluginServicer):
                 # unknown sync message type
                 raise NotImplementedError()
 
-    def Read(self, request, context):
-        raise NotImplementedError()
+    def Read(
+        self, request: plugin_pb2.Read.Request, context
+    ) -> Generator[plugin_pb2.Read.Response, None, None]:
+        schema = arrow.new_schema_from_bytes(request.table)
+        table = Table.from_arrow_schema(schema)
+        for msg in self._plugin.read(table):
+            buf = arrow.record_to_bytes(msg.record)
+            yield plugin_pb2.Read.Response(record=buf)
 
     def Write(
         self, request_iterator: Generator[plugin_pb2.Write.Request, None, None], context
@@ -93,7 +99,9 @@ class PluginServicer(plugin_pb2_grpc.PluginServicer):
                 if field == "migrate_table":
                     sc = arrow.new_schema_from_bytes(msg.migrate_table.table)
                     table = Table.from_arrow_schema(sc)
-                    yield WriteMigrateTableMessage(table=table)
+                    yield WriteMigrateTableMessage(
+                        table=table, migrate_force=msg.migrate_table.migrate_force
+                    )
                 elif field == "insert":
                     yield WriteInsertMessage(
                         record=arrow.new_record_from_bytes(msg.insert.record)

cloudquery/sdk/message/__init__.py

@@ -5,3 +5,4 @@ from .write import (
     WriteMigrateTableMessage,
     WriteDeleteStale,
 )
+from .read import ReadMessage

cloudquery/sdk/message/read.py

@@ -0,0 +1,6 @@
+import pyarrow as pa
+
+
+class ReadMessage:
+    def __init__(self, record: pa.RecordBatch):
+        self.record = record

cloudquery/sdk/message/write.py

@@ -12,8 +12,9 @@ class WriteInsertMessage(WriteMessage):
 
 
 class WriteMigrateTableMessage(WriteMessage):
-    def __init__(self, table: Table):
+    def __init__(self, table: Table, migrate_force: bool):
         self.table = table
+        self.migrate_force = migrate_force
 
 
 class WriteDeleteStale(WriteMessage):

cloudquery/sdk/plugin/plugin.py

@@ -93,5 +93,8 @@ class Plugin:
     def write(self, writer: Generator[message.WriteMessage, None, None]) -> None:
         raise NotImplementedError()
 
+    def read(self, table: Table) -> Generator[message.ReadMessage, None, None]:
+        raise NotImplementedError()
+
     def close(self) -> None:
         raise NotImplementedError()

cloudquery/sdk/schema/__init__.py

@@ -1,5 +1,16 @@
 from .column import Column
-from .table import Table, tables_to_arrow_schemas, filter_dfs
+from .table import (
+    Table,
+    tables_to_arrow_schemas,
+    filter_dfs,
+    TableColumnChangeType,
+    TableColumnChange,
+    TableColumnChangeType,
+    get_table_changes,
+    get_table_column,
+    flatten_tables_recursive,
+    flatten_tables,
+)
 from .resource import Resource
 
 # from .table_resolver import TableReso

cloudquery/sdk/schema/column.py

@@ -55,7 +55,9 @@ class Column:
                 arrow.METADATA_TRUE if self.incremental_key else arrow.METADATA_FALSE
             ),
         }
-        return pa.field(self.name, self.type, metadata=metadata)
+        return pa.field(
+            self.name, self.type, metadata=metadata, nullable=not self.not_null
+        )
 
     @staticmethod
     def from_arrow_field(field: pa.Field) -> Column:

cloudquery/sdk/schema/table.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
 import copy
+from enum import IntEnum
 import fnmatch
-from typing import List
+from typing import List, Optional
 
 import pyarrow as pa
 
@@ -10,6 +11,10 @@ from cloudquery.sdk.schema import arrow
 from .column import Column
 
 
+CQ_SYNC_TIME_COLUMN = "cq_sync_time"
+CQ_SOURCE_NAME_COLUMN = "cq_source_name"
+
+
 class Client:
     pass
 
@@ -192,9 +197,137 @@ def _filter_dfs_impl(t, parent_matched, include, exclude, skip_dependent_tables)
     return None
 
 
-def flatten_tables(tables: List[Table]) -> List[Table]:
-    flattened: List[Table] = []
+class TableColumnChangeType:
+    ADD = 1
+    REMOVE = 2
+    REMOVE_UNIQUE_CONSTRAINT = 3
+
+
+class TableColumnChange:
+    def __init__(
+        self,
+        type: TableColumnChangeType,
+        column_name: str,
+        current: Optional[Column],
+        previous: Optional[Column],
+    ):
+        self.type = type
+        self.column_name = column_name
+        self.current = current
+        self.previous = previous
+
+
+class TableColumnChangeType(IntEnum):
+    UNKNOWN = 0
+    ADD = 1
+    UPDATE = 2
+    REMOVE = 3
+    REMOVE_UNIQUE_CONSTRAINT = 4
+    MOVE_TO_CQ_ONLY = 5
+
+
+def get_table_changes(new: Table, old: Table) -> List[TableColumnChange]:
+    changes = []
+
+    # Special case: Moving from individual PKs to singular PK on _cq_id
+    new_pks = new.primary_keys
+    if (
+        len(new_pks) == 1
+        and new_pks[0] == "CqIDColumn"
+        and get_table_column(old, "CqIDColumn") is None
+        and len(old.primary_keys) > 0
+    ):
+        changes.append(
+            TableColumnChange(
+                type=TableColumnChangeType.MOVE_TO_CQ_ONLY,
+            )
+        )
+
+    for c in new.columns:
+        other_column = get_table_column(old, c.name)
+        # A column was added to the table definition
+        if other_column is None:
+            changes.append(
+                TableColumnChange(
+                    type=TableColumnChangeType.ADD,
+                    column_name=c.name,
+                    current=c,
+                    previous=None,
+                )
+            )
+            continue
+
+        # Column type or options (e.g. PK, Not Null) changed in the new table definition
+        if (
+            c.type != other_column.type
+            or c.not_null != other_column.not_null
+            or c.primary_key != other_column.primary_key
+        ):
+            changes.append(
+                TableColumnChange(
+                    type=TableColumnChangeType.UPDATE,
+                    column_name=c.name,
+                    current=c,
+                    previous=other_column,
+                )
+            )
+
+        # Unique constraint was removed
+        if not c.unique and other_column.unique:
+            changes.append(
+                TableColumnChange(
+                    type=TableColumnChangeType.REMOVE_UNIQUE_CONSTRAINT,
+                    column_name=c.name,
+                    current=c,
+                    previous=other_column,
+                )
+            )
+
+    # A column was removed from the table definition
+    for c in old.columns:
+        if get_table_column(new, c.name) is None:
+            changes.append(
+                TableColumnChange(
+                    type=TableColumnChangeType.REMOVE,
+                    column_name=c.name,
+                    current=None,
+                    previous=c,
+                )
+            )
+
+    return changes
+
+
+def get_table_column(table: Table, column_name: str) -> Optional[Column]:
+    for c in table.columns:
+        if c.name == column_name:
+            return c
+    return None
+
+
+def flatten_tables_recursive(original_tables: List[Table]) -> List[Table]:
+    tables = []
+    for table in original_tables:
+        table_copy = Table(
+            name=table.name,
+            columns=table.columns,
+            relations=table.relations,
+            title=table.title,
+            description=table.description,
+            is_incremental=table.is_incremental,
+            parent=table.parent,
+        )
+        tables.append(table_copy)
+        tables.extend(flatten_tables_recursive(table.relations))
+    return tables
+
+
+def flatten_tables(original_tables: List[Table]) -> List[Table]:
+    tables = flatten_tables_recursive(original_tables)
+    seen = set()
+    deduped = []
     for table in tables:
-        flattened.append(table)
-        flattened.extend(flatten_tables(table.relations))
-    return flattened
+        if table.name not in seen:
+            deduped.append(table)
+            seen.add(table.name)
+    return deduped

cloudquery/sdk/types/json.py

@@ -21,3 +21,6 @@ class JSONType(pa.ExtensionType):
         # return an instance of this subclass given the serialized
         # metadata.
         return JSONType()
+
+
+pa.register_extension_type(JSONType())

cloudquery/sdk/types/uuid.py

@@ -23,3 +23,6 @@ class UUIDType(pa.ExtensionType):
         # return an instance of this subclass given the serialized
         # metadata.
         return UUIDType()
+
+
+pa.register_extension_type(UUIDType())

{cloudquery_plugin_sdk-0.1.33.dist-info → cloudquery_plugin_sdk-0.1.34.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloudquery-plugin-sdk
-Version: 0.1.33
+Version: 0.1.34
 Summary: CloudQuery Plugin SDK for Python
 Home-page: https://github.com/cloudquery/plugin-sdk-python
 Author: CloudQuery LTD

{cloudquery_plugin_sdk-0.1.33.dist-info → cloudquery_plugin_sdk-0.1.34.dist-info}/RECORD

@@ -1,19 +1,20 @@
-cloudquery_plugin_sdk-0.1.33-py3.11-nspkg.pth,sha256=DPwVEJKI3l-PsvgSiQi4zcKjh50W15YbcAr9gUpExc4,487
+cloudquery_plugin_sdk-0.1.34-py3.11-nspkg.pth,sha256=DPwVEJKI3l-PsvgSiQi4zcKjh50W15YbcAr9gUpExc4,487
 cloudquery/sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloudquery/sdk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloudquery/sdk/internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloudquery/sdk/internal/memdb/__init__.py,sha256=fzRjFUy47sNj4GzDcxPQPg9MwgSlRJK-XXoOxb40qM8,25
-cloudquery/sdk/internal/memdb/memdb.py,sha256=MMpfKJVwYS5eXqanZh6vqyvgXSg0Vr9MtLbUgkapsxo,3980
+cloudquery/sdk/internal/memdb/memdb.py,sha256=YLr5F85ZTwbw5w9_NHvDvV7IfyOvWUzthGu64UK-1-M,4200
 cloudquery/sdk/internal/servers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloudquery/sdk/internal/servers/discovery_v1/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloudquery/sdk/internal/servers/discovery_v1/discovery.py,sha256=iYQpSw1aSXgKK4zcHxIZSvaA9j-IQfDkeHnVur4sogw,360
 cloudquery/sdk/internal/servers/plugin_v3/__init__.py,sha256=5sE_Gdw1MnNOTdlHkd3Von1YEIo1s-jvMxSiq9WyRiw,35
-cloudquery/sdk/internal/servers/plugin_v3/plugin.py,sha256=AXKUwcPPWvmrtOHlhq1y30g6y9gDzmxutx0eWi_YJoA,4597
-cloudquery/sdk/message/__init__.py,sha256=KfRu1ZmLz4r-9qgOsXqgAZYetj2_MKjKAJ2HRgARi2w,191
+cloudquery/sdk/internal/servers/plugin_v3/plugin.py,sha256=sxu9MNRpiAvDxjnGMoruJTYenuh1dDLByTsQJjJjyJo,5004
+cloudquery/sdk/message/__init__.py,sha256=iJ9PqhjjG4zsWgd2O5hbNHD6sZP0ergFUvaF14I5na4,221
+cloudquery/sdk/message/read.py,sha256=iBm5-TSTKMPl4YdfhN4viFD11hpmkgP8id7C6IQ2yaI,119
 cloudquery/sdk/message/sync.py,sha256=O2deyqCbZ84s5DvYgEozN1JMpWmlhXVPUOatM4fjq3w,283
-cloudquery/sdk/message/write.py,sha256=i30d2nC_YGcGlei0qBd3yHatbJ2C72ZF6QYWURdysYM,545
+cloudquery/sdk/message/write.py,sha256=zVhlElsFsjbfnruPEIlUqX1AfWoXLFbQpm8idFQ5SGk,609
 cloudquery/sdk/plugin/__init__.py,sha256=xksQLxBZNbWE0dhPLSttDDW-ns6oXrEjd03g_mH0Pkg,61
-cloudquery/sdk/plugin/plugin.py,sha256=1coTWQmQmt0oRsuqvwYkkqdvw2wgE7eHIW3qP1Cznu8,2382
+cloudquery/sdk/plugin/plugin.py,sha256=KPkdhNqjeKlFWtcpfSnJ5mGEWMQYL2zqvql7fcsoVQE,2499
 cloudquery/sdk/scalar/__init__.py,sha256=EEp4c0kNhYZpD_vt84OLnYcFW82HoyrYgKOik53uM0Y,412
 cloudquery/sdk/scalar/binary.py,sha256=8MxVoqjmZCkHyKtjUzsHQKmnpsZxX8UfL6Rd4sOVgBg,980
 cloudquery/sdk/scalar/bool.py,sha256=mtOqB8na2zRKga7OGDYg24XcbIZ6uo24cAe5HdwZfIg,1344
@@ -33,11 +34,11 @@ cloudquery/sdk/scalar/vector.py,sha256=uROcuTyla7rOARn0qhIwPH38StqfaHsaGOClT9W7x
 cloudquery/sdk/scheduler/__init__.py,sha256=ZliDWvyQ8CYAiwyRNt8-kkmkFyw1kpqnc74JYhASMv8,110
 cloudquery/sdk/scheduler/scheduler.py,sha256=1QGQg9vIWeTI2B25YS9pWew3QekczpetAx8zSDXuHCU,8101
 cloudquery/sdk/scheduler/table_resolver.py,sha256=VyxPZliVESRb_c1cL0nj1Gn2xmxmumlGMG_rCNzBQJY,1489
-cloudquery/sdk/schema/__init__.py,sha256=6y65eQdg7OhBevUAlLzVmuhOXe0dd5Gr28hzygMmBBw,161
+cloudquery/sdk/schema/__init__.py,sha256=dU_CQoqDEtNGwx4r431G9ZhLWfEPE2F7holEa2rKT7M,350
 cloudquery/sdk/schema/arrow.py,sha256=Voj9HeEuU4dTB-blsG4yKc-8UFdW_Z5qRcYNeI7wHFg,437
-cloudquery/sdk/schema/column.py,sha256=lsvscBXBP4QJFTi0xdzmJJUN245llZW4nmw0buOBe9k,2781
+cloudquery/sdk/schema/column.py,sha256=jTrg19Z7nBfF27s192bxQpOA73EeeGWmBhHf2Jt8XS4,2831
 cloudquery/sdk/schema/resource.py,sha256=R75VuFAGb1GBsRZdZgh5gvHbxhQkyYa_4_EVuFmbFIY,866
-cloudquery/sdk/schema/table.py,sha256=4jiuJkt_R96FAiQou7yusBjDiEOBX3Bq7dOHm3TX9h8,6033
+cloudquery/sdk/schema/table.py,sha256=56ILjDOmcoLe3jw5jQuu-XKKIIF10NGQw7cnphQLq44,9769
 cloudquery/sdk/serve/__init__.py,sha256=zNN32S3GdnVbJ9ouWwnvlxj0TpS5G_woty8vURoRk6s,34
 cloudquery/sdk/serve/plugin.py,sha256=q5XjxG-zkl_Y407UtOmDwus4gFVzYqO654axXinOzg0,14423
 cloudquery/sdk/stateclient/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -46,11 +47,11 @@ cloudquery/sdk/transformers/__init__.py,sha256=hGnQNiUbOE2qWrWZZLIIM6KqXvFrz_6Wh
 cloudquery/sdk/transformers/openapi.py,sha256=5qHM62OtybWvzVqGh1cqh2ZbN9KtKx-zmdSK2JH3pVg,1517
 cloudquery/sdk/transformers/transformers.py,sha256=9USd5VDEMkyk-RfQboMhr9BwC05935MTuM4DZIxhPv4,330
 cloudquery/sdk/types/__init__.py,sha256=VRRTI0mHh_TxkNht9qyr3eouNV71jv1v6z9UDFvLPU8,54
-cloudquery/sdk/types/json.py,sha256=d1EYb8-C74U71IUajHgbetD4wTS7fIsh04qY84mq2-U,653
-cloudquery/sdk/types/uuid.py,sha256=Xhax4Pyfwd-rxnBSBZLt9k2Mlx2FVCyUOcKWcRiHA0U,677
-cloudquery_plugin_sdk-0.1.33.dist-info/LICENSE,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
-cloudquery_plugin_sdk-0.1.33.dist-info/METADATA,sha256=TfDbsj5iE4DfHQDLOvAyG5e3_BvjbHmjzHRGHChA49o,1846
-cloudquery_plugin_sdk-0.1.33.dist-info/WHEEL,sha256=M4n4zmFKzQZk4mLCcycNIzIXO7YPKE_b5Cw4PnhHEg8,109
-cloudquery_plugin_sdk-0.1.33.dist-info/namespace_packages.txt,sha256=D13SSF-LACnBbtgkIVGVOAVT5hqwn2E-v0NGcuyprk4,11
-cloudquery_plugin_sdk-0.1.33.dist-info/top_level.txt,sha256=D13SSF-LACnBbtgkIVGVOAVT5hqwn2E-v0NGcuyprk4,11
-cloudquery_plugin_sdk-0.1.33.dist-info/RECORD,,
+cloudquery/sdk/types/json.py,sha256=U1aUwf_swF2HZw6Rytdw5HD_hHZMjqnezigu4PnSz4o,694
+cloudquery/sdk/types/uuid.py,sha256=hbC3BGeBXu9hyL4fD8NSo7ezZ-0oOcnvut02QD4TbA0,718
+cloudquery_plugin_sdk-0.1.34.dist-info/LICENSE,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
+cloudquery_plugin_sdk-0.1.34.dist-info/METADATA,sha256=YP41tdAtZ56HY2VpVyLIz6-j_ArNxvFY9MdZA3V7Rg8,1846
+cloudquery_plugin_sdk-0.1.34.dist-info/WHEEL,sha256=GUeE9LxUgRABPG7YM0jCNs9cBsAIx0YAkzCB88PMLgc,109
+cloudquery_plugin_sdk-0.1.34.dist-info/namespace_packages.txt,sha256=D13SSF-LACnBbtgkIVGVOAVT5hqwn2E-v0NGcuyprk4,11
+cloudquery_plugin_sdk-0.1.34.dist-info/top_level.txt,sha256=D13SSF-LACnBbtgkIVGVOAVT5hqwn2E-v0NGcuyprk4,11
+cloudquery_plugin_sdk-0.1.34.dist-info/RECORD,,

{cloudquery_plugin_sdk-0.1.33.dist-info → cloudquery_plugin_sdk-0.1.34.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (72.2.0)
+Generator: setuptools (73.0.1)
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any