cloudquery-plugin-sdk 0.1.55__tar.gz → 0.1.57__tar.gz

{cloudquery_plugin_sdk-0.1.55/cloudquery_plugin_sdk.egg-info → cloudquery_plugin_sdk-0.1.57}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudquery-plugin-sdk
-Version: 0.1.55
+Version: 0.1.57
 Summary: CloudQuery Plugin SDK for Python
 Author-email: CloudQuery LTD <pypi-packages@cloudquery.io>
 License-Expression: MPL-2.0
@@ -16,6 +16,7 @@ Classifier: Topic :: Internet
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: certifi>=2026.10.19
 Requires-Dist: cloudquery-plugin-pb>=0.0.55
 Requires-Dist: grpcio>=1.78.0
 Requires-Dist: grpcio-tools>=1.78.0

{cloudquery_plugin_sdk-0.1.55 → cloudquery_plugin_sdk-0.1.57}/cloudquery/sdk/internal/checkpoint.py

@@ -22,6 +22,14 @@ from typing import Dict, Iterable, Iterator, List, Optional, Set, Tuple
 from cloudquery.sdk.message import SyncCheckpointMessage
 
 
+class SyncNotRunningError(RuntimeError):
+    """
+    Raised when state checkpoints are flushed while their sync is not running, so they cannot be
+    sent: before or after Scheduler.sync, or from a resolver of an ended sync. Flush state that
+    summarizes the whole sync in the scheduler's post-sync hook instead.
+    """
+
+
 class Scope:
     """
     A table resolver call and the relations its resources spawn.
@@ -39,6 +47,10 @@ class Scope:
         self.resolving = True
         # checkpoints waiting for this scope to finish
         self.waiters: List[_Pending] = []
+        # set while a resource is resolved, until it is on the queue and its relations are submitted
+        self.resource_in_flight = False
+        # checkpoints waiting for that resource and its relations
+        self.after_resource: List[_Pending] = []
 
     def finished(self) -> bool:
         return not self.resolving and not self.open_children
@@ -112,6 +124,37 @@ class Checkpoints:
         """Returns the scope of the table resolver running in this context, or None outside table resolvers."""
         return self._scope.get()
 
+    @contextmanager
+    def resource(self, scope: Scope) -> Iterator[None]:
+        """
+        Marks a resource of scope as in flight until it is on the queue and its relations are
+        submitted, so that checkpoints flushed meanwhile, e.g. from its resource hooks, follow
+        the resource and its relations.
+        """
+        with self._lock:
+            scope.resource_in_flight = True
+        try:
+            yield
+        finally:
+            with self._lock:
+                scope.resource_in_flight = False
+                ready: List[_Pending] = []
+                for pending in scope.after_resource:
+                    pending.blockers -= 1
+                    self._wait_for_children(pending, scope)
+                    if pending.blockers == 0:
+                        ready.append(pending)
+                scope.after_resource = []
+                self._send(ready)
+
+    def superseded(self, scope: Optional[Scope]) -> bool:
+        """Returns True if scope belongs to a sync that ended while another sync runs now."""
+        return (
+            scope is not None
+            and self._sync is not None
+            and scope.sync is not self._sync
+        )
+
     def resolver_finished(self, scope: Scope) -> None:
         """
         Marks the resolver of scope as returned, once every resource it produced is on the queue.
@@ -143,17 +186,17 @@ class Checkpoints:
         wait for nothing. Schedules all of them or, if it raises, none of them.
 
         Raises:
-            RuntimeError: if no sync is running, as the checkpoints could not be sent.
+            SyncNotRunningError: if the sync of the checkpoints is not running.
         """
         with self._lock:
             if self._res is None:
-                raise RuntimeError(
+                raise SyncNotRunningError(
                     "state checkpoints can only be flushed while the scheduler syncs, "
                     "e.g. from a table resolver or the post-sync hook"
                 )
             for scopes, _, _ in checkpoints:
-                if any(s is not None and s.sync is not self._sync for s in scopes):
-                    raise RuntimeError(
+                if any(self.superseded(s) for s in scopes):
+                    raise SyncNotRunningError(
                         "state checkpoints of a table resolver can only be flushed during its own sync"
                     )
             for scopes, keys, msg in checkpoints:
@@ -167,11 +210,11 @@ class Checkpoints:
     ) -> None:
         pending = _Pending(msg, keys, self._sync)
         for scope in {s for s in scopes if s is not None}:
-            # the scope's own resources are on the queue already: resolvers put each
-            # resource on it before they resume, so only its relations can be in flight
-            for child in scope.open_children:
-                child.waiters.append(pending)
+            if scope.resource_in_flight:
+                scope.after_resource.append(pending)
                 pending.blockers += 1
+            else:
+                self._wait_for_children(pending, scope)
         earlier = set()
         for key in keys:
             previous = self._unsent.get(key)
@@ -183,6 +226,16 @@ class Checkpoints:
         if pending.blockers == 0:
             self._send([pending])
 
+    @staticmethod
+    def _wait_for_children(pending: _Pending, scope: Scope) -> None:
+        """
+        Makes pending wait for the relations of scope's resources. Its resources are on the queue
+        already, as none is in flight. Called with the lock held.
+        """
+        for child in scope.open_children:
+            child.waiters.append(pending)
+            pending.blockers += 1
+
     def _send(self, ready: List[_Pending]) -> None:
         """Puts ready checkpoints on the queue, then the ones that only waited for them. Called with the lock held."""
         while ready:

{cloudquery_plugin_sdk-0.1.55 → cloudquery_plugin_sdk-0.1.57}/cloudquery/sdk/scheduler/scheduler.py

@@ -1,4 +1,5 @@
 import queue
+import threading
 from concurrent import futures
 from typing import List, Generator, Any, Optional
 
@@ -19,9 +20,22 @@ QUEUE_PER_WORKER = 100
 
 
 class ThreadPoolExecutorWithQueueSizeLimit(futures.ThreadPoolExecutor):
-    def __init__(self, maxsize, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._work_queue = queue.Queue(maxsize=maxsize)
+    def __init__(self, maxsize: int, max_workers: int, *args, **kwargs):
+        super().__init__(max_workers, *args, **kwargs)
+        self.maxsize = maxsize
+        # submit() puts on the work queue while holding a lock that every pool shares, so a
+        # full bounded queue would block the other pools' workers too: limit it before submit
+        self._slots = threading.BoundedSemaphore(maxsize + max_workers)
+
+    def submit(self, fn, /, *args, **kwargs):
+        self._slots.acquire()
+        try:
+            future = super().submit(fn, *args, **kwargs)
+        except BaseException:
+            self._slots.release()
+            raise
+        future.add_done_callback(lambda _: self._slots.release())
+        return future
 
 
 class TableResolverStarted:
@@ -61,7 +75,7 @@ class Scheduler:
         )
         self._pools: List[ThreadPoolExecutorWithQueueSizeLimit] = []
         current_depth_concurrency = concurrency
-        current_depth_queue_size = queue_size
+        current_depth_queue_size = self._queue_size
         for _ in range(max_depth + 1):
             self._pools.append(
                 ThreadPoolExecutorWithQueueSizeLimit(
@@ -134,37 +148,38 @@ class Scheduler:
                     depth=depth,
                 )
             for item in resolver.resolve(client, parent_item):
-                try:
-                    resource = self.resolve_resource(
-                        resolver, client, parent_item, item
-                    )
-                except Exception as e:
-                    self._logger.error(
-                        "failed to resolve resource",
-                        client_id=client.id(),
-                        table=resolver.table.name,
-                        depth=depth,
-                        exc_info=e,
-                    )
-                    self._logger.debug(
-                        "details about resource that failed to resolve",
-                        client_id=client.id(),
-                        table=resolver.table.name,
-                        resource=item,
-                    )
-                    continue
-                res.put(SyncInsertMessage(resource.to_arrow_record()))
-                for child_resolvers in resolver.child_resolvers:
-                    res.put(TableResolverStarted())
-                    self._pools[depth + 1].submit(
-                        self.resolve_table,
-                        child_resolvers,
-                        depth + 1,
-                        client,
-                        resource,
-                        res,
-                        self.checkpoints.new_scope(scope),
-                    )
+                with self.checkpoints.resource(scope):
+                    try:
+                        resource = self.resolve_resource(
+                            resolver, client, parent_item, item
+                        )
+                    except Exception as e:
+                        self._logger.error(
+                            "failed to resolve resource",
+                            client_id=client.id(),
+                            table=resolver.table.name,
+                            depth=depth,
+                            exc_info=e,
+                        )
+                        self._logger.debug(
+                            "details about resource that failed to resolve",
+                            client_id=client.id(),
+                            table=resolver.table.name,
+                            resource=item,
+                        )
+                        continue
+                    res.put(SyncInsertMessage(resource.to_arrow_record()))
+                    for child_resolvers in resolver.child_resolvers:
+                        res.put(TableResolverStarted())
+                        self._pools[depth + 1].submit(
+                            self.resolve_table,
+                            child_resolvers,
+                            depth + 1,
+                            client,
+                            resource,
+                            res,
+                            self.checkpoints.new_scope(scope),
+                        )
                 total_resources += 1
             if depth == 0:
                 self._logger.info(

cloudquery_plugin_sdk-0.1.57/cloudquery/sdk/stateclient/__init__.py

@@ -0,0 +1,2 @@
+from cloudquery.sdk.internal.checkpoint import SyncNotRunningError
+from .keys import state_key

{cloudquery_plugin_sdk-0.1.55 → cloudquery_plugin_sdk-0.1.57}/cloudquery/sdk/stateclient/stateclient.py

@@ -79,14 +79,22 @@ class StateClient(ABC):
 
         self.migrate_state_table()
         self.read_all_state()
+        self._flushed = dict(self.mem)
 
     def get_key(self, key: str) -> Optional[str]:
         with self._lock:
+            self._forget_changes_of_ended_syncs(key)
             return self.mem.get(key)
 
     def set_key(self, key: str, value: str) -> None:
         scope = self.checkpoints.current_scope() if self.checkpoints else None
+        if self.checkpoints and self.checkpoints.superseded(scope):
+            return
         with self._lock:
+            # with checkpoints, the value also covers the caller's resources, so later checkpoints
+            # of the key must still follow them even if it is unchanged
+            if self.checkpoints is None and self.mem.get(key) == value:
+                return
             self.mem[key] = value
             self.changes.setdefault(key, set()).add(scope)
 
@@ -95,9 +103,17 @@ class StateClient(ABC):
         Persists the keys changed since the last flush: writes them to the backend, or, with
         checkpoints, sends them on the sync stream after the resources they cover.
         If that fails, the keys stay changed, so a later flush retries them.
+        Changes made by table resolvers of an ended sync are dropped once another sync runs,
+        as the resources they cover may never have been committed.
+
+        Raises:
+            SyncNotRunningError: with checkpoints, if the sync is not running, e.g. once
+                Scheduler.sync returned. Flush in the scheduler's post-sync hook instead.
         """
         with self._flush_lock:
             with self._lock:
+                for key in list(self.changes):
+                    self._forget_changes_of_ended_syncs(key)
                 if not self.changes:
                     return
                 changes, self.changes = self.changes, {}
@@ -112,6 +128,21 @@ class StateClient(ABC):
                     for key, scopes in changes.items():
                         self.changes.setdefault(key, set()).update(scopes)
                 raise
+            with self._lock:
+                self._flushed.update(values)
+
+    def _forget_changes_of_ended_syncs(self, key: str) -> None:
+        if self.checkpoints is None or key not in self.changes:
+            return
+        scopes = {s for s in self.changes[key] if not self.checkpoints.superseded(s)}
+        if scopes:
+            self.changes[key] = scopes
+            return
+        del self.changes[key]
+        if key in self._flushed:
+            self.mem[key] = self._flushed[key]
+        else:
+            self.mem.pop(key, None)
 
     def _send_checkpoints(
         self, changes: Dict[str, Set[Optional[Scope]]], values: Dict[str, str]

{cloudquery_plugin_sdk-0.1.55 → cloudquery_plugin_sdk-0.1.57/cloudquery_plugin_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudquery-plugin-sdk
-Version: 0.1.55
+Version: 0.1.57
 Summary: CloudQuery Plugin SDK for Python
 Author-email: CloudQuery LTD <pypi-packages@cloudquery.io>
 License-Expression: MPL-2.0
@@ -16,6 +16,7 @@ Classifier: Topic :: Internet
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: certifi>=2026.10.19
 Requires-Dist: cloudquery-plugin-pb>=0.0.55
 Requires-Dist: grpcio>=1.78.0
 Requires-Dist: grpcio-tools>=1.78.0

{cloudquery_plugin_sdk-0.1.55 → cloudquery_plugin_sdk-0.1.57}/cloudquery_plugin_sdk.egg-info/requires.txt

@@ -1,3 +1,4 @@
+certifi>=2026.10.19
 cloudquery-plugin-pb>=0.0.55
 grpcio>=1.78.0
 grpcio-tools>=1.78.0

{cloudquery_plugin_sdk-0.1.55 → cloudquery_plugin_sdk-0.1.57}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "cloudquery-plugin-sdk"
-version = "0.1.55"
+version = "0.1.57"
 description = "CloudQuery Plugin SDK for Python"
 readme = "README.md"
 license = "MPL-2.0"
@@ -23,6 +23,7 @@ classifiers = [
     "Topic :: Internet",
 ]
 dependencies = [
+    "certifi>=2026.10.19",
     "cloudquery-plugin-pb>=0.0.55",
     "grpcio>=1.78.0",
     "grpcio-tools>=1.78.0",
@@ -61,4 +62,4 @@ python_files = "tests/*.py"
 
 [tool.uv]
 exclude-newer = "1 week"
-exclude-newer-package = { cloudquery-plugin-pb = false }
+exclude-newer-package = { cloudquery-plugin-pb = false, certifi = false }

cloudquery_plugin_sdk-0.1.55/cloudquery/sdk/stateclient/__init__.py

@@ -1 +0,0 @@
-from .keys import state_key