cloudquery-plugin-sdk 0.1.55__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cloudquery/sdk/__init__.py +0 -0
- cloudquery/sdk/internal/__init__.py +0 -0
- cloudquery/sdk/internal/checkpoint.py +199 -0
- cloudquery/sdk/internal/memdb/__init__.py +1 -0
- cloudquery/sdk/internal/memdb/memdb.py +221 -0
- cloudquery/sdk/internal/servers/__init__.py +0 -0
- cloudquery/sdk/internal/servers/discovery_v1/__init__.py +0 -0
- cloudquery/sdk/internal/servers/discovery_v1/discovery.py +11 -0
- cloudquery/sdk/internal/servers/plugin_v3/__init__.py +1 -0
- cloudquery/sdk/internal/servers/plugin_v3/plugin.py +156 -0
- cloudquery/sdk/message/__init__.py +14 -0
- cloudquery/sdk/message/read.py +6 -0
- cloudquery/sdk/message/sync.py +31 -0
- cloudquery/sdk/message/write.py +24 -0
- cloudquery/sdk/plugin/__init__.py +1 -0
- cloudquery/sdk/plugin/plugin.py +102 -0
- cloudquery/sdk/py.typed +0 -0
- cloudquery/sdk/scalar/__init__.py +15 -0
- cloudquery/sdk/scalar/binary.py +35 -0
- cloudquery/sdk/scalar/bool.py +48 -0
- cloudquery/sdk/scalar/date32.py +39 -0
- cloudquery/sdk/scalar/date64.py +39 -0
- cloudquery/sdk/scalar/float.py +49 -0
- cloudquery/sdk/scalar/int.py +56 -0
- cloudquery/sdk/scalar/json.py +27 -0
- cloudquery/sdk/scalar/list.py +56 -0
- cloudquery/sdk/scalar/scalar.py +30 -0
- cloudquery/sdk/scalar/scalar_factory.py +64 -0
- cloudquery/sdk/scalar/string.py +31 -0
- cloudquery/sdk/scalar/timestamp.py +41 -0
- cloudquery/sdk/scalar/uint.py +58 -0
- cloudquery/sdk/scalar/uuid.py +43 -0
- cloudquery/sdk/scalar/vector.py +43 -0
- cloudquery/sdk/scheduler/__init__.py +3 -0
- cloudquery/sdk/scheduler/scheduler.py +267 -0
- cloudquery/sdk/scheduler/table_resolver.py +48 -0
- cloudquery/sdk/schema/__init__.py +16 -0
- cloudquery/sdk/schema/arrow.py +11 -0
- cloudquery/sdk/schema/column.py +77 -0
- cloudquery/sdk/schema/resource.py +29 -0
- cloudquery/sdk/schema/table.py +332 -0
- cloudquery/sdk/serve/__init__.py +1 -0
- cloudquery/sdk/serve/plugin.py +407 -0
- cloudquery/sdk/stateclient/__init__.py +1 -0
- cloudquery/sdk/stateclient/keys.py +44 -0
- cloudquery/sdk/stateclient/stateclient.py +278 -0
- cloudquery/sdk/transformers/__init__.py +2 -0
- cloudquery/sdk/transformers/openapi.py +49 -0
- cloudquery/sdk/transformers/transformers.py +13 -0
- cloudquery/sdk/types/__init__.py +2 -0
- cloudquery/sdk/types/json.py +26 -0
- cloudquery/sdk/types/uuid.py +28 -0
- cloudquery_plugin_sdk-0.1.55.dist-info/METADATA +50 -0
- cloudquery_plugin_sdk-0.1.55.dist-info/RECORD +57 -0
- cloudquery_plugin_sdk-0.1.55.dist-info/WHEEL +5 -0
- cloudquery_plugin_sdk-0.1.55.dist-info/licenses/LICENSE +353 -0
- cloudquery_plugin_sdk-0.1.55.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import grpc
|
|
4
|
+
import threading
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
from typing import Dict, FrozenSet, Generator, List, Optional, Set, Tuple
|
|
7
|
+
|
|
8
|
+
import pyarrow as pa
|
|
9
|
+
from cloudquery.sdk import schema
|
|
10
|
+
from cloudquery.sdk.internal.checkpoint import Checkpoints, Scope
|
|
11
|
+
from cloudquery.sdk.message import SyncCheckpointMessage
|
|
12
|
+
from cloudquery.sdk.plugin.plugin import BackendOptions
|
|
13
|
+
from cloudquery.plugin_v3 import plugin_pb2, plugin_pb2_grpc, arrow
|
|
14
|
+
from functools import wraps
|
|
15
|
+
|
|
16
|
+
keyColumn = "key"
|
|
17
|
+
valueColumn = "value"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class StateClientBuilder:
|
|
21
|
+
"""
|
|
22
|
+
Provides a `build` method that creates a `ConnectedStateClient` if you pass backend_options,
|
|
23
|
+
or `NoOpStateClient` otherwise.
|
|
24
|
+
|
|
25
|
+
Args:
|
|
26
|
+
backend_options (BackendOptions): which has `connection` & `table_name` strings.
|
|
27
|
+
checkpoints (Checkpoints): optional, the `checkpoints` of the `Scheduler` that syncs.
|
|
28
|
+
When the CLI supports checkpoints, `flush` then sends the state on the sync stream
|
|
29
|
+
instead of writing it to the backend, and the CLI persists it once the destinations
|
|
30
|
+
commit the resources synced before it. Changes made within a table resolver wait only
|
|
31
|
+
for the resources that resolver produced before the flush, and their relations.
|
|
32
|
+
Changes made outside table resolvers wait for nothing, so flush them after the
|
|
33
|
+
resources they summarize are synced, e.g. in the scheduler's post-sync hook.
|
|
34
|
+
|
|
35
|
+
Returns:
|
|
36
|
+
`ConnectedStateClient` or `NoOpStateClient`
|
|
37
|
+
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
@staticmethod
|
|
41
|
+
def build(
|
|
42
|
+
*, backend_options: BackendOptions, checkpoints: Optional[Checkpoints] = None
|
|
43
|
+
) -> StateClient:
|
|
44
|
+
if not backend_options or not backend_options.table_name:
|
|
45
|
+
return NoOpStateClient(backend_options=backend_options)
|
|
46
|
+
|
|
47
|
+
if not backend_options.supports_checkpoints:
|
|
48
|
+
checkpoints = None
|
|
49
|
+
return ConnectedStateClient(
|
|
50
|
+
backend_options=backend_options, checkpoints=checkpoints
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class StateClient(ABC):
|
|
55
|
+
"""
|
|
56
|
+
Abstract class that defines the interface for a state client.
|
|
57
|
+
|
|
58
|
+
It implements all methods except those that require a connection,
|
|
59
|
+
so it is a succinct overview of what a state client does.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
def __init__(
|
|
63
|
+
self,
|
|
64
|
+
*,
|
|
65
|
+
backend_options: BackendOptions,
|
|
66
|
+
checkpoints: Optional[Checkpoints] = None,
|
|
67
|
+
):
|
|
68
|
+
self.mem = {}
|
|
69
|
+
# changed keys, each with the scopes of the table resolvers that changed it since the
|
|
70
|
+
# last flush (None for changes made outside table resolvers)
|
|
71
|
+
self.changes: Dict[str, Set[Optional[Scope]]] = {}
|
|
72
|
+
self.connection = getattr(backend_options, "connection", None)
|
|
73
|
+
self.table = Table(getattr(backend_options, "table_name", None))
|
|
74
|
+
self.checkpoints = checkpoints
|
|
75
|
+
# guards mem and changes, which table resolvers change from several threads
|
|
76
|
+
self._lock = threading.RLock()
|
|
77
|
+
# serializes flushes, so that a key's flushed values reach the state in the order they were set
|
|
78
|
+
self._flush_lock = threading.Lock()
|
|
79
|
+
|
|
80
|
+
self.migrate_state_table()
|
|
81
|
+
self.read_all_state()
|
|
82
|
+
|
|
83
|
+
def get_key(self, key: str) -> Optional[str]:
|
|
84
|
+
with self._lock:
|
|
85
|
+
return self.mem.get(key)
|
|
86
|
+
|
|
87
|
+
def set_key(self, key: str, value: str) -> None:
|
|
88
|
+
scope = self.checkpoints.current_scope() if self.checkpoints else None
|
|
89
|
+
with self._lock:
|
|
90
|
+
self.mem[key] = value
|
|
91
|
+
self.changes.setdefault(key, set()).add(scope)
|
|
92
|
+
|
|
93
|
+
def flush(self):
|
|
94
|
+
"""
|
|
95
|
+
Persists the keys changed since the last flush: writes them to the backend, or, with
|
|
96
|
+
checkpoints, sends them on the sync stream after the resources they cover.
|
|
97
|
+
If that fails, the keys stay changed, so a later flush retries them.
|
|
98
|
+
"""
|
|
99
|
+
with self._flush_lock:
|
|
100
|
+
with self._lock:
|
|
101
|
+
if not self.changes:
|
|
102
|
+
return
|
|
103
|
+
changes, self.changes = self.changes, {}
|
|
104
|
+
values = {key: self.mem[key] for key in changes}
|
|
105
|
+
try:
|
|
106
|
+
if self.checkpoints is None:
|
|
107
|
+
self.write_all_state(iter(values.items()))
|
|
108
|
+
else:
|
|
109
|
+
self._send_checkpoints(changes, values)
|
|
110
|
+
except Exception:
|
|
111
|
+
with self._lock:
|
|
112
|
+
for key, scopes in changes.items():
|
|
113
|
+
self.changes.setdefault(key, set()).update(scopes)
|
|
114
|
+
raise
|
|
115
|
+
|
|
116
|
+
def _send_checkpoints(
|
|
117
|
+
self, changes: Dict[str, Set[Optional[Scope]]], values: Dict[str, str]
|
|
118
|
+
):
|
|
119
|
+
# one checkpoint per set of scopes, so each waits only for the resources of the scopes that changed its keys
|
|
120
|
+
groups: Dict[FrozenSet[Optional[Scope]], List[str]] = {}
|
|
121
|
+
for key, scopes in changes.items():
|
|
122
|
+
groups.setdefault(frozenset(scopes), []).append(key)
|
|
123
|
+
self.checkpoints.schedule(
|
|
124
|
+
[
|
|
125
|
+
(
|
|
126
|
+
scopes,
|
|
127
|
+
keys,
|
|
128
|
+
SyncCheckpointMessage(
|
|
129
|
+
self.table.record([(key, values[key]) for key in keys])
|
|
130
|
+
),
|
|
131
|
+
)
|
|
132
|
+
for scopes, keys in groups.items()
|
|
133
|
+
]
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
@abstractmethod
|
|
137
|
+
def migrate_state_table(self):
|
|
138
|
+
pass
|
|
139
|
+
|
|
140
|
+
@abstractmethod
|
|
141
|
+
def read_all_state(self):
|
|
142
|
+
pass
|
|
143
|
+
|
|
144
|
+
@abstractmethod
|
|
145
|
+
def write_all_state(self, changed_keys: Generator[Tuple[str, str], None, None]):
|
|
146
|
+
pass
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
class NoOpStateClient(StateClient):
|
|
150
|
+
"""
|
|
151
|
+
A state client implementation that does nothing. Used when there is no backend connection.
|
|
152
|
+
"""
|
|
153
|
+
|
|
154
|
+
def get_key(self, key: str) -> Optional[str]:
|
|
155
|
+
pass
|
|
156
|
+
|
|
157
|
+
def set_key(self, key: str, value: str) -> None:
|
|
158
|
+
pass
|
|
159
|
+
|
|
160
|
+
def flush(self):
|
|
161
|
+
pass
|
|
162
|
+
|
|
163
|
+
def migrate_state_table(self):
|
|
164
|
+
pass
|
|
165
|
+
|
|
166
|
+
def read_all_state(self):
|
|
167
|
+
pass
|
|
168
|
+
|
|
169
|
+
def write_all_state(self, changed_keys: Generator[Tuple[str, str], None, None]):
|
|
170
|
+
pass
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def connected(func):
|
|
174
|
+
"""
|
|
175
|
+
Decorator that provides a `backend_plugin` with a gRPC connection to the decorated function.
|
|
176
|
+
"""
|
|
177
|
+
|
|
178
|
+
@wraps(func)
|
|
179
|
+
def wrapper(self, *args, **kwargs):
|
|
180
|
+
with grpc.insecure_channel(self.connection) as channel:
|
|
181
|
+
backend_plugin = plugin_pb2_grpc.PluginStub(channel)
|
|
182
|
+
return func(self, backend_plugin, *args, **kwargs)
|
|
183
|
+
|
|
184
|
+
return wrapper
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
class ConnectedStateClient(StateClient):
|
|
188
|
+
"""
|
|
189
|
+
A state client implementation that connects to a backend plugin via gRPC to read/write state.
|
|
190
|
+
"""
|
|
191
|
+
|
|
192
|
+
@connected
|
|
193
|
+
def migrate_state_table(self, backend_plugin: plugin_pb2_grpc.PluginStub):
|
|
194
|
+
backend_plugin.Write(self._migrate_table_request())
|
|
195
|
+
|
|
196
|
+
@connected
|
|
197
|
+
def read_all_state(self, backend_plugin: plugin_pb2_grpc.PluginStub):
|
|
198
|
+
response = backend_plugin.Read(
|
|
199
|
+
plugin_pb2.Read.Request(table=self.table.bytes())
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
for record in read_response_to_records(response):
|
|
203
|
+
self.mem[record[keyColumn]] = record[valueColumn]
|
|
204
|
+
|
|
205
|
+
@connected
|
|
206
|
+
def write_all_state(
|
|
207
|
+
self,
|
|
208
|
+
backend_plugin: plugin_pb2_grpc.PluginStub,
|
|
209
|
+
changed_keys: Generator[Tuple[str, str], None, None],
|
|
210
|
+
):
|
|
211
|
+
backend_plugin.Write(self._write_request(k, v) for k, v in changed_keys)
|
|
212
|
+
|
|
213
|
+
def _write_request(self, key: str, value: str) -> plugin_pb2.Write.Request:
|
|
214
|
+
record = self.table.record([(key, value)])
|
|
215
|
+
return plugin_pb2.Write.Request(
|
|
216
|
+
insert=plugin_pb2.Write.MessageInsert(record=arrow.record_to_bytes(record))
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
def _migrate_table_request(self):
|
|
220
|
+
yield plugin_pb2.Write.Request(
|
|
221
|
+
migrate_table=plugin_pb2.Write.MessageMigrateTable(table=self.table.bytes())
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def read_response_to_records(response) -> Generator[dict[str, str], None, None]:
|
|
226
|
+
for record in response:
|
|
227
|
+
record_batch = arrow.new_record_from_bytes(record.record)
|
|
228
|
+
for record in recordbatch_to_list_of_maps(record_batch):
|
|
229
|
+
yield record
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def recordbatch_to_list_of_maps(
|
|
233
|
+
record_batch: pa.RecordBatch,
|
|
234
|
+
) -> Generator[dict[str, str], None, None]:
|
|
235
|
+
table = pa.Table.from_batches([record_batch])
|
|
236
|
+
for row in table.to_pandas().to_dict(orient="records"):
|
|
237
|
+
yield row
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
class Table:
|
|
241
|
+
"""
|
|
242
|
+
Represents a state table with two columns: key and value.
|
|
243
|
+
Provides convenience methods for whatever the gRPC requests need.
|
|
244
|
+
"""
|
|
245
|
+
|
|
246
|
+
def __init__(self, name):
|
|
247
|
+
self.name = name
|
|
248
|
+
self._arrow_schema = None
|
|
249
|
+
self._bytes = None
|
|
250
|
+
|
|
251
|
+
if self.name:
|
|
252
|
+
self._arrow_schema = self._state_table_schema().to_arrow_schema()
|
|
253
|
+
self._bytes = arrow.schema_to_bytes(self._arrow_schema)
|
|
254
|
+
|
|
255
|
+
def arrow_schema(self):
|
|
256
|
+
return self._arrow_schema
|
|
257
|
+
|
|
258
|
+
def bytes(self):
|
|
259
|
+
return self._bytes
|
|
260
|
+
|
|
261
|
+
def record(self, rows: List[Tuple[str, str]]) -> pa.RecordBatch:
|
|
262
|
+
"""Returns the state rows as a record of this table, with the table name in its schema metadata."""
|
|
263
|
+
return pa.RecordBatch.from_arrays(
|
|
264
|
+
[
|
|
265
|
+
pa.array([key for key, _ in rows], type=pa.string()),
|
|
266
|
+
pa.array([value for _, value in rows], type=pa.string()),
|
|
267
|
+
],
|
|
268
|
+
schema=self._arrow_schema,
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
def _state_table_schema(self):
|
|
272
|
+
return schema.Table(
|
|
273
|
+
name=self.name,
|
|
274
|
+
columns=[
|
|
275
|
+
schema.Column(name=keyColumn, type=pa.string(), primary_key=True),
|
|
276
|
+
schema.Column(name=valueColumn, type=pa.string()),
|
|
277
|
+
],
|
|
278
|
+
)
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
from typing import Dict, List, Optional
|
|
2
|
+
import pyarrow as pa
|
|
3
|
+
from cloudquery.sdk.types import JSONType
|
|
4
|
+
from cloudquery.sdk.schema import Column
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def oapi_type_to_arrow_type(field) -> pa.DataType:
|
|
8
|
+
oapi_type = field.get("type")
|
|
9
|
+
type_map = {
|
|
10
|
+
"string": pa.string,
|
|
11
|
+
"number": pa.int64,
|
|
12
|
+
"integer": pa.int64,
|
|
13
|
+
"boolean": pa.bool_,
|
|
14
|
+
"array": JSONType,
|
|
15
|
+
"object": JSONType,
|
|
16
|
+
}
|
|
17
|
+
_type = pa.string
|
|
18
|
+
if oapi_type in type_map:
|
|
19
|
+
_type = type_map[oapi_type]
|
|
20
|
+
elif oapi_type is None and "$ref" in field:
|
|
21
|
+
_type = JSONType
|
|
22
|
+
return _type()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def get_column_by_name(columns: List[Column], name: str) -> Optional[Column]:
|
|
26
|
+
for column in columns:
|
|
27
|
+
if column.name == name:
|
|
28
|
+
return column
|
|
29
|
+
return None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def oapi_definition_to_columns(
|
|
33
|
+
definition: Dict, override_columns: Optional[List] = None
|
|
34
|
+
) -> List[Column]:
|
|
35
|
+
columns = []
|
|
36
|
+
for key, value in definition["properties"].items():
|
|
37
|
+
column_type = oapi_type_to_arrow_type(value)
|
|
38
|
+
column = Column(
|
|
39
|
+
name=key, type=column_type, description=value.get("description")
|
|
40
|
+
)
|
|
41
|
+
override_column = get_column_by_name(
|
|
42
|
+
override_columns if override_columns is not None else [], key
|
|
43
|
+
)
|
|
44
|
+
if override_column is not None:
|
|
45
|
+
column.type = override_column.type
|
|
46
|
+
column.primary_key = override_column.primary_key
|
|
47
|
+
column.unique = override_column.unique
|
|
48
|
+
columns.append(column)
|
|
49
|
+
return columns
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from typing import List, Dict
|
|
2
|
+
|
|
3
|
+
import pyarrow as pa
|
|
4
|
+
|
|
5
|
+
from cloudquery.sdk.schema import Table
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def transform_list_of_dict(data: List[Dict], table: Table):
|
|
9
|
+
"""
|
|
10
|
+
Transform json data to arrow record using the given table schema
|
|
11
|
+
"""
|
|
12
|
+
schema = table.to_arrow_schemas()
|
|
13
|
+
return pa.RecordBatch.from_pylist(data, schema)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import pyarrow as pa
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class JSONType(pa.ExtensionType):
|
|
5
|
+
def __init__(self):
|
|
6
|
+
pa.ExtensionType.__init__(self, extension_name="json", storage_type=pa.binary())
|
|
7
|
+
|
|
8
|
+
def __reduce__(self):
|
|
9
|
+
return JSONType, ()
|
|
10
|
+
|
|
11
|
+
def __arrow_ext_serialize__(self):
|
|
12
|
+
# since we don't have a parameterized type, we don't need extra
|
|
13
|
+
# metadata to be deserialized
|
|
14
|
+
return b"json-serialized"
|
|
15
|
+
|
|
16
|
+
def __str__(self):
|
|
17
|
+
return "json"
|
|
18
|
+
|
|
19
|
+
@classmethod
|
|
20
|
+
def __arrow_ext_deserialize__(self, storage_type, serialized):
|
|
21
|
+
# return an instance of this subclass given the serialized
|
|
22
|
+
# metadata.
|
|
23
|
+
return JSONType()
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
pa.register_extension_type(JSONType())
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import pyarrow as pa
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class UUIDType(pa.ExtensionType):
|
|
5
|
+
def __init__(self):
|
|
6
|
+
pa.ExtensionType.__init__(
|
|
7
|
+
self, extension_name="uuid", storage_type=pa.binary(16)
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
def __reduce__(self):
|
|
11
|
+
return UUIDType, ()
|
|
12
|
+
|
|
13
|
+
def __arrow_ext_serialize__(self):
|
|
14
|
+
# since we don't have a parameterized type, we don't need extra
|
|
15
|
+
# metadata to be deserialized
|
|
16
|
+
return b"uuid-serialized"
|
|
17
|
+
|
|
18
|
+
def __str__(self):
|
|
19
|
+
return "uuid"
|
|
20
|
+
|
|
21
|
+
@classmethod
|
|
22
|
+
def __arrow_ext_deserialize__(self, storage_type, serialized):
|
|
23
|
+
# return an instance of this subclass given the serialized
|
|
24
|
+
# metadata.
|
|
25
|
+
return UUIDType()
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
pa.register_extension_type(UUIDType())
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cloudquery-plugin-sdk
|
|
3
|
+
Version: 0.1.55
|
|
4
|
+
Summary: CloudQuery Plugin SDK for Python
|
|
5
|
+
Author-email: CloudQuery LTD <pypi-packages@cloudquery.io>
|
|
6
|
+
License-Expression: MPL-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/cloudquery/plugin-sdk-python
|
|
8
|
+
Classifier: Intended Audience :: Developers
|
|
9
|
+
Classifier: Programming Language :: Python
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Topic :: Internet
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: cloudquery-plugin-pb>=0.0.55
|
|
20
|
+
Requires-Dist: grpcio>=1.78.0
|
|
21
|
+
Requires-Dist: grpcio-tools>=1.78.0
|
|
22
|
+
Requires-Dist: Jinja2>=3.1.6
|
|
23
|
+
Requires-Dist: MarkupSafe>=3.0.3
|
|
24
|
+
Requires-Dist: numpy>=2.4.2
|
|
25
|
+
Requires-Dist: packaging>=26.0
|
|
26
|
+
Requires-Dist: pandas>=3.0.0
|
|
27
|
+
Requires-Dist: protobuf>=6.31.1
|
|
28
|
+
Requires-Dist: pyarrow>=23.0.0
|
|
29
|
+
Requires-Dist: python-dateutil>=2.8.1
|
|
30
|
+
Requires-Dist: pytz>=2025.2
|
|
31
|
+
Requires-Dist: six>=1.17.0
|
|
32
|
+
Requires-Dist: structlog>=25.5.0
|
|
33
|
+
Requires-Dist: tomli>=2.4.0
|
|
34
|
+
Requires-Dist: tzdata>=2025.3
|
|
35
|
+
Dynamic: license-file
|
|
36
|
+
|
|
37
|
+
# CloudQuery Plugin SDK for Python
|
|
38
|
+
|
|
39
|
+
This is the high-level package to use for developing CloudQuery plugins in Python.
|
|
40
|
+
|
|
41
|
+
## Installation
|
|
42
|
+
|
|
43
|
+
```commandline
|
|
44
|
+
uv add cloudquery-plugin-sdk
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Links
|
|
48
|
+
|
|
49
|
+
- [Developer Guide](https://www.cloudquery.io/docs/developers/creating-new-plugin/python-source)
|
|
50
|
+
- [PyPI](https://pypi.org/project/cloudquery-plugin-sdk/)
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
cloudquery/sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
cloudquery/sdk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
+
cloudquery/sdk/internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
+
cloudquery/sdk/internal/checkpoint.py,sha256=yfjmGHrDuDiVxvsZkLtKauJ8pVB0_u_Lw0sFYNgwk3U,7869
|
|
5
|
+
cloudquery/sdk/internal/memdb/__init__.py,sha256=fzRjFUy47sNj4GzDcxPQPg9MwgSlRJK-XXoOxb40qM8,25
|
|
6
|
+
cloudquery/sdk/internal/memdb/memdb.py,sha256=WkJsBhEDrQTaL4PB7j70q66_dyTIifN8LuKX2iBM1gg,6514
|
|
7
|
+
cloudquery/sdk/internal/servers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
cloudquery/sdk/internal/servers/discovery_v1/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
+
cloudquery/sdk/internal/servers/discovery_v1/discovery.py,sha256=iYQpSw1aSXgKK4zcHxIZSvaA9j-IQfDkeHnVur4sogw,360
|
|
10
|
+
cloudquery/sdk/internal/servers/plugin_v3/__init__.py,sha256=5sE_Gdw1MnNOTdlHkd3Von1YEIo1s-jvMxSiq9WyRiw,35
|
|
11
|
+
cloudquery/sdk/internal/servers/plugin_v3/plugin.py,sha256=jQr5F7XiJWvVRztgKETmm9e9CvPsOLfoGX8vmLmD_3A,6131
|
|
12
|
+
cloudquery/sdk/message/__init__.py,sha256=zEiUYcit3TdAMAISYimcLB8z0MOLRBFDDXWSGMLLuGA,287
|
|
13
|
+
cloudquery/sdk/message/read.py,sha256=iBm5-TSTKMPl4YdfhN4viFD11hpmkgP8id7C6IQ2yaI,119
|
|
14
|
+
cloudquery/sdk/message/sync.py,sha256=KQEgFFjok1_xIFArgWJ_s7n25BcIHEknIvunuW10Ry4,746
|
|
15
|
+
cloudquery/sdk/message/write.py,sha256=zVhlElsFsjbfnruPEIlUqX1AfWoXLFbQpm8idFQ5SGk,609
|
|
16
|
+
cloudquery/sdk/plugin/__init__.py,sha256=xksQLxBZNbWE0dhPLSttDDW-ns6oXrEjd03g_mH0Pkg,61
|
|
17
|
+
cloudquery/sdk/plugin/plugin.py,sha256=QuCaOQE7lZC7mD-o0KyZrC-KIG0DpnqnxN1fU9GpkJI,2615
|
|
18
|
+
cloudquery/sdk/scalar/__init__.py,sha256=EEp4c0kNhYZpD_vt84OLnYcFW82HoyrYgKOik53uM0Y,412
|
|
19
|
+
cloudquery/sdk/scalar/binary.py,sha256=8MxVoqjmZCkHyKtjUzsHQKmnpsZxX8UfL6Rd4sOVgBg,980
|
|
20
|
+
cloudquery/sdk/scalar/bool.py,sha256=mtOqB8na2zRKga7OGDYg24XcbIZ6uo24cAe5HdwZfIg,1344
|
|
21
|
+
cloudquery/sdk/scalar/date32.py,sha256=-1leYdI46f3tscbyLqverYzwWbN4BVQMxHSXQKIzFfM,1149
|
|
22
|
+
cloudquery/sdk/scalar/date64.py,sha256=YWBbd4PHq1xJV-rh4S0OWqZNOJ9Z4rT6yNp6MoaTYXI,1149
|
|
23
|
+
cloudquery/sdk/scalar/float.py,sha256=Dgam0NXDSSr9ku_gjE684Nvc7pOyl3bdZ7pKpKgWqso,1532
|
|
24
|
+
cloudquery/sdk/scalar/int.py,sha256=JA7_rtAgXbFUIpTAa4NlXQ695dnrVnh7w5oJ3_okcbY,1777
|
|
25
|
+
cloudquery/sdk/scalar/json.py,sha256=R4uW8WzoEqGX3jwBszmxwWG0OzTyW6q__LGboUZPw-k,709
|
|
26
|
+
cloudquery/sdk/scalar/list.py,sha256=MTVfBbM0OwX3Avy4xSm-IvWV-BtapFzImgamGiafcec,1590
|
|
27
|
+
cloudquery/sdk/scalar/scalar.py,sha256=IL7KO0lj11q9_wfYH9Hy28UVXhL9M1k9_rl7ytWaKGQ,565
|
|
28
|
+
cloudquery/sdk/scalar/scalar_factory.py,sha256=0MnnxU9MA5VlMxd9ok5f-y_FnWKW88tE3rYn32dtkdw,2487
|
|
29
|
+
cloudquery/sdk/scalar/string.py,sha256=bAjw4jsXL8dvKd5x6QmM6hfYvJthPK8We1WWiopNtnM,836
|
|
30
|
+
cloudquery/sdk/scalar/timestamp.py,sha256=CfHiXbGwtoNH3lZrWu-HcYkEHcbk8GnzV0cxLVUM3cg,1244
|
|
31
|
+
cloudquery/sdk/scalar/uint.py,sha256=ZDKOmHk9b9CLjKplHF0N-nHAeQVw-fPckAkueriRSXw,1747
|
|
32
|
+
cloudquery/sdk/scalar/uuid.py,sha256=bINT73Wux4ydJjLuJPq9qduVAmyUtfEhjYryJ-RhY_A,1333
|
|
33
|
+
cloudquery/sdk/scalar/vector.py,sha256=uROcuTyla7rOARn0qhIwPH38StqfaHsaGOClT9W7x10,1248
|
|
34
|
+
cloudquery/sdk/scheduler/__init__.py,sha256=ZliDWvyQ8CYAiwyRNt8-kkmkFyw1kpqnc74JYhASMv8,110
|
|
35
|
+
cloudquery/sdk/scheduler/scheduler.py,sha256=nOvJapLbXJLGDUQwyf_haGNSTnU3ew2uVUEgWcmu0hs,9551
|
|
36
|
+
cloudquery/sdk/scheduler/table_resolver.py,sha256=VyxPZliVESRb_c1cL0nj1Gn2xmxmumlGMG_rCNzBQJY,1489
|
|
37
|
+
cloudquery/sdk/schema/__init__.py,sha256=dU_CQoqDEtNGwx4r431G9ZhLWfEPE2F7holEa2rKT7M,350
|
|
38
|
+
cloudquery/sdk/schema/arrow.py,sha256=Voj9HeEuU4dTB-blsG4yKc-8UFdW_Z5qRcYNeI7wHFg,437
|
|
39
|
+
cloudquery/sdk/schema/column.py,sha256=jTrg19Z7nBfF27s192bxQpOA73EeeGWmBhHf2Jt8XS4,2831
|
|
40
|
+
cloudquery/sdk/schema/resource.py,sha256=R75VuFAGb1GBsRZdZgh5gvHbxhQkyYa_4_EVuFmbFIY,866
|
|
41
|
+
cloudquery/sdk/schema/table.py,sha256=5C5OXGzKpctkI5qHKOlSNq8BLtntS6CH6GoO0ch4dwM,9768
|
|
42
|
+
cloudquery/sdk/serve/__init__.py,sha256=zNN32S3GdnVbJ9ouWwnvlxj0TpS5G_woty8vURoRk6s,34
|
|
43
|
+
cloudquery/sdk/serve/plugin.py,sha256=hClxNg8yJbuc4cYAYn-zAbphldqXv7vaIcm4LMH3rLw,14423
|
|
44
|
+
cloudquery/sdk/stateclient/__init__.py,sha256=GOHcBorhHixMeXgQDS3G_jCjsV4soTZRpQq8GkdEnlM,28
|
|
45
|
+
cloudquery/sdk/stateclient/keys.py,sha256=FmTwzg-2begOVS3_2mlGfCw5xdFi8yb-NRmhulFW4Z8,1669
|
|
46
|
+
cloudquery/sdk/stateclient/stateclient.py,sha256=UgAT6ZzMVjSQNYsSH0UhhcLGrkwrA9KaGFhQteWjtJo,9576
|
|
47
|
+
cloudquery/sdk/transformers/__init__.py,sha256=hGnQNiUbOE2qWrWZZLIIM6KqXvFrz_6WhWGMPdch8jY,97
|
|
48
|
+
cloudquery/sdk/transformers/openapi.py,sha256=5qHM62OtybWvzVqGh1cqh2ZbN9KtKx-zmdSK2JH3pVg,1517
|
|
49
|
+
cloudquery/sdk/transformers/transformers.py,sha256=9USd5VDEMkyk-RfQboMhr9BwC05935MTuM4DZIxhPv4,330
|
|
50
|
+
cloudquery/sdk/types/__init__.py,sha256=VRRTI0mHh_TxkNht9qyr3eouNV71jv1v6z9UDFvLPU8,54
|
|
51
|
+
cloudquery/sdk/types/json.py,sha256=U1aUwf_swF2HZw6Rytdw5HD_hHZMjqnezigu4PnSz4o,694
|
|
52
|
+
cloudquery/sdk/types/uuid.py,sha256=hbC3BGeBXu9hyL4fD8NSo7ezZ-0oOcnvut02QD4TbA0,718
|
|
53
|
+
cloudquery_plugin_sdk-0.1.55.dist-info/licenses/LICENSE,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
|
|
54
|
+
cloudquery_plugin_sdk-0.1.55.dist-info/METADATA,sha256=W0O4-UTPVxD6klJU4tyCKnhiH8Fkm_U1ysc2wcaTgxI,1598
|
|
55
|
+
cloudquery_plugin_sdk-0.1.55.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
56
|
+
cloudquery_plugin_sdk-0.1.55.dist-info/top_level.txt,sha256=D13SSF-LACnBbtgkIVGVOAVT5hqwn2E-v0NGcuyprk4,11
|
|
57
|
+
cloudquery_plugin_sdk-0.1.55.dist-info/RECORD,,
|