cloudquery-plugin-sdk 0.1.8__tar.gz → 0.1.9__tar.gz

{cloudquery-plugin-sdk-0.1.8 → cloudquery-plugin-sdk-0.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloudquery-plugin-sdk
-Version: 0.1.8
+Version: 0.1.9
 Summary: CloudQuery Plugin SDK for Python
 Home-page: https://github.com/cloudquery/plugin-sdk-python
 Author: CloudQuery LTD
@@ -20,24 +20,24 @@ Classifier: Operating System :: OS Independent
 Classifier: Topic :: Internet
 Requires-Python: >=3.7
 Requires-Dist: cloudquery-plugin-pb==0.0.20
-Requires-Dist: exceptiongroup==1.1.3
-Requires-Dist: black==23.10.1
-Requires-Dist: grpcio==1.59.2
-Requires-Dist: grpcio-tools==1.59.2
+Requires-Dist: exceptiongroup==1.2.0
+Requires-Dist: black==23.12.0
+Requires-Dist: grpcio==1.60.0
+Requires-Dist: grpcio-tools==1.60.0
 Requires-Dist: iniconfig==2.0.0
 Requires-Dist: Jinja2==3.1.2
 Requires-Dist: MarkupSafe==2.1.3
-Requires-Dist: numpy==1.26.1
+Requires-Dist: numpy==1.26.2
 Requires-Dist: packaging==23.2
-Requires-Dist: pandas==2.1.2
+Requires-Dist: pandas==2.1.4
 Requires-Dist: pluggy==1.3.0
-Requires-Dist: protobuf==4.24.4
+Requires-Dist: protobuf==4.25.1
 Requires-Dist: pyarrow==14.0.1
 Requires-Dist: pytest==7.4.3
 Requires-Dist: python-dateutil==2.8.2
 Requires-Dist: pytz==2023.3.post1
 Requires-Dist: six==1.16.0
-Requires-Dist: structlog==23.1.0
+Requires-Dist: structlog==23.2.0
 Requires-Dist: tomli==2.0.1
 Requires-Dist: tzdata==2023.3
 

cloudquery-plugin-sdk-0.1.9/cloudquery/sdk/internal/memdb/memdb.py

@@ -0,0 +1,99 @@
+from cloudquery.sdk import plugin
+from cloudquery.sdk import message
+from cloudquery.sdk import schema
+from typing import List, Generator, Dict
+import pyarrow as pa
+from cloudquery.sdk.types import JSONType
+
+NAME = "memdb"
+VERSION = "development"
+
+
+class MemDB(plugin.Plugin):
+    def __init__(self) -> None:
+        super().__init__(
+            NAME, VERSION, opts=plugin.plugin.Options(team="cloudquery", kind="source")
+        )
+        self._db: Dict[str, pa.RecordBatch] = {}
+        self._tables: Dict[str, schema.Table] = {
+            "table_1": schema.Table(
+                name="table_1",
+                columns=[
+                    schema.Column(
+                        name="name",
+                        type=pa.string(),
+                        primary_key=True,
+                        not_null=True,
+                        unique=True,
+                    ),
+                    schema.Column(
+                        name="id",
+                        type=pa.string(),
+                        primary_key=True,
+                        not_null=True,
+                        unique=True,
+                        incremental_key=True,
+                    ),
+                ],
+                title="Table 1",
+                description="Test Table 1",
+                is_incremental=True,
+                relations=[
+                    schema.Table(
+                        name="table_1_relation_1",
+                        columns=[
+                            schema.Column(
+                                name="name",
+                                type=pa.string(),
+                                primary_key=True,
+                                not_null=True,
+                                unique=True,
+                            ),
+                            schema.Column(name="data", type=JSONType()),
+                        ],
+                        title="Table 1 Relation 1",
+                        description="Test Table 1 Relation 1",
+                    )
+                ],
+            ),
+            "table_2": schema.Table(
+                name="table_2",
+                columns=[
+                    schema.Column(
+                        name="name",
+                        type=pa.string(),
+                        primary_key=True,
+                        not_null=True,
+                        unique=True,
+                    ),
+                    schema.Column(name="id", type=pa.string()),
+                ],
+                title="Table 2",
+                description="Test Table 2",
+            ),
+        }
+
+    def get_tables(self, options: plugin.TableOptions = None) -> List[plugin.Table]:
+        tables = list(self._tables.values())
+        return schema.filter_dfs(tables, options.tables, options.skip_tables)
+
+    def sync(
+        self, options: plugin.SyncOptions
+    ) -> Generator[message.SyncMessage, None, None]:
+        for table, record in self._db.items():
+            yield message.SyncInsertMessage(record)
+
+    def write(self, writer: Generator[message.WriteMessage, None, None]) -> None:
+        for msg in writer:
+            if isinstance(msg, message.WriteMigrateTableMessage):
+                if msg.table.name not in self._db:
+                    self._db[msg.table.name] = msg.table
+                    self._tables[msg.table.name] = msg.table
+            elif isinstance(msg, message.WriteInsertMessage):
+                table = schema.Table.from_arrow_schema(msg.record.schema)
+                self._db[table.name] = msg.record
+            else:
+                raise NotImplementedError(f"Unknown message type {type(msg)}")
+
+    def close(self) -> None:
+        self._db = {}

{cloudquery-plugin-sdk-0.1.8 → cloudquery-plugin-sdk-0.1.9}/cloudquery/sdk/plugin/plugin.py

@@ -29,10 +29,32 @@ class SyncOptions:
     backend_options: BackendOptions = None
 
 
+@dataclass
+class BuildTarget:
+    os: str = None
+    arch: str = None
+
+
+@dataclass
+class Options:
+    dockerfile: str = None
+    build_targets: List[BuildTarget] = None
+    team: str = None
+    kind: str = None
+
+
 class Plugin:
-    def __init__(self, name: str, version: str) -> None:
+    def __init__(self, name: str, version: str, opts: Options = None) -> None:
         self._name = name
         self._version = version
+        self._opts = Options() if opts is None else opts
+        if self._opts.dockerfile is None:
+            self._opts.dockerfile = "Dockerfile"
+        if self._opts.build_targets is None:
+            self._opts.build_targets = [
+                BuildTarget("linux", "amd64"),
+                BuildTarget("linux", "arm64"),
+            ]
 
     def init(self, spec: bytes, no_connection: bool = False) -> None:
         pass
@@ -46,6 +68,18 @@ class Plugin:
     def version(self) -> str:
         return self._version
 
+    def team(self) -> str:
+        return self._opts.team
+
+    def kind(self) -> str:
+        return self._opts.kind
+
+    def dockerfile(self) -> str:
+        return self._opts.dockerfile
+
+    def build_targets(self) -> List[BuildTarget]:
+        return self._opts.build_targets
+
     def get_tables(self, options: TableOptions) -> List[Table]:
         raise NotImplementedError()
 

{cloudquery-plugin-sdk-0.1.8 → cloudquery-plugin-sdk-0.1.9}/cloudquery/sdk/schema/table.py

@@ -151,6 +151,8 @@ def filter_dfs_func(tt: List[Table], include, exclude, skip_dependent_tables: bo
     filtered_tables = []
     for t in tt:
         filtered_table = copy.deepcopy(t)
+        for r in filtered_table.relations:
+            r.parent = filtered_table
         filtered_table = _filter_dfs_impl(
             filtered_table, False, include, exclude, skip_dependent_tables
         )

cloudquery-plugin-sdk-0.1.9/cloudquery/sdk/serve/plugin.py

@@ -0,0 +1,400 @@
+import argparse
+import hashlib
+import json
+import logging
+import os
+import shutil
+import subprocess
+import tarfile
+from concurrent import futures
+from pathlib import Path
+
+import grpc
+import structlog
+import sys
+from cloudquery.discovery_v1 import discovery_pb2_grpc
+from cloudquery.plugin_v3 import plugin_pb2_grpc
+from structlog import wrap_logger
+from cloudquery.sdk import plugin
+
+
+from cloudquery.sdk.internal.servers.discovery_v1.discovery import DiscoveryServicer
+from cloudquery.sdk.internal.servers.plugin_v3 import PluginServicer
+from cloudquery.sdk.plugin.plugin import Plugin
+from cloudquery.sdk.schema import table
+
+_IS_WINDOWS = sys.platform == "win32"
+
+try:
+    import colorama
+except ImportError:
+    colorama = None
+
+if _IS_WINDOWS:  # pragma: no cover
+    # On Windows, use colors by default only if Colorama is installed.
+    _has_colors = colorama is not None
+else:
+    # On other OSes, use colors by default.
+    _has_colors = True
+
+
+def get_logger(args):
+    log_level_map = {
+        "debug": logging.DEBUG,
+        "info": logging.INFO,
+        "warning": logging.WARNING,
+        "error": logging.ERROR,
+        "critical": logging.CRITICAL,
+    }
+
+    logging.basicConfig(
+        format="%(message)s",
+        stream=sys.stdout,
+        level=log_level_map.get(args.log_level.lower(), logging.INFO),
+    )
+
+    processors = [
+        structlog.contextvars.merge_contextvars,
+        structlog.processors.add_log_level,
+        structlog.processors.StackInfoRenderer(),
+        structlog.dev.set_exc_info,
+        structlog.stdlib.filter_by_level,
+        structlog.processors.TimeStamper(fmt="%Y-%m-%dT%H:%M:%SZ", utc=True),
+    ]
+    if args.log_format == "text":
+        processors.append(
+            structlog.dev.ConsoleRenderer(
+                colors=os.environ.get("NO_COLOR", "") == ""
+                and (
+                    os.environ.get("FORCE_COLOR", "") != ""
+                    or (
+                        _has_colors
+                        and sys.stdout is not None
+                        and hasattr(sys.stdout, "isatty")
+                        and sys.stdout.isatty()
+                    )
+                )
+            )
+        )
+    else:
+        processors.append(structlog.processors.JSONRenderer())
+
+    log = wrap_logger(logging.getLogger(), processors=processors)
+    return log
+
+
+def calc_sha256_checksum(filename: str):
+    with open(filename, "rb") as f:
+        file_hash = hashlib.sha256()
+        while chunk := f.read(32768):
+            file_hash.update(chunk)
+        return file_hash.hexdigest()
+
+
+class PluginCommand:
+    def __init__(self, plugin: Plugin):
+        self._plugin = plugin
+
+    def run(self, args):
+        parser = argparse.ArgumentParser()
+        subparsers = parser.add_subparsers(dest="command", required=True)
+
+        self._register_serve_command(subparsers)
+        self._register_package_command(subparsers)
+
+        parsed_args = parser.parse_args(args)
+
+        if parsed_args.command == "serve":
+            self._serve(parsed_args)
+        elif parsed_args.command == "package":
+            self._package(parsed_args)
+        else:
+            parser.print_help()
+            sys.exit(1)
+
+    def _register_serve_command(self, subparsers):
+        serve_parser = subparsers.add_parser("serve", help="Start plugin server")
+        serve_parser.add_argument(
+            "--log-format",
+            type=str,
+            default="text",
+            choices=["text", "json"],
+            help="logging format",
+        )
+        serve_parser.add_argument(
+            "--log-level",
+            type=str,
+            default="info",
+            choices=["trace", "debug", "info", "warn", "error"],
+            help="log level",
+        )
+        # ignored for now
+        serve_parser.add_argument(
+            "--no-sentry",
+            action="store_true",
+            help="disable sentry (placeholder for future use)",
+        )
+        # ignored for now
+        serve_parser.add_argument(
+            "--otel-endpoint",
+            type=str,
+            default="",
+            help="Open Telemetry HTTP collector endpoint (placeholder for future use)",
+        )
+        # ignored for now
+        serve_parser.add_argument(
+            "--otel-endpoint-insecure",
+            type=str,
+            default="",
+            help="Open Telemetry HTTP collector endpoint (for development only) (placeholder for future use)",
+        )
+        serve_parser.add_argument(
+            "--address",
+            type=str,
+            default="localhost:7777",
+            help="address to serve on. can be tcp: 'localhost:7777' or unix socket: '/tmp/plugin.rpc.sock'",
+        )
+        serve_parser.add_argument(
+            "--network",
+            type=str,
+            default="tcp",
+            choices=["tcp", "unix"],
+            help="network to serve on. can be tcp or unix",
+        )
+
+    def _register_package_command(self, subparsers):
+        package_parser = subparsers.add_parser(
+            "package", help="Package the plugin as a Docker image"
+        )
+        package_parser.add_argument(
+            "version", help="version to tag the Docker image with"
+        )
+        package_parser.add_argument("plugin-directory")
+        package_parser.add_argument(
+            "--log-format",
+            type=str,
+            default="text",
+            choices=["text", "json"],
+            help="logging format",
+        )
+        package_parser.add_argument(
+            "--log-level",
+            type=str,
+            default="info",
+            choices=["trace", "debug", "info", "warn", "error"],
+            help="log level",
+        )
+        package_parser.add_argument(
+            "-D",
+            "--dist-dir",
+            type=str,
+            help="dist directory to output the built plugin. (default: <plugin_directory>/dist)",
+        )
+        package_parser.add_argument(
+            "--docs-dir",
+            type=str,
+            help="docs directory containing markdown files to copy to the dist directory. (default: <plugin_directory>/docs)",
+        )
+        package_parser.add_argument(
+            "-m",
+            "--message",
+            type=str,
+            required=True,
+            help="message that summarizes what is new or changed in this version. Use @<file> to read from file. Supports markdown.",
+        )
+
+    def _package(self, args):
+        logger = get_logger(args)
+        self._plugin.set_logger(logger)
+
+        def _is_empty(val):
+            return val == None or len(val) == 0
+
+        if _is_empty(self._plugin.name()):
+            raise Exception("plugin name is required")
+        if _is_empty(self._plugin.team()):
+            raise Exception("plugin team is required")
+        if _is_empty(self._plugin.kind()):
+            raise Exception("plugin kind is required")
+        if _is_empty(self._plugin.dockerfile()):
+            raise Exception("plugin dockerfile is required")
+        if _is_empty(self._plugin.build_targets()):
+            raise Exception("at least one build target is required")
+
+        plugin_directory, version, message = (
+            getattr(args, "plugin-directory"),
+            getattr(args, "version"),
+            getattr(args, "message"),
+        )
+        dist_dir = (
+            "%s/dist" % plugin_directory if args.dist_dir == None else args.dist_dir
+        )
+        docs_dir = (
+            "%s/docs" % plugin_directory if args.docs_dir == None else args.docs_dir
+        )
+        Path(dist_dir).mkdir(0o755, exist_ok=True, parents=True)
+
+        self._copy_docs(logger, docs_dir, dist_dir)
+        self._write_tables_json(logger, dist_dir)
+        supported_targets = self._build_dockerfile(
+            logger, plugin_directory, dist_dir, version
+        )
+        self._write_package_json(logger, dist_dir, message, version, supported_targets)
+        logger.info("Done packaging plugin to '%s'" % dist_dir)
+
+    def _write_package_json(self, logger, dist_dir, message, version, supportedTargets):
+        package_json_path = "%s/package.json" % dist_dir
+        logger.info("Writing package.json to '%s'" % package_json_path)
+        content = {
+            "schema_version": 1,
+            "name": self._plugin.name(),
+            "team": self._plugin.team(),
+            "kind": self._plugin.kind(),
+            "version": version,
+            "message": message,
+            "protocols": [3],
+            "supported_targets": supportedTargets,
+            "package_type": "docker",
+        }
+        with open("%s/package.json" % dist_dir, "w") as f:
+            f.write(json.dumps(content, indent=2))
+
+    def _copy_docs(self, logger, docs_dir, dist_dir):
+        # check is docs_dir exists
+        if not os.path.isdir(docs_dir):
+            raise Exception("docs directory '%s' does not exist" % docs_dir)
+
+        output_docs_dir = "%s/docs" % dist_dir
+        logger.info("Copying docs from '%s' to '%s'" % (docs_dir, output_docs_dir))
+        shutil.copytree(docs_dir, output_docs_dir, dirs_exist_ok=True)
+
+    def _write_tables_json(self, logger, dist_dir):
+        if self._plugin.kind() != "source":
+            return
+
+        tables_json_output_path = "%s/tables.json" % dist_dir
+        logger.info("Writing tables to '%s'" % tables_json_output_path)
+        self._plugin.init(spec=b"", no_connection=True)
+        tables = self._plugin.get_tables(
+            options=plugin.plugin.TableOptions(
+                tables=["*"], skip_tables=[], skip_dependent_tables=False
+            )
+        )
+        flattened_tables = table.flatten_tables(tables)
+
+        def column_to_json(column: table.Column):
+            return {
+                "name": column.name,
+                "type": str(column.type),
+                "description": column.description,
+                "incremental_key": column.incremental_key,
+                "primary_key": column.primary_key,
+                "not_null": column.not_null,
+                "unique": column.unique,
+            }
+
+        def table_to_json(table: table.Table):
+            return {
+                "name": table.name,
+                "title": table.title,
+                "description": table.description,
+                "is_incremental": table.is_incremental,
+                "parent": table.parent.name if table.parent else "",
+                "relations": list(map(lambda r: r.name, table.relations)),
+                "columns": list(map(column_to_json, table.columns)),
+            }
+
+        tables_json = list(map(table_to_json, flattened_tables))
+        with open(tables_json_output_path, "w") as f:
+            f.write(json.dumps(tables_json))
+        logger.info(
+            "Wrote %d tables to '%s'" % (len(tables_json), tables_json_output_path)
+        )
+
+    def _build_dockerfile(self, logger, plugin_dir, dist_dir, version):
+        dockerfile_path = "%s/%s" % (plugin_dir, self._plugin.dockerfile())
+        if not os.path.isfile(dockerfile_path):
+            raise Exception("Dockerfile '%s' does not exist" % dockerfile_path)
+
+        def run_docker_cmd(cmd, plugin_dir):
+            result = subprocess.run(cmd, capture_output=True, cwd=plugin_dir)
+            if result.returncode != 0:
+                err = (
+                    ""
+                    if result.stderr is None
+                    else result.stderr.decode("ascii").strip()
+                )
+                raise ChildProcessError("Unable to run Docker command: %s" % err)
+
+        def build_target(target: plugin.plugin.BuildTarget):
+            image_repository = "registry.cloudquery.io/%s/%s-%s" % (
+                self._plugin.team(),
+                self._plugin.kind(),
+                self._plugin.name(),
+            )
+            image_tag = "%s:%s-%s-%s" % (
+                image_repository,
+                version,
+                target.os,
+                target.arch,
+            )
+            image_tar = "plugin-%s-%s-%s-%s.tar" % (
+                self._plugin.name(),
+                version,
+                target.os,
+                target.arch,
+            )
+            image_path = "%s/%s" % (dist_dir, image_tar)
+            logger.info("Building docker image %s" % image_tag)
+            docker_build_arguments = [
+                "docker",
+                "buildx",
+                "build",
+                "-t",
+                image_tag,
+                "--platform",
+                "%s/%s" % (target.os, target.arch),
+                "-f",
+                dockerfile_path,
+                ".",
+                "--progress",
+                "plain",
+                "--load",
+            ]
+            logger.debug(
+                "Running command 'docker %s'" % " ".join(docker_build_arguments)
+            )
+            run_docker_cmd(docker_build_arguments, plugin_dir)
+            logger.debug("Saving docker image '%s' to '%s'" % (image_tag, image_path))
+            docker_save_arguments = ["docker", "save", "-o", image_path, image_tag]
+            logger.debug("Running command 'docker %s'", " ".join(docker_save_arguments))
+            run_docker_cmd(docker_save_arguments, plugin_dir)
+            return {
+                "os": target.os,
+                "arch": target.arch,
+                "path": image_tar,
+                "checksum": calc_sha256_checksum(image_path),
+                "docker_image_tag": image_tag,
+            }
+
+        logger.info("Building %d targets" % len(self._plugin.build_targets()))
+        supported_targets = list(map(build_target, self._plugin.build_targets()))
+        return supported_targets
+
+    def _serve(self, args):
+        logger = get_logger(args)
+        self._plugin.set_logger(logger)
+        self._server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+        discovery_pb2_grpc.add_DiscoveryServicer_to_server(
+            DiscoveryServicer([3]), self._server
+        )
+        plugin_pb2_grpc.add_PluginServicer_to_server(
+            PluginServicer(self._plugin, logger), self._server
+        )
+        self._server.add_insecure_port(args.address)
+        logger.info("Starting server", address=args.address)
+        self._server.start()
+        self._server.wait_for_termination()
+
+    def stop(self):
+        self._server.stop(5)

{cloudquery-plugin-sdk-0.1.8 → cloudquery-plugin-sdk-0.1.9}/cloudquery/sdk/types/json.py

@@ -13,6 +13,9 @@ class JSONType(pa.ExtensionType):
         # metadata to be deserialized
         return b"json-serialized"
 
+    def __str__(self):
+        return "json"
+
     @classmethod
     def __arrow_ext_deserialize__(self, storage_type, serialized):
         # return an instance of this subclass given the serialized

{cloudquery-plugin-sdk-0.1.8 → cloudquery-plugin-sdk-0.1.9}/cloudquery/sdk/types/uuid.py

@@ -15,6 +15,9 @@ class UUIDType(pa.ExtensionType):
         # metadata to be deserialized
         return b"uuid-serialized"
 
+    def __str__(self):
+        return "uuid"
+
     @classmethod
     def __arrow_ext_deserialize__(self, storage_type, serialized):
         # return an instance of this subclass given the serialized

{cloudquery-plugin-sdk-0.1.8 → cloudquery-plugin-sdk-0.1.9}/cloudquery_plugin_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloudquery-plugin-sdk
-Version: 0.1.8
+Version: 0.1.9
 Summary: CloudQuery Plugin SDK for Python
 Home-page: https://github.com/cloudquery/plugin-sdk-python
 Author: CloudQuery LTD
@@ -20,24 +20,24 @@ Classifier: Operating System :: OS Independent
 Classifier: Topic :: Internet
 Requires-Python: >=3.7
 Requires-Dist: cloudquery-plugin-pb==0.0.20
-Requires-Dist: exceptiongroup==1.1.3
-Requires-Dist: black==23.10.1
-Requires-Dist: grpcio==1.59.2
-Requires-Dist: grpcio-tools==1.59.2
+Requires-Dist: exceptiongroup==1.2.0
+Requires-Dist: black==23.12.0
+Requires-Dist: grpcio==1.60.0
+Requires-Dist: grpcio-tools==1.60.0
 Requires-Dist: iniconfig==2.0.0
 Requires-Dist: Jinja2==3.1.2
 Requires-Dist: MarkupSafe==2.1.3
-Requires-Dist: numpy==1.26.1
+Requires-Dist: numpy==1.26.2
 Requires-Dist: packaging==23.2
-Requires-Dist: pandas==2.1.2
+Requires-Dist: pandas==2.1.4
 Requires-Dist: pluggy==1.3.0
-Requires-Dist: protobuf==4.24.4
+Requires-Dist: protobuf==4.25.1
 Requires-Dist: pyarrow==14.0.1
 Requires-Dist: pytest==7.4.3
 Requires-Dist: python-dateutil==2.8.2
 Requires-Dist: pytz==2023.3.post1
 Requires-Dist: six==1.16.0
-Requires-Dist: structlog==23.1.0
+Requires-Dist: structlog==23.2.0
 Requires-Dist: tomli==2.0.1
 Requires-Dist: tzdata==2023.3
 

{cloudquery-plugin-sdk-0.1.8 → cloudquery-plugin-sdk-0.1.9}/cloudquery_plugin_sdk.egg-info/requires.txt

@@ -1,21 +1,21 @@
 cloudquery-plugin-pb==0.0.20
-exceptiongroup==1.1.3
-black==23.10.1
-grpcio==1.59.2
-grpcio-tools==1.59.2
+exceptiongroup==1.2.0
+black==23.12.0
+grpcio==1.60.0
+grpcio-tools==1.60.0
 iniconfig==2.0.0
 Jinja2==3.1.2
 MarkupSafe==2.1.3
-numpy==1.26.1
+numpy==1.26.2
 packaging==23.2
-pandas==2.1.2
+pandas==2.1.4
 pluggy==1.3.0
-protobuf==4.24.4
+protobuf==4.25.1
 pyarrow==14.0.1
 pytest==7.4.3
 python-dateutil==2.8.2
 pytz==2023.3.post1
 six==1.16.0
-structlog==23.1.0
+structlog==23.2.0
 tomli==2.0.1
 tzdata==2023.3

{cloudquery-plugin-sdk-0.1.8 → cloudquery-plugin-sdk-0.1.9}/setup.py

@@ -11,24 +11,24 @@ description = "CloudQuery Plugin SDK for Python"
 
 dependencies = [
     "cloudquery-plugin-pb==0.0.20",
-    "exceptiongroup==1.1.3",
-    "black==23.10.1",
-    "grpcio==1.59.2",
-    "grpcio-tools==1.59.2",
+    "exceptiongroup==1.2.0",
+    "black==23.12.0",
+    "grpcio==1.60.0",
+    "grpcio-tools==1.60.0",
     "iniconfig==2.0.0",
     "Jinja2==3.1.2",
     "MarkupSafe==2.1.3",
-    "numpy==1.26.1",
+    "numpy==1.26.2",
     "packaging==23.2",
-    "pandas==2.1.2",
+    "pandas==2.1.4",
     "pluggy==1.3.0",
-    "protobuf==4.24.4",
+    "protobuf==4.25.1",
     "pyarrow==14.0.1",
     "pytest==7.4.3",
     "python-dateutil==2.8.2",
     "pytz==2023.3.post1",
     "six==1.16.0",
-    "structlog==23.1.0",
+    "structlog==23.2.0",
     "tomli==2.0.1",
     "tzdata==2023.3",
 ]
@@ -53,7 +53,7 @@ packages = [
 ]
 setuptools.setup(
     name=name,
-    version="0.1.8",
+    version="0.1.9",
     description=description,
     long_description=long_description,
     author="CloudQuery LTD",

cloudquery-plugin-sdk-0.1.8/cloudquery/sdk/internal/memdb/memdb.py

@@ -1,40 +0,0 @@
-from cloudquery.sdk import plugin
-from cloudquery.sdk import message
-from cloudquery.sdk import schema
-from typing import List, Generator, Dict
-import pyarrow as pa
-
-NAME = "memdb"
-VERSION = "development"
-
-
-class MemDB(plugin.Plugin):
-    def __init__(self) -> None:
-        super().__init__(NAME, VERSION)
-        self._db: Dict[str, pa.RecordBatch] = {}
-        self._tables: Dict[str, schema.Table] = {}
-
-    def get_tables(self, options: plugin.TableOptions = None) -> List[plugin.Table]:
-        tables = list(self._tables.values())
-        return schema.filter_dfs(tables, options.tables, options.skip_tables)
-
-    def sync(
-        self, options: plugin.SyncOptions
-    ) -> Generator[message.SyncMessage, None, None]:
-        for table, record in self._db.items():
-            yield message.SyncInsertMessage(record)
-
-    def write(self, writer: Generator[message.WriteMessage, None, None]) -> None:
-        for msg in writer:
-            if isinstance(msg, message.WriteMigrateTableMessage):
-                if msg.table.name not in self._db:
-                    self._db[msg.table.name] = msg.table
-                    self._tables[msg.table.name] = msg.table
-            elif isinstance(msg, message.WriteInsertMessage):
-                table = schema.Table.from_arrow_schema(msg.record.schema)
-                self._db[table.name] = msg.record
-            else:
-                raise NotImplementedError(f"Unknown message type {type(msg)}")
-
-    def close(self) -> None:
-        self._db = {}

cloudquery-plugin-sdk-0.1.8/cloudquery/sdk/serve/plugin.py

@@ -1,160 +0,0 @@
-import argparse
-import logging
-import os
-from concurrent import futures
-
-import grpc
-import structlog
-import sys
-from cloudquery.discovery_v1 import discovery_pb2_grpc
-from cloudquery.plugin_v3 import plugin_pb2_grpc
-from structlog import wrap_logger
-
-from cloudquery.sdk.internal.servers.discovery_v1.discovery import DiscoveryServicer
-from cloudquery.sdk.internal.servers.plugin_v3 import PluginServicer
-from cloudquery.sdk.plugin.plugin import Plugin
-
-_IS_WINDOWS = sys.platform == "win32"
-
-try:
-    import colorama
-except ImportError:
-    colorama = None
-
-if _IS_WINDOWS:  # pragma: no cover
-    # On Windows, use colors by default only if Colorama is installed.
-    _has_colors = colorama is not None
-else:
-    # On other OSes, use colors by default.
-    _has_colors = True
-
-
-def get_logger(args):
-    log_level_map = {
-        "debug": logging.DEBUG,
-        "info": logging.INFO,
-        "warning": logging.WARNING,
-        "error": logging.ERROR,
-        "critical": logging.CRITICAL,
-    }
-
-    logging.basicConfig(
-        format="%(message)s",
-        stream=sys.stdout,
-        level=log_level_map.get(args.log_level.lower(), logging.INFO),
-    )
-
-    processors = [
-        structlog.contextvars.merge_contextvars,
-        structlog.processors.add_log_level,
-        structlog.processors.StackInfoRenderer(),
-        structlog.dev.set_exc_info,
-        structlog.stdlib.filter_by_level,
-        structlog.processors.TimeStamper(fmt="%Y-%m-%dT%H:%M:%SZ", utc=True),
-    ]
-    if args.log_format == "text":
-        processors.append(
-            structlog.dev.ConsoleRenderer(
-                colors=os.environ.get("NO_COLOR", "") == ""
-                and (
-                    os.environ.get("FORCE_COLOR", "") != ""
-                    or (
-                        _has_colors
-                        and sys.stdout is not None
-                        and hasattr(sys.stdout, "isatty")
-                        and sys.stdout.isatty()
-                    )
-                )
-            )
-        )
-    else:
-        processors.append(structlog.processors.JSONRenderer())
-
-    log = wrap_logger(logging.getLogger(), processors=processors)
-    return log
-
-
-class PluginCommand:
-    def __init__(self, plugin: Plugin):
-        self._plugin = plugin
-
-    def run(self, args):
-        parser = argparse.ArgumentParser()
-        subparsers = parser.add_subparsers(dest="command", required=True)
-
-        serve_parser = subparsers.add_parser("serve", help="Start plugin server")
-        serve_parser.add_argument(
-            "--log-format",
-            type=str,
-            default="text",
-            choices=["text", "json"],
-            help="logging format",
-        )
-        serve_parser.add_argument(
-            "--log-level",
-            type=str,
-            default="info",
-            choices=["trace", "debug", "info", "warn", "error"],
-            help="log level",
-        )
-
-        # ignored for now
-        serve_parser.add_argument(
-            "--no-sentry",
-            action="store_true",
-            help="disable sentry (placeholder for future use)",
-        )
-        # ignored for now
-        serve_parser.add_argument(
-            "--otel-endpoint",
-            type=str,
-            default="",
-            help="Open Telemetry HTTP collector endpoint (placeholder for future use)",
-        )
-        # ignored for now
-        serve_parser.add_argument(
-            "--otel-endpoint-insecure",
-            type=str,
-            default="",
-            help="Open Telemetry HTTP collector endpoint (for development only) (placeholder for future use)",
-        )
-
-        serve_parser.add_argument(
-            "--address",
-            type=str,
-            default="localhost:7777",
-            help="address to serve on. can be tcp: 'localhost:7777' or unix socket: '/tmp/plugin.rpc.sock'",
-        )
-        serve_parser.add_argument(
-            "--network",
-            type=str,
-            default="tcp",
-            choices=["tcp", "unix"],
-            help="network to serve on. can be tcp or unix",
-        )
-
-        parsed_args = parser.parse_args(args)
-
-        if parsed_args.command == "serve":
-            self._serve(parsed_args)
-        else:
-            parser.print_help()
-            sys.exit(1)
-
-    def _serve(self, args):
-        logger = get_logger(args)
-        self._plugin.set_logger(logger)
-        self._server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
-        discovery_pb2_grpc.add_DiscoveryServicer_to_server(
-            DiscoveryServicer([3]), self._server
-        )
-        plugin_pb2_grpc.add_PluginServicer_to_server(
-            PluginServicer(self._plugin, logger), self._server
-        )
-        self._server.add_insecure_port(args.address)
-        logger.info("Starting server", address=args.address)
-        self._server.start()
-        self._server.wait_for_termination()
-
-    def stop(self):
-        self._server.stop(5)