airbyte-source-github 2.1.35__tar.gz → 2.1.36rc1__tar.gz

{airbyte_source_github-2.1.35 → airbyte_source_github-2.1.36rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-github
-Version: 2.1.35
+Version: 2.1.36rc1
 Summary: Source implementation for GitHub.
 Home-page: https://airbyte.com
 License: ELv2

{airbyte_source_github-2.1.35 → airbyte_source_github-2.1.36rc1}/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "2.1.35"
+version = "2.1.36-rc.1"
 name = "airbyte-source-github"
 description = "Source implementation for GitHub."
 authors = [ "Airbyte <contact@airbyte.io>",]

airbyte_source_github-2.1.36rc1/source_github/manifest.yaml

@@ -0,0 +1,6 @@
+version: "7.12.0"
+type: DeclarativeSource
+check:
+  type: CheckStream
+  stream_names: []
+streams: []

airbyte_source_github-2.1.36rc1/source_github/run.py

@@ -0,0 +1,63 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import sys
+import traceback
+from datetime import datetime
+from typing import List
+
+from orjson import orjson
+
+from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch, logger
+from airbyte_cdk.exception_handler import init_uncaught_exception_handler
+from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type
+from source_github import SourceGithub
+from source_github.config_migrations import MigrateBranch, MigrateRepository
+
+
+def _run_migrations(args: List[str], source: SourceGithub) -> None:
+    MigrateRepository.migrate(args, source)
+    MigrateBranch.migrate(args, source)
+
+
+def _get_source(args: List[str]):
+    catalog_path = AirbyteEntrypoint.extract_catalog(args)
+    config_path = AirbyteEntrypoint.extract_config(args)
+    state_path = AirbyteEntrypoint.extract_state(args)
+    try:
+        return SourceGithub(
+            SourceGithub.read_catalog(catalog_path) if catalog_path else None,
+            SourceGithub.read_config(config_path) if config_path else None,
+            SourceGithub.read_state(state_path) if state_path else None,
+        )
+    except Exception as error:
+        print(
+            orjson.dumps(
+                AirbyteMessageSerializer.dump(
+                    AirbyteMessage(
+                        type=Type.TRACE,
+                        trace=AirbyteTraceMessage(
+                            type=TraceType.ERROR,
+                            emitted_at=int(datetime.now().timestamp() * 1000),
+                            error=AirbyteErrorTraceMessage(
+                                message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}",
+                                stack_trace=traceback.format_exc(),
+                            ),
+                        ),
+                    )
+                )
+            ).decode()
+        )
+        return None
+
+
+def run() -> None:
+    init_uncaught_exception_handler(logger)
+    _args = sys.argv[1:]
+    # Run config migrations before constructing the source so that
+    # self._config inside YamlDeclarativeSource sees the migrated config.
+    _run_migrations(_args, SourceGithub())
+    source = _get_source(_args)
+    if source:
+        launch(source, _args)

{airbyte_source_github-2.1.35 → airbyte_source_github-2.1.36rc1}/source_github/source.py

@@ -3,11 +3,21 @@
 #
 import logging
 from os import getenv
-from typing import Any, List, Mapping, MutableMapping, Optional, Tuple
+from typing import Any, Iterator, List, Mapping, MutableMapping, Optional, Tuple
 from urllib.parse import urlparse
 
-from airbyte_cdk.models import FailureType
+from airbyte_cdk.models import (
+    AirbyteCatalog,
+    AirbyteConnectionStatus,
+    AirbyteMessage,
+    AirbyteStateMessage,
+    ConfiguredAirbyteCatalog,
+    FailureType,
+    Status,
+)
 from airbyte_cdk.sources import AbstractSource
+from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
+from airbyte_cdk.sources.source import TState
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.http.requests_native_auth import MultipleTokenAuthenticator
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
@@ -59,9 +69,70 @@ from .streams import (
 from .utils import read_full_refresh
 
 
-class SourceGithub(AbstractSource):
+class SourceGithub(YamlDeclarativeSource, AbstractSource):
     continue_sync_on_stream_failure = True
 
+    def __init__(
+        self,
+        catalog: Optional[ConfiguredAirbyteCatalog] = None,
+        config: Optional[Mapping[str, Any]] = None,
+        state: Optional[TState] = None,
+    ) -> None:
+        super().__init__(catalog=catalog, config=config, state=state, path_to_yaml="manifest.yaml")
+
+    def check(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus:
+        check_succeeded, error = self.check_connection(logger, config)
+        if not check_succeeded:
+            return AirbyteConnectionStatus(status=Status.FAILED, message=repr(error))
+        return AirbyteConnectionStatus(status=Status.SUCCEEDED)
+
+    def read(
+        self,
+        logger: logging.Logger,
+        config: Mapping[str, Any],
+        catalog: ConfiguredAirbyteCatalog,
+        state: Optional[List[AirbyteStateMessage]] = None,
+    ) -> Iterator[AirbyteMessage]:
+        """Route manifest streams to `ConcurrentDeclarativeSource` and Python streams to `AbstractSource`.
+
+        CDK v7 removed the `_group_streams` mechanism that CDK v6 had. This override
+        replicates that behavior: manifest-backed `AbstractStream` objects are read
+        concurrently, while regular Python `Stream` objects are read through
+        `AbstractSource.read()`.
+
+        The concurrent streams come from `super().streams()` (the manifest), which is
+        empty today and does no network work. The synchronous streams are resolved by
+        `AbstractSource.read()` itself, so `SourceGithub.streams()` (which performs live
+        org/repo resolution) is only invoked once per sync. As streams are migrated from
+        Python to the manifest, they automatically move from the synchronous to the
+        concurrent path.
+        """
+        concurrent_streams = super().streams(config=self._config or config)
+        concurrent_stream_names = {stream.name for stream in concurrent_streams}
+
+        concurrent_catalog = ConfiguredAirbyteCatalog(streams=[s for s in catalog.streams if s.stream.name in concurrent_stream_names])
+        if concurrent_catalog.streams:
+            selected = self._select_streams(streams=concurrent_streams, configured_catalog=concurrent_catalog)
+            if selected:
+                yield from self._concurrent_source.read(selected)
+
+        synchronous_catalog = ConfiguredAirbyteCatalog(streams=[s for s in catalog.streams if s.stream.name not in concurrent_stream_names])
+        if synchronous_catalog.streams:
+            yield from AbstractSource.read(self, logger, config, synchronous_catalog, state)
+
+    def discover(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteCatalog:
+        """Return the union of Python `Stream` objects and manifest-backed streams.
+
+        `ConcurrentDeclarativeSource.discover()` only reports manifest streams, so this
+        override adds the Python streams from `SourceGithub.streams()`. As streams move
+        into the manifest they leave the Python list and are reported via
+        `super().streams()`, keeping the discovered catalog complete throughout the migration.
+        """
+        effective_config = self._config or config
+        streams = [stream.as_airbyte_stream() for stream in self.streams(config=effective_config)]
+        streams += [stream.as_airbyte_stream() for stream in super().streams(config=effective_config)]
+        return AirbyteCatalog(streams=streams)
+
     @staticmethod
     def _get_org_repositories(
         config: Mapping[str, Any], authenticator: MultipleTokenAuthenticator, is_check_connection: bool = False

airbyte_source_github-2.1.35/source_github/run.py

@@ -1,17 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-
-import sys
-
-from airbyte_cdk.entrypoint import launch
-from source_github import SourceGithub
-from source_github.config_migrations import MigrateBranch, MigrateRepository
-
-
-def run():
-    source = SourceGithub()
-    MigrateRepository.migrate(sys.argv[1:], source)
-    MigrateBranch.migrate(sys.argv[1:], source)
-    launch(source, sys.argv[1:])