airbyte-source-github 2.2.1__tar.gz → 2.2.3__tar.gz

{airbyte_source_github-2.2.1 → airbyte_source_github-2.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-github
-Version: 2.2.1
+Version: 2.2.3
 Summary: Source implementation for GitHub.
 Home-page: https://airbyte.com
 License: ELv2

{airbyte_source_github-2.2.1 → airbyte_source_github-2.2.3}/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "2.2.1"
+version = "2.2.3"
 name = "airbyte-source-github"
 description = "Source implementation for GitHub."
 authors = [ "Airbyte <contact@airbyte.io>",]

{airbyte_source_github-2.2.1 → airbyte_source_github-2.2.3}/source_github/manifest.yaml

@@ -405,8 +405,11 @@ definitions:
                   - string
 
   # Shared partition routers. Repo-scoped streams reference
-  # `repository_partition_router`; org-scoped streams reference
-  # `organization_partition_router`.
+  # `repository_partition_router`. For orgs there are deliberately two routers with
+  # different audiences: the `repositories` stream references
+  # `organization_partition_router` (config-derived candidates), the org-scoped Python
+  # streams reference `organization_resolution_partition_router` (payload-confirmed).
+  # Picking the wrong one for an org-scoped stream is what broke 2.2.0 — see below.
   #
   # `repository_partition_router` yields the deduplicated union of
   # wildcard-expanded repos and validated explicit repos.
@@ -415,6 +418,16 @@ definitions:
   # orgs (derived inline from config) and the orgs owning explicit repos.
   # User-owned repos have no `organization` object, so the substream child
   # skips them (they contribute a repository but not an organization).
+  # It drives the `repositories` stream, which has to *attempt* `orgs/{org}/repos`
+  # for a wildcard entry to learn whether the login is an organization at all.
+  #
+  # `organization_resolution_partition_router` is the narrower list the org-scoped
+  # Python streams slice on, and must not be the router above: a login taken from
+  # config text is only a *candidate* org, and `orgs/{login}` 404s on every
+  # org-scoped stream when it is a user. Both branches here take the org from a
+  # response payload instead — `owner/login` of the repos `orgs/{org}/repos`
+  # returned, `organization/login` of `repos/{owner}/{repo}` (absent for user-owned
+  # repos) — which is how the legacy Python resolver built the list.
   repository_partition_router:
     type: UnionPartitionRouter
     partition_field: repository
@@ -450,6 +463,25 @@ definitions:
             parent_key: "organization/login"
             partition_field: "organization"
 
+  organization_resolution_partition_router:
+    type: UnionPartitionRouter
+    partition_field: organization
+    partition_routers:
+      - type: SubstreamPartitionRouter
+        parent_stream_configs:
+          - type: ParentStreamConfig
+            stream:
+              $ref: "#/definitions/repositories_resolver_stream"
+            parent_key: "owner/login"
+            partition_field: "organization"
+      - type: SubstreamPartitionRouter
+        parent_stream_configs:
+          - type: ParentStreamConfig
+            stream:
+              $ref: "#/definitions/repository_stats_stream"
+            parent_key: "organization/login"
+            partition_field: "organization"
+
   repositories_stream:
     type: DeclarativeStream
     name: repositories

{airbyte_source_github-2.2.1 → airbyte_source_github-2.2.3}/source_github/source.py

@@ -143,8 +143,8 @@ class SourceGithub(YamlDeclarativeSource, AbstractSource):
 
     def _resolve_repositories_and_organizations(self, config: Mapping[str, Any]) -> Tuple[List[str], List[str]]:
         """Resolve wildcard patterns and explicit repos by enumerating the manifest's
-        shared partition routers — the same components manifest streams slice on at
-        read time, so the Python streams are guaranteed to see identical lists.
+        partition routers — the same components manifest streams slice on at read
+        time, so repo-scoped Python streams see the same repository list.
 
         Wildcard patterns (`org/*`, `org/prefix*`) expand via `repositories_resolver`;
         explicit `org/repo` entries validate via `repository_stats`; entries that 404
@@ -153,6 +153,11 @@ class SourceGithub(YamlDeclarativeSource, AbstractSource):
         The resolver streams' `use_cache: true` means this enumeration warms the HTTP
         cache the manifest streams reuse when reading.
 
+        Organizations come from `organization_resolution_partition_router` — payload-
+        confirmed logins only — which is deliberately narrower than the
+        `organization_partition_router` the `repositories` stream slices on; see that
+        definition's comment in the manifest.
+
         Returns (organizations, repositories), both sorted and deduplicated.
         """
         try:
@@ -175,15 +180,15 @@ class SourceGithub(YamlDeclarativeSource, AbstractSource):
             return sorted({stream_slice.partition[partition_key] for stream_slice in router.stream_slices()})
 
         repositories = enumerate_router("repository_partition_router", "repository")
-        # The organization router unions config-derived wildcard orgs with the orgs
-        # owning explicit repos, so a wildcard entry whose org doesn't exist (404) or
-        # matched no repos would still yield a partition. Keep only orgs that own at
-        # least one resolved repository — the legacy resolver derived orgs from
-        # fetched repo metadata, so orgs without any synced repo never surfaced.
+        # `organization_resolution_partition_router`, not `organization_partition_router`: a
+        # login that config only *claims* is an org must never reach the org-scoped streams.
+        # Config-derived orgs put a user login on `Organizations`, `Teams` and `Users`, where
+        # `orgs/{user}` 404s on every request. The `in repository_owners` filter is a backstop:
+        # an org owning no resolved repository has nothing to sync.
         repository_owners = {repository.split("/", 1)[0] for repository in repositories}
         organizations = [
             organization
-            for organization in enumerate_router("organization_partition_router", "organization")
+            for organization in enumerate_router("organization_resolution_partition_router", "organization")
             if organization in repository_owners
         ]
         return organizations, repositories

{airbyte_source_github-2.2.1 → airbyte_source_github-2.2.3}/source_github/streams.py

@@ -213,11 +213,34 @@ class GithubStreamABC(HttpStream, ABC):
                 raise e
 
             self.logger.warning(error_msg)
+            self._close_slice_after_swallowed_error(stream_slice)
         # Exhausting every token no longer raises a connector-specific exception here: the
         # shared authenticator raises AirbyteTracedException(transient_error) itself, which the
         # `except AirbyteTracedException` branch above re-raises untouched (it carries no
         # response to classify).
 
+    def _close_slice_after_swallowed_error(self, stream_slice: Optional[Mapping[str, Any]]) -> None:
+        """Mark the slice complete when `read_records` skipped it instead of raising.
+
+        `HttpStream._read_pages` closes a resumable-full-refresh slice only after the last
+        page, so an error swallowed mid-slice leaves the partition's cursor state empty and
+        `CursorBasedCheckpointReader._find_next_slice` hands the same partition back forever
+        — no record, no STATE, and the platform kills the attempt on the source heartbeat.
+        Closing the slice makes a swallowed error terminal, as the warning above implies.
+        """
+        cursor = self.get_cursor()
+        if not isinstance(cursor, SubstreamResumableFullRefreshCursor):
+            return
+        # Only close a partition the slice actually names. Several substreams read their parent
+        # by calling its `read_records` straight from `stream_slices()` with a bare mapping that
+        # has no `partition` key (`TeamMembers`, `IssueTimelineEvents`, `utils.read_full_refresh`,
+        # ...), and those parents are shared instances that emit their own STATE later — closing
+        # `_extract_slice_fields`' `{}` fallback would put a meaningless entry in it.
+        partition = (stream_slice or {}).get("partition")
+        if not partition:
+            return
+        cursor.close_slice(StreamSlice(cursor_slice={}, partition=partition))
+
 
 class GithubStream(GithubStreamABC):
     def __init__(self, repositories: List[str], page_size_for_large_streams: int, **kwargs):
@@ -1876,9 +1899,7 @@ class ContributorActivity(GithubStream):
                     # In order to retain the existing stream behavior before we added RFR to this stream, we need to close out the
                     # partition after we give up the maximum number of retries on the 202 response. This does lead to the question
                     # of if we should prematurely exit in the first place, but for now we're going to aim for feature parity
-                    partition_obj = stream_slice.get("partition")
-                    if self.cursor and partition_obj:
-                        self.cursor.close_slice(StreamSlice(cursor_slice={}, partition=partition_obj))
+                    self._close_slice_after_swallowed_error(stream_slice)
                 else:
                     raise e
             else: