airbyte-cdk 6.62.0.dev2__py3-none-any.whl → 6.62.0.dev4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. airbyte_cdk/__init__.py +2 -2
  2. airbyte_cdk/connector_builder/connector_builder_handler.py +7 -7
  3. airbyte_cdk/connector_builder/main.py +2 -2
  4. airbyte_cdk/connector_builder/test_reader/reader.py +2 -2
  5. airbyte_cdk/{sources → legacy/sources}/declarative/declarative_stream.py +2 -2
  6. airbyte_cdk/legacy/sources/declarative/incremental/__init__.py +30 -0
  7. airbyte_cdk/{sources → legacy/sources}/declarative/incremental/datetime_based_cursor.py +1 -1
  8. airbyte_cdk/{sources → legacy/sources}/declarative/incremental/global_substream_cursor.py +4 -2
  9. airbyte_cdk/legacy/sources/declarative/incremental/per_partition_cursor.py +1 -1
  10. airbyte_cdk/{sources → legacy/sources}/declarative/incremental/per_partition_with_global.py +8 -6
  11. airbyte_cdk/{sources → legacy/sources}/declarative/incremental/resumable_full_refresh_cursor.py +1 -1
  12. airbyte_cdk/manifest_server/Dockerfile +2 -2
  13. airbyte_cdk/manifest_server/README.md +22 -0
  14. airbyte_cdk/manifest_server/api_models/__init__.py +2 -0
  15. airbyte_cdk/manifest_server/api_models/manifest.py +12 -0
  16. airbyte_cdk/manifest_server/api_models/stream.py +2 -2
  17. airbyte_cdk/manifest_server/app.py +6 -0
  18. airbyte_cdk/manifest_server/cli/_common.py +1 -0
  19. airbyte_cdk/manifest_server/command_processor/processor.py +2 -5
  20. airbyte_cdk/manifest_server/command_processor/utils.py +1 -1
  21. airbyte_cdk/manifest_server/helpers/__init__.py +0 -0
  22. airbyte_cdk/manifest_server/helpers/tracing.py +36 -0
  23. airbyte_cdk/manifest_server/routers/manifest.py +38 -2
  24. airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py +6 -3
  25. airbyte_cdk/sources/declarative/checks/check_stream.py +6 -3
  26. airbyte_cdk/sources/declarative/checks/connection_checker.py +5 -2
  27. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +38 -451
  28. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +0 -27
  29. airbyte_cdk/sources/declarative/incremental/__init__.py +0 -24
  30. airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +58 -5
  31. airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +1 -2
  32. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +1 -21
  33. airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +0 -5
  34. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +35 -23
  35. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +3 -3
  36. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +2 -2
  37. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +21 -7
  38. airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +1 -1
  39. airbyte_cdk/sources/declarative/yaml_declarative_source.py +1 -1
  40. airbyte_cdk/sources/streams/concurrent/abstract_stream.py +0 -4
  41. airbyte_cdk/sources/utils/schema_helpers.py +29 -9
  42. airbyte_cdk/sources/utils/transform.py +25 -13
  43. airbyte_cdk/utils/spec_schema_transformations.py +7 -5
  44. {airbyte_cdk-6.62.0.dev2.dist-info → airbyte_cdk-6.62.0.dev4.dist-info}/METADATA +4 -2
  45. {airbyte_cdk-6.62.0.dev2.dist-info → airbyte_cdk-6.62.0.dev4.dist-info}/RECORD +51 -49
  46. /airbyte_cdk/{sources → legacy/sources}/declarative/incremental/declarative_cursor.py +0 -0
  47. /airbyte_cdk/manifest_server/{auth.py → helpers/auth.py} +0 -0
  48. {airbyte_cdk-6.62.0.dev2.dist-info → airbyte_cdk-6.62.0.dev4.dist-info}/LICENSE.txt +0 -0
  49. {airbyte_cdk-6.62.0.dev2.dist-info → airbyte_cdk-6.62.0.dev4.dist-info}/LICENSE_SHORT +0 -0
  50. {airbyte_cdk-6.62.0.dev2.dist-info → airbyte_cdk-6.62.0.dev4.dist-info}/WHEEL +0 -0
  51. {airbyte_cdk-6.62.0.dev2.dist-info → airbyte_cdk-6.62.0.dev4.dist-info}/entry_points.txt +0 -0
airbyte_cdk/__init__.py CHANGED
@@ -65,6 +65,8 @@ from .config_observation import (
65
65
  from .connector import BaseConnector, Connector
66
66
  from .destinations import Destination
67
67
  from .entrypoint import AirbyteEntrypoint, launch
68
+ from .legacy.sources.declarative.declarative_stream import DeclarativeStream
69
+ from .legacy.sources.declarative.incremental import DatetimeBasedCursor
68
70
  from .logger import AirbyteLogFormatter, init_logger
69
71
  from .models import (
70
72
  AdvancedAuth,
@@ -99,13 +101,11 @@ from .sources.declarative.auth.token import (
99
101
  BearerAuthenticator,
100
102
  )
101
103
  from .sources.declarative.datetime.min_max_datetime import MinMaxDatetime
102
- from .sources.declarative.declarative_stream import DeclarativeStream
103
104
  from .sources.declarative.decoders import Decoder, JsonDecoder
104
105
  from .sources.declarative.exceptions import ReadException
105
106
  from .sources.declarative.extractors import DpathExtractor, RecordSelector
106
107
  from .sources.declarative.extractors.record_extractor import RecordExtractor
107
108
  from .sources.declarative.extractors.record_filter import RecordFilter
108
- from .sources.declarative.incremental import DatetimeBasedCursor
109
109
  from .sources.declarative.interpolation import InterpolatedBoolean, InterpolatedString
110
110
  from .sources.declarative.migrations.legacy_to_per_partition_state_migration import (
111
111
  LegacyToPerPartitionStateMigration,
@@ -62,10 +62,10 @@ def should_normalize_manifest(config: Mapping[str, Any]) -> bool:
62
62
 
63
63
  def create_source(
64
64
  config: Mapping[str, Any],
65
- limits: TestLimits,
66
- catalog: Optional[ConfiguredAirbyteCatalog],
67
- state: Optional[List[AirbyteStateMessage]],
68
- ) -> ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]]:
65
+ limits: TestLimits | None = None,
66
+ catalog: ConfiguredAirbyteCatalog | None = None,
67
+ state: List[AirbyteStateMessage] | None = None,
68
+ ) -> ConcurrentDeclarativeSource:
69
69
  manifest = config["__injected_declarative_manifest"]
70
70
 
71
71
  # We enforce a concurrency level of 1 so that the stream is processed on a single thread
@@ -88,7 +88,7 @@ def create_source(
88
88
 
89
89
 
90
90
  def read_stream(
91
- source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
91
+ source: ConcurrentDeclarativeSource,
92
92
  config: Mapping[str, Any],
93
93
  configured_catalog: ConfiguredAirbyteCatalog,
94
94
  state: List[AirbyteStateMessage],
@@ -127,7 +127,7 @@ def read_stream(
127
127
 
128
128
 
129
129
  def resolve_manifest(
130
- source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
130
+ source: ConcurrentDeclarativeSource,
131
131
  ) -> AirbyteMessage:
132
132
  try:
133
133
  return AirbyteMessage(
@@ -146,7 +146,7 @@ def resolve_manifest(
146
146
 
147
147
 
148
148
  def full_resolve_manifest(
149
- source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]], limits: TestLimits
149
+ source: ConcurrentDeclarativeSource, limits: TestLimits
150
150
  ) -> AirbyteMessage:
151
151
  try:
152
152
  manifest = {**source.resolved_manifest}
@@ -34,7 +34,7 @@ from airbyte_cdk.utils.traced_exception import AirbyteTracedException
34
34
 
35
35
  def get_config_and_catalog_from_args(
36
36
  args: List[str],
37
- ) -> Tuple[str, Mapping[str, Any], Optional[ConfiguredAirbyteCatalog], Any]:
37
+ ) -> Tuple[str, Mapping[str, Any], Optional[ConfiguredAirbyteCatalog], List[AirbyteStateMessage]]:
38
38
  # TODO: Add functionality for the `debug` logger.
39
39
  # Currently, no one `debug` level log will be displayed during `read` a stream for a connector created through `connector-builder`.
40
40
  parsed_args = AirbyteEntrypoint.parse_args(args)
@@ -70,7 +70,7 @@ def get_config_and_catalog_from_args(
70
70
 
71
71
 
72
72
  def handle_connector_builder_request(
73
- source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
73
+ source: ConcurrentDeclarativeSource,
74
74
  command: str,
75
75
  config: Mapping[str, Any],
76
76
  catalog: Optional[ConfiguredAirbyteCatalog],
@@ -85,7 +85,7 @@ class TestReader:
85
85
 
86
86
  def run_test_read(
87
87
  self,
88
- source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
88
+ source: ConcurrentDeclarativeSource,
89
89
  config: Mapping[str, Any],
90
90
  configured_catalog: ConfiguredAirbyteCatalog,
91
91
  stream_name: str,
@@ -383,7 +383,7 @@ class TestReader:
383
383
 
384
384
  def _read_stream(
385
385
  self,
386
- source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
386
+ source: ConcurrentDeclarativeSource,
387
387
  config: Mapping[str, Any],
388
388
  configured_catalog: ConfiguredAirbyteCatalog,
389
389
  state: List[AirbyteStateMessage],
@@ -5,12 +5,12 @@ import logging
5
5
  from dataclasses import InitVar, dataclass, field
6
6
  from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union
7
7
 
8
- from airbyte_cdk.models import SyncMode
9
- from airbyte_cdk.sources.declarative.incremental import (
8
+ from airbyte_cdk.legacy.sources.declarative.incremental import (
10
9
  GlobalSubstreamCursor,
11
10
  PerPartitionCursor,
12
11
  PerPartitionWithGlobalCursor,
13
12
  )
13
+ from airbyte_cdk.models import SyncMode
14
14
  from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
15
15
  from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration
16
16
  from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever
@@ -1 +1,31 @@
1
1
  # Copyright (c) 2025 Airbyte, Inc., all rights reserved.
2
+
3
+ from airbyte_cdk.legacy.sources.declarative.incremental.datetime_based_cursor import (
4
+ DatetimeBasedCursor,
5
+ )
6
+ from airbyte_cdk.legacy.sources.declarative.incremental.declarative_cursor import DeclarativeCursor
7
+ from airbyte_cdk.legacy.sources.declarative.incremental.global_substream_cursor import (
8
+ GlobalSubstreamCursor,
9
+ )
10
+ from airbyte_cdk.legacy.sources.declarative.incremental.per_partition_cursor import (
11
+ CursorFactory,
12
+ PerPartitionCursor,
13
+ )
14
+ from airbyte_cdk.legacy.sources.declarative.incremental.per_partition_with_global import (
15
+ PerPartitionWithGlobalCursor,
16
+ )
17
+ from airbyte_cdk.legacy.sources.declarative.incremental.resumable_full_refresh_cursor import (
18
+ ChildPartitionResumableFullRefreshCursor,
19
+ ResumableFullRefreshCursor,
20
+ )
21
+
22
+ __all__ = [
23
+ "CursorFactory",
24
+ "DatetimeBasedCursor",
25
+ "DeclarativeCursor",
26
+ "GlobalSubstreamCursor",
27
+ "PerPartitionCursor",
28
+ "PerPartitionWithGlobalCursor",
29
+ "ResumableFullRefreshCursor",
30
+ "ChildPartitionResumableFullRefreshCursor",
31
+ ]
@@ -9,10 +9,10 @@ from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optio
9
9
 
10
10
  from isodate import Duration, duration_isoformat, parse_duration
11
11
 
12
+ from airbyte_cdk.legacy.sources.declarative.incremental.declarative_cursor import DeclarativeCursor
12
13
  from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, Type
13
14
  from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser
14
15
  from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime
15
- from airbyte_cdk.sources.declarative.incremental.declarative_cursor import DeclarativeCursor
16
16
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
17
17
  from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation
18
18
  from airbyte_cdk.sources.declarative.requesters.request_option import (
@@ -6,8 +6,10 @@ import threading
6
6
  import time
7
7
  from typing import Any, Callable, Iterable, Mapping, Optional, TypeVar, Union
8
8
 
9
- from airbyte_cdk.sources.declarative.incremental.datetime_based_cursor import DatetimeBasedCursor
10
- from airbyte_cdk.sources.declarative.incremental.declarative_cursor import DeclarativeCursor
9
+ from airbyte_cdk.legacy.sources.declarative.incremental.datetime_based_cursor import (
10
+ DatetimeBasedCursor,
11
+ )
12
+ from airbyte_cdk.legacy.sources.declarative.incremental.declarative_cursor import DeclarativeCursor
11
13
  from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter
12
14
  from airbyte_cdk.sources.types import Record, StreamSlice, StreamState
13
15
 
@@ -6,7 +6,7 @@ import logging
6
6
  from collections import OrderedDict
7
7
  from typing import Any, Callable, Iterable, Mapping, Optional, Union
8
8
 
9
- from airbyte_cdk.sources.declarative.incremental.declarative_cursor import DeclarativeCursor
9
+ from airbyte_cdk.legacy.sources.declarative.incremental.declarative_cursor import DeclarativeCursor
10
10
  from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter
11
11
  from airbyte_cdk.sources.streams.checkpoint.per_partition_key_serializer import (
12
12
  PerPartitionKeySerializer,
@@ -3,16 +3,18 @@
3
3
  #
4
4
  from typing import Any, Iterable, Mapping, MutableMapping, Optional, Union
5
5
 
6
- from airbyte_cdk.legacy.sources.declarative.incremental.per_partition_cursor import (
7
- CursorFactory,
8
- PerPartitionCursor,
6
+ from airbyte_cdk.legacy.sources.declarative.incremental.datetime_based_cursor import (
7
+ DatetimeBasedCursor,
9
8
  )
10
- from airbyte_cdk.sources.declarative.incremental.datetime_based_cursor import DatetimeBasedCursor
11
- from airbyte_cdk.sources.declarative.incremental.declarative_cursor import DeclarativeCursor
12
- from airbyte_cdk.sources.declarative.incremental.global_substream_cursor import (
9
+ from airbyte_cdk.legacy.sources.declarative.incremental.declarative_cursor import DeclarativeCursor
10
+ from airbyte_cdk.legacy.sources.declarative.incremental.global_substream_cursor import (
13
11
  GlobalSubstreamCursor,
14
12
  iterate_with_last_flag_and_state,
15
13
  )
14
+ from airbyte_cdk.legacy.sources.declarative.incremental.per_partition_cursor import (
15
+ CursorFactory,
16
+ PerPartitionCursor,
17
+ )
16
18
  from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter
17
19
  from airbyte_cdk.sources.types import Record, StreamSlice, StreamState
18
20
 
@@ -3,7 +3,7 @@
3
3
  from dataclasses import InitVar, dataclass
4
4
  from typing import Any, Iterable, Mapping, Optional
5
5
 
6
- from airbyte_cdk.sources.declarative.incremental import DeclarativeCursor
6
+ from airbyte_cdk.legacy.sources.declarative.incremental import DeclarativeCursor
7
7
  from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState
8
8
  from airbyte_cdk.sources.streams.checkpoint.checkpoint_reader import FULL_REFRESH_COMPLETE_STATE
9
9
 
@@ -11,7 +11,7 @@ FROM python:3.12-slim-bookworm
11
11
  RUN apt-get update && \
12
12
  apt-get install -y git && \
13
13
  rm -rf /var/lib/apt/lists/* && \
14
- pip install poetry==1.8.3
14
+ pip install poetry==2.0.1
15
15
 
16
16
  # Configure poetry to not create virtual environments and disable interactive mode
17
17
  ENV POETRY_NO_INTERACTION=1 \
@@ -42,4 +42,4 @@ USER airbyte:airbyte
42
42
 
43
43
  EXPOSE 8080
44
44
 
45
- CMD ["uvicorn", "airbyte_cdk.manifest_server.app:app", "--host", "0.0.0.0", "--port", "8080"]
45
+ CMD ["uvicorn", "airbyte_cdk.manifest_server.app:app", "--host", "0.0.0.0", "--port", "8080"]
@@ -154,3 +154,25 @@ docker run -p 8080:8080 manifest-server
154
154
  ```
155
155
 
156
156
  Note: The container runs on port 8080 by default.
157
+
158
+ ## Datadog APM
159
+
160
+ The manifest server supports Datadog APM tracing for monitoring and observability:
161
+
162
+ ### Configuration
163
+
164
+ To enable Datadog tracing, set the environment variable:
165
+
166
+ ```bash
167
+ export DD_ENABLED=true
168
+ ```
169
+
170
+ This requires the `ddtrace` dependency, which is included in the `manifest-server` extra. For additional configuration options via environment variables, see [ddtrace configuration](https://ddtrace.readthedocs.io/en/stable/configuration.html).
171
+
172
+ ### Usage
173
+
174
+ ```bash
175
+ # Run with Datadog tracing enabled
176
+ DD_ENABLED=true manifest-server start
177
+ ```
178
+
@@ -12,6 +12,7 @@ from .manifest import (
12
12
  DiscoverResponse,
13
13
  FullResolveRequest,
14
14
  ManifestResponse,
15
+ RequestContext,
15
16
  ResolveRequest,
16
17
  StreamTestReadRequest,
17
18
  )
@@ -30,6 +31,7 @@ __all__ = [
30
31
  "ConnectorConfig",
31
32
  "Manifest",
32
33
  # Manifest request/response models
34
+ "RequestContext",
33
35
  "FullResolveRequest",
34
36
  "ManifestResponse",
35
37
  "StreamTestReadRequest",
@@ -13,6 +13,13 @@ from pydantic import BaseModel, Field
13
13
  from .dicts import ConnectorConfig, Manifest
14
14
 
15
15
 
16
+ class RequestContext(BaseModel):
17
+ """Optional context information for tracing and observability."""
18
+
19
+ workspace_id: Optional[str] = None
20
+ project_id: Optional[str] = None
21
+
22
+
16
23
  class StreamTestReadRequest(BaseModel):
17
24
  """Request to test read from a specific stream."""
18
25
 
@@ -24,6 +31,7 @@ class StreamTestReadRequest(BaseModel):
24
31
  record_limit: int = Field(default=100, ge=1, le=5000)
25
32
  page_limit: int = Field(default=5, ge=1, le=20)
26
33
  slice_limit: int = Field(default=5, ge=1, le=20)
34
+ context: Optional[RequestContext] = None
27
35
 
28
36
 
29
37
  class CheckRequest(BaseModel):
@@ -31,6 +39,7 @@ class CheckRequest(BaseModel):
31
39
 
32
40
  manifest: Manifest
33
41
  config: ConnectorConfig
42
+ context: Optional[RequestContext] = None
34
43
 
35
44
 
36
45
  class CheckResponse(BaseModel):
@@ -45,6 +54,7 @@ class DiscoverRequest(BaseModel):
45
54
 
46
55
  manifest: Manifest
47
56
  config: ConnectorConfig
57
+ context: Optional[RequestContext] = None
48
58
 
49
59
 
50
60
  class DiscoverResponse(BaseModel):
@@ -57,6 +67,7 @@ class ResolveRequest(BaseModel):
57
67
  """Request to resolve a manifest."""
58
68
 
59
69
  manifest: Manifest
70
+ context: Optional[RequestContext] = None
60
71
 
61
72
 
62
73
  class ManifestResponse(BaseModel):
@@ -71,3 +82,4 @@ class FullResolveRequest(BaseModel):
71
82
  manifest: Manifest
72
83
  config: ConnectorConfig
73
84
  stream_limit: int = Field(default=100, ge=1, le=100)
85
+ context: Optional[RequestContext] = None
@@ -6,7 +6,7 @@ They accurately reflect the runtime types returned by the CDK, particularly
6
6
  fixing type mismatches like slice_descriptor being a string rather than an object.
7
7
  """
8
8
 
9
- from typing import Any, Dict, List, Optional
9
+ from typing import Any, Dict, List, Optional, Union
10
10
 
11
11
  from pydantic import BaseModel
12
12
 
@@ -59,7 +59,7 @@ class StreamReadSlices(BaseModel):
59
59
  """Slices of data read from a stream."""
60
60
 
61
61
  pages: List[StreamReadPages]
62
- slice_descriptor: Optional[str] # This is actually a string at runtime, not Dict[str, Any]
62
+ slice_descriptor: Optional[Union[Dict[str, Any], str]] # We're seeing strings at runtime
63
63
  state: Optional[List[Dict[str, Any]]] = None
64
64
  auxiliary_requests: Optional[List[AuxiliaryRequest]] = None
65
65
 
@@ -1,3 +1,9 @@
1
+ import os
2
+
3
+ if os.getenv("DD_ENABLED", "false").lower() == "true":
4
+ # Auto-instrumentation should be imported as early as possible.
5
+ import ddtrace.auto # noqa: F401
6
+
1
7
  from fastapi import FastAPI
2
8
 
3
9
  from .routers import capabilities, health, manifest
@@ -7,6 +7,7 @@ import rich_click as click
7
7
 
8
8
  # Import server dependencies with graceful fallback
9
9
  try:
10
+ import ddtrace # noqa: F401
10
11
  import fastapi # noqa: F401
11
12
  import uvicorn # noqa: F401
12
13
 
@@ -21,12 +21,10 @@ from airbyte_cdk.test.entrypoint_wrapper import AirbyteEntrypointException, Entr
21
21
 
22
22
 
23
23
  class ManifestCommandProcessor:
24
- _source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]]
24
+ _source: ConcurrentDeclarativeSource
25
25
  _logger = logging.getLogger("airbyte.manifest-server")
26
26
 
27
- def __init__(
28
- self, source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]]
29
- ) -> None:
27
+ def __init__(self, source: ConcurrentDeclarativeSource) -> None:
30
28
  self._source = source
31
29
 
32
30
  def test_read(
@@ -41,7 +39,6 @@ class ManifestCommandProcessor:
41
39
  """
42
40
  Test the read method of the source.
43
41
  """
44
-
45
42
  test_read_handler = TestReader(
46
43
  max_pages_per_slice=page_limit,
47
44
  max_slices=slice_limit,
@@ -63,7 +63,7 @@ def build_source(
63
63
  record_limit: Optional[int] = None,
64
64
  page_limit: Optional[int] = None,
65
65
  slice_limit: Optional[int] = None,
66
- ) -> ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]]:
66
+ ) -> ConcurrentDeclarativeSource:
67
67
  # We enforce a concurrency level of 1 so that the stream is processed on a single thread
68
68
  # to retain ordering for the grouping of the builder message responses.
69
69
  definition = copy.deepcopy(manifest)
File without changes
@@ -0,0 +1,36 @@
1
+ import logging
2
+ from typing import Optional
3
+
4
+ import ddtrace
5
+
6
+ logger = logging.getLogger(__name__)
7
+
8
+
9
+ def apply_trace_tags_from_context(
10
+ workspace_id: Optional[str] = None,
11
+ project_id: Optional[str] = None,
12
+ ) -> None:
13
+ """Apply trace tags from context to the current span."""
14
+ if not workspace_id and not project_id:
15
+ return
16
+
17
+ # Log the trace IDs for observability
18
+ log_parts = []
19
+ if workspace_id:
20
+ log_parts.append(f"workspace_id={workspace_id}")
21
+ if project_id:
22
+ log_parts.append(f"project_id={project_id}")
23
+
24
+ if log_parts:
25
+ logger.info(f"Processing request with trace tags: {', '.join(log_parts)}")
26
+
27
+ try:
28
+ span = ddtrace.tracer.current_span()
29
+ if span:
30
+ if workspace_id:
31
+ span.set_tag("workspace_id", workspace_id)
32
+ if project_id:
33
+ span.set_tag("project_id", project_id)
34
+ except Exception:
35
+ # Silently ignore any ddtrace-related errors (e.g. if ddtrace.auto wasn't run)
36
+ pass
@@ -27,9 +27,10 @@ from ..api_models import (
27
27
  StreamReadResponse,
28
28
  StreamTestReadRequest,
29
29
  )
30
- from ..auth import verify_jwt_token
31
30
  from ..command_processor.processor import ManifestCommandProcessor
32
31
  from ..command_processor.utils import build_catalog, build_source
32
+ from ..helpers.auth import verify_jwt_token
33
+ from ..helpers.tracing import apply_trace_tags_from_context
33
34
 
34
35
 
35
36
  def safe_build_source(
@@ -40,7 +41,7 @@ def safe_build_source(
40
41
  page_limit: Optional[int] = None,
41
42
  slice_limit: Optional[int] = None,
42
43
  record_limit: Optional[int] = None,
43
- ) -> ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]]:
44
+ ) -> ConcurrentDeclarativeSource:
44
45
  """Wrapper around build_source that converts ValidationError to HTTPException."""
45
46
  try:
46
47
  return build_source(
@@ -68,6 +69,13 @@ def test_read(request: StreamTestReadRequest) -> StreamReadResponse:
68
69
  """
69
70
  Test reading from a specific stream in the manifest.
70
71
  """
72
+ # Apply trace tags from context if provided
73
+ if request.context:
74
+ apply_trace_tags_from_context(
75
+ workspace_id=request.context.workspace_id,
76
+ project_id=request.context.project_id,
77
+ )
78
+
71
79
  config_dict = request.config.model_dump()
72
80
 
73
81
  catalog = build_catalog(request.stream_name)
@@ -104,6 +112,13 @@ def test_read(request: StreamTestReadRequest) -> StreamReadResponse:
104
112
  @router.post("/check", operation_id="check")
105
113
  def check(request: CheckRequest) -> CheckResponse:
106
114
  """Check configuration against a manifest"""
115
+ # Apply trace tags from context if provided
116
+ if request.context:
117
+ apply_trace_tags_from_context(
118
+ workspace_id=request.context.workspace_id,
119
+ project_id=request.context.project_id,
120
+ )
121
+
107
122
  source = safe_build_source(request.manifest.model_dump(), request.config.model_dump())
108
123
  runner = ManifestCommandProcessor(source)
109
124
  success, message = runner.check_connection(request.config.model_dump())
@@ -113,6 +128,13 @@ def check(request: CheckRequest) -> CheckResponse:
113
128
  @router.post("/discover", operation_id="discover")
114
129
  def discover(request: DiscoverRequest) -> DiscoverResponse:
115
130
  """Discover streams from a manifest"""
131
+ # Apply trace tags from context if provided
132
+ if request.context:
133
+ apply_trace_tags_from_context(
134
+ workspace_id=request.context.workspace_id,
135
+ project_id=request.context.project_id,
136
+ )
137
+
116
138
  source = safe_build_source(request.manifest.model_dump(), request.config.model_dump())
117
139
  runner = ManifestCommandProcessor(source)
118
140
  catalog = runner.discover(request.config.model_dump())
@@ -124,6 +146,13 @@ def discover(request: DiscoverRequest) -> DiscoverResponse:
124
146
  @router.post("/resolve", operation_id="resolve")
125
147
  def resolve(request: ResolveRequest) -> ManifestResponse:
126
148
  """Resolve a manifest to its final configuration."""
149
+ # Apply trace tags from context if provided
150
+ if request.context:
151
+ apply_trace_tags_from_context(
152
+ workspace_id=request.context.workspace_id,
153
+ project_id=request.context.project_id,
154
+ )
155
+
127
156
  source = safe_build_source(request.manifest.model_dump(), {})
128
157
  return ManifestResponse(manifest=Manifest(**source.resolved_manifest))
129
158
 
@@ -135,6 +164,13 @@ def full_resolve(request: FullResolveRequest) -> ManifestResponse:
135
164
 
136
165
  This is a similar operation to resolve, but has an extra step which generates streams from dynamic stream templates if the manifest contains any. This is used when a user clicks the generate streams button on a stream template in the Builder UI
137
166
  """
167
+ # Apply trace tags from context if provided
168
+ if request.context:
169
+ apply_trace_tags_from_context(
170
+ workspace_id=request.context.workspace_id,
171
+ project_id=request.context.project_id,
172
+ )
173
+
138
174
  source = safe_build_source(request.manifest.model_dump(), request.config.model_dump())
139
175
  manifest = {**source.resolved_manifest}
140
176
  streams = manifest.get("streams", [])
@@ -6,11 +6,11 @@ import logging
6
6
  from dataclasses import InitVar, dataclass
7
7
  from typing import Any, List, Mapping, Tuple, Union
8
8
 
9
- from airbyte_cdk.sources.abstract_source import AbstractSource
9
+ from airbyte_cdk.sources import Source
10
10
  from airbyte_cdk.sources.declarative.checks.check_stream import evaluate_availability
11
11
  from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
12
+ from airbyte_cdk.sources.streams import Stream
12
13
  from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
13
- from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy
14
14
 
15
15
 
16
16
  @dataclass
@@ -33,7 +33,10 @@ class CheckDynamicStream(ConnectionChecker):
33
33
  self._parameters = parameters
34
34
 
35
35
  def check_connection(
36
- self, source: AbstractSource, logger: logging.Logger, config: Mapping[str, Any]
36
+ self,
37
+ source: Source,
38
+ logger: logging.Logger,
39
+ config: Mapping[str, Any],
37
40
  ) -> Tuple[bool, Any]:
38
41
  streams: List[Union[Stream, AbstractStream]] = source.streams(config=config) # type: ignore # this is a migration step and we expect the declarative CDK to migrate off of ConnectionChecker
39
42
 
@@ -7,7 +7,7 @@ import traceback
7
7
  from dataclasses import InitVar, dataclass
8
8
  from typing import Any, Dict, List, Mapping, Optional, Tuple, Union
9
9
 
10
- from airbyte_cdk.sources.abstract_source import AbstractSource
10
+ from airbyte_cdk.sources import Source
11
11
  from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
12
12
  from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
13
13
  from airbyte_cdk.sources.streams.core import Stream
@@ -64,7 +64,10 @@ class CheckStream(ConnectionChecker):
64
64
  return False, error_message
65
65
 
66
66
  def check_connection(
67
- self, source: AbstractSource, logger: logging.Logger, config: Mapping[str, Any]
67
+ self,
68
+ source: Source,
69
+ logger: logging.Logger,
70
+ config: Mapping[str, Any],
68
71
  ) -> Tuple[bool, Any]:
69
72
  """Checks the connection to the source and its streams."""
70
73
  try:
@@ -118,7 +121,7 @@ class CheckStream(ConnectionChecker):
118
121
 
119
122
  def _check_dynamic_streams_availability(
120
123
  self,
121
- source: AbstractSource,
124
+ source: Source,
122
125
  stream_name_to_stream: Dict[str, Union[Stream, AbstractStream]],
123
126
  logger: logging.Logger,
124
127
  ) -> Tuple[bool, Any]:
@@ -6,7 +6,7 @@ import logging
6
6
  from abc import ABC, abstractmethod
7
7
  from typing import Any, Mapping, Tuple
8
8
 
9
- from airbyte_cdk import AbstractSource
9
+ from airbyte_cdk.sources import Source
10
10
 
11
11
 
12
12
  class ConnectionChecker(ABC):
@@ -16,7 +16,10 @@ class ConnectionChecker(ABC):
16
16
 
17
17
  @abstractmethod
18
18
  def check_connection(
19
- self, source: AbstractSource, logger: logging.Logger, config: Mapping[str, Any]
19
+ self,
20
+ source: Source,
21
+ logger: logging.Logger,
22
+ config: Mapping[str, Any],
20
23
  ) -> Tuple[bool, Any]:
21
24
  """
22
25
  Tests if the input configuration can be used to successfully connect to the integration e.g: if a provided Stripe API token can be used to connect