airbyte-cdk 6.60.16__py3-none-any.whl → 6.60.16.post40.dev17219503797__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. airbyte_cdk/connector_builder/connector_builder_handler.py +32 -36
  2. airbyte_cdk/connector_builder/main.py +3 -3
  3. airbyte_cdk/connector_builder/test_reader/helpers.py +24 -2
  4. airbyte_cdk/connector_builder/test_reader/message_grouper.py +1 -1
  5. airbyte_cdk/manifest_server/Dockerfile +45 -0
  6. airbyte_cdk/manifest_server/README.md +142 -0
  7. airbyte_cdk/manifest_server/__init__.py +3 -0
  8. airbyte_cdk/manifest_server/api_models/__init__.py +41 -0
  9. airbyte_cdk/manifest_server/api_models/capabilities.py +7 -0
  10. airbyte_cdk/manifest_server/api_models/dicts.py +17 -0
  11. airbyte_cdk/manifest_server/api_models/manifest.py +73 -0
  12. airbyte_cdk/manifest_server/api_models/stream.py +76 -0
  13. airbyte_cdk/manifest_server/app.py +17 -0
  14. airbyte_cdk/manifest_server/auth.py +43 -0
  15. airbyte_cdk/manifest_server/cli/__init__.py +5 -0
  16. airbyte_cdk/manifest_server/cli/_common.py +28 -0
  17. airbyte_cdk/manifest_server/cli/_info.py +30 -0
  18. airbyte_cdk/manifest_server/cli/_openapi.py +43 -0
  19. airbyte_cdk/manifest_server/cli/_start.py +38 -0
  20. airbyte_cdk/manifest_server/cli/run.py +59 -0
  21. airbyte_cdk/manifest_server/command_processor/__init__.py +0 -0
  22. airbyte_cdk/manifest_server/command_processor/processor.py +151 -0
  23. airbyte_cdk/manifest_server/command_processor/utils.py +76 -0
  24. airbyte_cdk/manifest_server/main.py +24 -0
  25. airbyte_cdk/manifest_server/openapi.yaml +641 -0
  26. airbyte_cdk/manifest_server/routers/__init__.py +0 -0
  27. airbyte_cdk/manifest_server/routers/capabilities.py +25 -0
  28. airbyte_cdk/manifest_server/routers/health.py +13 -0
  29. airbyte_cdk/manifest_server/routers/manifest.py +137 -0
  30. airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +15 -22
  31. airbyte_cdk/sources/concurrent_source/concurrent_source.py +30 -18
  32. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +73 -3
  33. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +4 -0
  34. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +42 -4
  35. airbyte_cdk/sources/declarative/stream_slicers/stream_slicer_test_read_decorator.py +2 -2
  36. airbyte_cdk/sources/message/concurrent_repository.py +47 -0
  37. airbyte_cdk/sources/streams/concurrent/cursor.py +23 -7
  38. airbyte_cdk/sources/streams/concurrent/partition_reader.py +46 -5
  39. airbyte_cdk/sources/streams/concurrent/partitions/types.py +7 -1
  40. airbyte_cdk/sources/streams/http/http_client.py +4 -1
  41. airbyte_cdk/sources/utils/slice_logger.py +4 -0
  42. {airbyte_cdk-6.60.16.dist-info → airbyte_cdk-6.60.16.post40.dev17219503797.dist-info}/METADATA +4 -1
  43. {airbyte_cdk-6.60.16.dist-info → airbyte_cdk-6.60.16.post40.dev17219503797.dist-info}/RECORD +47 -21
  44. {airbyte_cdk-6.60.16.dist-info → airbyte_cdk-6.60.16.post40.dev17219503797.dist-info}/entry_points.txt +1 -0
  45. {airbyte_cdk-6.60.16.dist-info → airbyte_cdk-6.60.16.post40.dev17219503797.dist-info}/LICENSE.txt +0 -0
  46. {airbyte_cdk-6.60.16.dist-info → airbyte_cdk-6.60.16.post40.dev17219503797.dist-info}/LICENSE_SHORT +0 -0
  47. {airbyte_cdk-6.60.16.dist-info → airbyte_cdk-6.60.16.post40.dev17219503797.dist-info}/WHEEL +0 -0
@@ -0,0 +1,641 @@
1
+ # This file is auto-generated. Do not edit manually.
2
+ # To regenerate, run: manifest-server generate-openapi
3
+
4
+ openapi: 3.1.0
5
+ info:
6
+ title: Manifest Server Service
7
+ description: A service for running low-code Airbyte connectors
8
+ contact:
9
+ name: Airbyte
10
+ url: https://airbyte.com/
11
+ version: 0.1.0
12
+ paths:
13
+ /health/:
14
+ get:
15
+ tags:
16
+ - health
17
+ summary: Health
18
+ operationId: health_health__get
19
+ responses:
20
+ '200':
21
+ description: Successful Response
22
+ content:
23
+ application/json:
24
+ schema:
25
+ additionalProperties:
26
+ type: string
27
+ type: object
28
+ title: Response Health Health Get
29
+ /capabilities/:
30
+ get:
31
+ tags:
32
+ - capabilities
33
+ summary: Get Capabilities
34
+ description: "Get the capabilities available for the manifest server.\n\nReturns:\n\
35
+ \ Dict containing the service capabilities including custom code execution\
36
+ \ support."
37
+ operationId: getCapabilities
38
+ responses:
39
+ '200':
40
+ description: Successful Response
41
+ content:
42
+ application/json:
43
+ schema:
44
+ $ref: '#/components/schemas/CapabilitiesResponse'
45
+ /v1/manifest/test_read:
46
+ post:
47
+ tags:
48
+ - manifest
49
+ summary: Test Read
50
+ description: Test reading from a specific stream in the manifest.
51
+ operationId: testRead
52
+ requestBody:
53
+ content:
54
+ application/json:
55
+ schema:
56
+ $ref: '#/components/schemas/StreamTestReadRequest'
57
+ required: true
58
+ responses:
59
+ '200':
60
+ description: Successful Response
61
+ content:
62
+ application/json:
63
+ schema:
64
+ $ref: '#/components/schemas/StreamRead'
65
+ '422':
66
+ description: Validation Error
67
+ content:
68
+ application/json:
69
+ schema:
70
+ $ref: '#/components/schemas/HTTPValidationError'
71
+ security:
72
+ - HTTPBearer: []
73
+ /v1/manifest/check:
74
+ post:
75
+ tags:
76
+ - manifest
77
+ summary: Check
78
+ description: Check configuration against a manifest
79
+ operationId: check
80
+ requestBody:
81
+ content:
82
+ application/json:
83
+ schema:
84
+ $ref: '#/components/schemas/CheckRequest'
85
+ required: true
86
+ responses:
87
+ '200':
88
+ description: Successful Response
89
+ content:
90
+ application/json:
91
+ schema:
92
+ $ref: '#/components/schemas/CheckResponse'
93
+ '422':
94
+ description: Validation Error
95
+ content:
96
+ application/json:
97
+ schema:
98
+ $ref: '#/components/schemas/HTTPValidationError'
99
+ security:
100
+ - HTTPBearer: []
101
+ /v1/manifest/discover:
102
+ post:
103
+ tags:
104
+ - manifest
105
+ summary: Discover
106
+ description: Discover streams from a manifest
107
+ operationId: discover
108
+ requestBody:
109
+ content:
110
+ application/json:
111
+ schema:
112
+ $ref: '#/components/schemas/DiscoverRequest'
113
+ required: true
114
+ responses:
115
+ '200':
116
+ description: Successful Response
117
+ content:
118
+ application/json:
119
+ schema:
120
+ $ref: '#/components/schemas/DiscoverResponse'
121
+ '422':
122
+ description: Validation Error
123
+ content:
124
+ application/json:
125
+ schema:
126
+ $ref: '#/components/schemas/HTTPValidationError'
127
+ security:
128
+ - HTTPBearer: []
129
+ /v1/manifest/resolve:
130
+ post:
131
+ tags:
132
+ - manifest
133
+ summary: Resolve
134
+ description: Resolve a manifest to its final configuration.
135
+ operationId: resolve
136
+ requestBody:
137
+ content:
138
+ application/json:
139
+ schema:
140
+ $ref: '#/components/schemas/ResolveRequest'
141
+ required: true
142
+ responses:
143
+ '200':
144
+ description: Successful Response
145
+ content:
146
+ application/json:
147
+ schema:
148
+ $ref: '#/components/schemas/ManifestResponse'
149
+ '422':
150
+ description: Validation Error
151
+ content:
152
+ application/json:
153
+ schema:
154
+ $ref: '#/components/schemas/HTTPValidationError'
155
+ security:
156
+ - HTTPBearer: []
157
+ /v1/manifest/full_resolve:
158
+ post:
159
+ tags:
160
+ - manifest
161
+ summary: Full Resolve
162
+ description: 'Fully resolve a manifest including dynamic streams.
163
+
164
+
165
+ Generates dynamic streams up to the specified limit and includes
166
+
167
+ them in the resolved manifest.'
168
+ operationId: fullResolve
169
+ requestBody:
170
+ content:
171
+ application/json:
172
+ schema:
173
+ $ref: '#/components/schemas/FullResolveRequest'
174
+ required: true
175
+ responses:
176
+ '200':
177
+ description: Successful Response
178
+ content:
179
+ application/json:
180
+ schema:
181
+ $ref: '#/components/schemas/ManifestResponse'
182
+ '422':
183
+ description: Validation Error
184
+ content:
185
+ application/json:
186
+ schema:
187
+ $ref: '#/components/schemas/HTTPValidationError'
188
+ security:
189
+ - HTTPBearer: []
190
+ components:
191
+ schemas:
192
+ AirbyteCatalog:
193
+ properties:
194
+ streams:
195
+ items:
196
+ $ref: '#/components/schemas/AirbyteStream'
197
+ type: array
198
+ title: Streams
199
+ type: object
200
+ required:
201
+ - streams
202
+ title: AirbyteCatalog
203
+ AirbyteStream:
204
+ properties:
205
+ name:
206
+ type: string
207
+ title: Name
208
+ json_schema:
209
+ type: object
210
+ title: Json Schema
211
+ supported_sync_modes:
212
+ items:
213
+ $ref: '#/components/schemas/SyncMode'
214
+ type: array
215
+ title: Supported Sync Modes
216
+ source_defined_cursor:
217
+ anyOf:
218
+ - type: boolean
219
+ - type: 'null'
220
+ title: Source Defined Cursor
221
+ default_cursor_field:
222
+ anyOf:
223
+ - items:
224
+ type: string
225
+ type: array
226
+ - type: 'null'
227
+ title: Default Cursor Field
228
+ source_defined_primary_key:
229
+ anyOf:
230
+ - items:
231
+ items:
232
+ type: string
233
+ type: array
234
+ type: array
235
+ - type: 'null'
236
+ title: Source Defined Primary Key
237
+ namespace:
238
+ anyOf:
239
+ - type: string
240
+ - type: 'null'
241
+ title: Namespace
242
+ is_resumable:
243
+ anyOf:
244
+ - type: boolean
245
+ - type: 'null'
246
+ title: Is Resumable
247
+ is_file_based:
248
+ anyOf:
249
+ - type: boolean
250
+ - type: 'null'
251
+ title: Is File Based
252
+ type: object
253
+ required:
254
+ - name
255
+ - json_schema
256
+ - supported_sync_modes
257
+ title: AirbyteStream
258
+ AuxiliaryRequest:
259
+ properties:
260
+ title:
261
+ type: string
262
+ title: Title
263
+ type:
264
+ type: string
265
+ title: Type
266
+ description:
267
+ type: string
268
+ title: Description
269
+ request:
270
+ $ref: '#/components/schemas/HttpRequest'
271
+ response:
272
+ $ref: '#/components/schemas/HttpResponse'
273
+ type: object
274
+ required:
275
+ - title
276
+ - type
277
+ - description
278
+ - request
279
+ - response
280
+ title: AuxiliaryRequest
281
+ description: Auxiliary HTTP request made during stream processing.
282
+ CapabilitiesResponse:
283
+ properties:
284
+ custom_code_execution:
285
+ type: boolean
286
+ title: Custom Code Execution
287
+ type: object
288
+ required:
289
+ - custom_code_execution
290
+ title: CapabilitiesResponse
291
+ description: Capabilities of the manifest server.
292
+ CheckRequest:
293
+ properties:
294
+ manifest:
295
+ $ref: '#/components/schemas/Manifest'
296
+ config:
297
+ $ref: '#/components/schemas/ConnectorConfig'
298
+ type: object
299
+ required:
300
+ - manifest
301
+ - config
302
+ title: CheckRequest
303
+ description: Request to check a manifest.
304
+ CheckResponse:
305
+ properties:
306
+ success:
307
+ type: boolean
308
+ title: Success
309
+ message:
310
+ anyOf:
311
+ - type: string
312
+ - type: 'null'
313
+ title: Message
314
+ type: object
315
+ required:
316
+ - success
317
+ title: CheckResponse
318
+ description: Response to check a manifest.
319
+ ConnectorConfig:
320
+ properties: {}
321
+ additionalProperties: true
322
+ type: object
323
+ title: ConnectorConfig
324
+ description: Base connector configuration model. Allows client generation to
325
+ replace with proper JsonNode types.
326
+ DiscoverRequest:
327
+ properties:
328
+ manifest:
329
+ $ref: '#/components/schemas/Manifest'
330
+ config:
331
+ $ref: '#/components/schemas/ConnectorConfig'
332
+ type: object
333
+ required:
334
+ - manifest
335
+ - config
336
+ title: DiscoverRequest
337
+ description: Request to discover a manifest.
338
+ DiscoverResponse:
339
+ properties:
340
+ catalog:
341
+ $ref: '#/components/schemas/AirbyteCatalog'
342
+ type: object
343
+ required:
344
+ - catalog
345
+ title: DiscoverResponse
346
+ description: Response to discover a manifest.
347
+ FullResolveRequest:
348
+ properties:
349
+ manifest:
350
+ $ref: '#/components/schemas/Manifest'
351
+ config:
352
+ $ref: '#/components/schemas/ConnectorConfig'
353
+ stream_limit:
354
+ type: integer
355
+ maximum: 100.0
356
+ minimum: 1.0
357
+ title: Stream Limit
358
+ default: 100
359
+ type: object
360
+ required:
361
+ - manifest
362
+ - config
363
+ title: FullResolveRequest
364
+ description: Request to fully resolve a manifest.
365
+ HTTPValidationError:
366
+ properties:
367
+ detail:
368
+ items:
369
+ $ref: '#/components/schemas/ValidationError'
370
+ type: array
371
+ title: Detail
372
+ type: object
373
+ title: HTTPValidationError
374
+ HttpRequest:
375
+ properties:
376
+ url:
377
+ type: string
378
+ title: Url
379
+ headers:
380
+ anyOf:
381
+ - type: object
382
+ - type: 'null'
383
+ title: Headers
384
+ http_method:
385
+ type: string
386
+ title: Http Method
387
+ body:
388
+ anyOf:
389
+ - type: string
390
+ - type: 'null'
391
+ title: Body
392
+ type: object
393
+ required:
394
+ - url
395
+ - headers
396
+ - http_method
397
+ title: HttpRequest
398
+ description: HTTP request details.
399
+ HttpResponse:
400
+ properties:
401
+ status:
402
+ type: integer
403
+ title: Status
404
+ body:
405
+ anyOf:
406
+ - type: string
407
+ - type: 'null'
408
+ title: Body
409
+ headers:
410
+ anyOf:
411
+ - type: object
412
+ - type: 'null'
413
+ title: Headers
414
+ type: object
415
+ required:
416
+ - status
417
+ title: HttpResponse
418
+ description: HTTP response details.
419
+ LogMessage:
420
+ properties:
421
+ message:
422
+ type: string
423
+ title: Message
424
+ level:
425
+ type: string
426
+ title: Level
427
+ internal_message:
428
+ anyOf:
429
+ - type: string
430
+ - type: 'null'
431
+ title: Internal Message
432
+ stacktrace:
433
+ anyOf:
434
+ - type: string
435
+ - type: 'null'
436
+ title: Stacktrace
437
+ type: object
438
+ required:
439
+ - message
440
+ - level
441
+ title: LogMessage
442
+ description: Log message from stream processing.
443
+ Manifest:
444
+ properties: {}
445
+ additionalProperties: true
446
+ type: object
447
+ title: Manifest
448
+ description: Base manifest model. Allows client generation to replace with proper
449
+ JsonNode types.
450
+ ManifestResponse:
451
+ properties:
452
+ manifest:
453
+ $ref: '#/components/schemas/Manifest'
454
+ type: object
455
+ required:
456
+ - manifest
457
+ title: ManifestResponse
458
+ description: Response containing a manifest.
459
+ ResolveRequest:
460
+ properties:
461
+ manifest:
462
+ $ref: '#/components/schemas/Manifest'
463
+ type: object
464
+ required:
465
+ - manifest
466
+ title: ResolveRequest
467
+ description: Request to resolve a manifest.
468
+ StreamRead:
469
+ properties:
470
+ logs:
471
+ items:
472
+ $ref: '#/components/schemas/LogMessage'
473
+ type: array
474
+ title: Logs
475
+ slices:
476
+ items:
477
+ $ref: '#/components/schemas/StreamReadSlices'
478
+ type: array
479
+ title: Slices
480
+ test_read_limit_reached:
481
+ type: boolean
482
+ title: Test Read Limit Reached
483
+ auxiliary_requests:
484
+ items:
485
+ $ref: '#/components/schemas/AuxiliaryRequest'
486
+ type: array
487
+ title: Auxiliary Requests
488
+ inferred_schema:
489
+ anyOf:
490
+ - type: object
491
+ - type: 'null'
492
+ title: Inferred Schema
493
+ inferred_datetime_formats:
494
+ anyOf:
495
+ - additionalProperties:
496
+ type: string
497
+ type: object
498
+ - type: 'null'
499
+ title: Inferred Datetime Formats
500
+ latest_config_update:
501
+ anyOf:
502
+ - type: object
503
+ - type: 'null'
504
+ title: Latest Config Update
505
+ type: object
506
+ required:
507
+ - logs
508
+ - slices
509
+ - test_read_limit_reached
510
+ - auxiliary_requests
511
+ - inferred_schema
512
+ - inferred_datetime_formats
513
+ - latest_config_update
514
+ title: StreamRead
515
+ description: Complete stream read response with properly typed fields.
516
+ StreamReadPages:
517
+ properties:
518
+ records:
519
+ items: {}
520
+ type: array
521
+ title: Records
522
+ request:
523
+ anyOf:
524
+ - $ref: '#/components/schemas/HttpRequest'
525
+ - type: 'null'
526
+ response:
527
+ anyOf:
528
+ - $ref: '#/components/schemas/HttpResponse'
529
+ - type: 'null'
530
+ type: object
531
+ required:
532
+ - records
533
+ title: StreamReadPages
534
+ description: Pages of data read from a stream slice.
535
+ StreamReadSlices:
536
+ properties:
537
+ pages:
538
+ items:
539
+ $ref: '#/components/schemas/StreamReadPages'
540
+ type: array
541
+ title: Pages
542
+ slice_descriptor:
543
+ anyOf:
544
+ - type: string
545
+ - type: 'null'
546
+ title: Slice Descriptor
547
+ state:
548
+ anyOf:
549
+ - items:
550
+ type: object
551
+ type: array
552
+ - type: 'null'
553
+ title: State
554
+ auxiliary_requests:
555
+ anyOf:
556
+ - items:
557
+ $ref: '#/components/schemas/AuxiliaryRequest'
558
+ type: array
559
+ - type: 'null'
560
+ title: Auxiliary Requests
561
+ type: object
562
+ required:
563
+ - pages
564
+ - slice_descriptor
565
+ title: StreamReadSlices
566
+ description: Slices of data read from a stream.
567
+ StreamTestReadRequest:
568
+ properties:
569
+ manifest:
570
+ $ref: '#/components/schemas/Manifest'
571
+ config:
572
+ $ref: '#/components/schemas/ConnectorConfig'
573
+ stream_name:
574
+ type: string
575
+ title: Stream Name
576
+ state:
577
+ items: {}
578
+ type: array
579
+ title: State
580
+ default: []
581
+ custom_components_code:
582
+ anyOf:
583
+ - type: string
584
+ - type: 'null'
585
+ title: Custom Components Code
586
+ record_limit:
587
+ type: integer
588
+ maximum: 5000.0
589
+ minimum: 1.0
590
+ title: Record Limit
591
+ default: 100
592
+ page_limit:
593
+ type: integer
594
+ maximum: 20.0
595
+ minimum: 1.0
596
+ title: Page Limit
597
+ default: 5
598
+ slice_limit:
599
+ type: integer
600
+ maximum: 20.0
601
+ minimum: 1.0
602
+ title: Slice Limit
603
+ default: 5
604
+ type: object
605
+ required:
606
+ - manifest
607
+ - config
608
+ - stream_name
609
+ title: StreamTestReadRequest
610
+ description: Request to test read from a specific stream.
611
+ SyncMode:
612
+ type: string
613
+ enum:
614
+ - full_refresh
615
+ - incremental
616
+ title: SyncMode
617
+ ValidationError:
618
+ properties:
619
+ loc:
620
+ items:
621
+ anyOf:
622
+ - type: string
623
+ - type: integer
624
+ type: array
625
+ title: Location
626
+ msg:
627
+ type: string
628
+ title: Message
629
+ type:
630
+ type: string
631
+ title: Error Type
632
+ type: object
633
+ required:
634
+ - loc
635
+ - msg
636
+ - type
637
+ title: ValidationError
638
+ securitySchemes:
639
+ HTTPBearer:
640
+ type: http
641
+ scheme: bearer
File without changes
@@ -0,0 +1,25 @@
1
+ import os
2
+ from typing import Any, Dict
3
+
4
+ from fastapi import APIRouter
5
+
6
+ from ..api_models.capabilities import CapabilitiesResponse
7
+
8
+ router = APIRouter(
9
+ prefix="/capabilities",
10
+ tags=["capabilities"],
11
+ )
12
+
13
+
14
+ @router.get("/", operation_id="getCapabilities")
15
+ def get_capabilities() -> CapabilitiesResponse:
16
+ """
17
+ Get the capabilities available for the manifest server.
18
+
19
+ Returns:
20
+ Dict containing the service capabilities including custom code execution support.
21
+ """
22
+ # Read the same environment variable as the connector builder server
23
+ enable_unsafe_code = os.getenv("AIRBYTE_ENABLE_UNSAFE_CODE", "false").lower() == "true"
24
+
25
+ return CapabilitiesResponse(custom_code_execution=enable_unsafe_code)
@@ -0,0 +1,13 @@
1
+ from typing import Dict
2
+
3
+ from fastapi import APIRouter
4
+
5
+ router = APIRouter(
6
+ prefix="/health",
7
+ tags=["health"],
8
+ )
9
+
10
+
11
+ @router.get("/")
12
+ def health() -> Dict[str, str]:
13
+ return {"status": "ok"}