vectoramp 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vectoramp/__init__.py +45 -0
- vectoramp/client.py +133 -0
- vectoramp/connections.py +123 -0
- vectoramp/embeddings.py +39 -0
- vectoramp/exceptions.py +22 -0
- vectoramp/py.typed +0 -0
- vectoramp/resources.py +1531 -0
- vectoramp/sources.py +506 -0
- vectoramp/transport.py +198 -0
- vectoramp/types.py +145 -0
- vectoramp-0.1.0.dist-info/METADATA +464 -0
- vectoramp-0.1.0.dist-info/RECORD +15 -0
- vectoramp-0.1.0.dist-info/WHEEL +4 -0
- vectoramp-0.1.0.dist-info/licenses/LICENSE +201 -0
- vectoramp-0.1.0.dist-info/licenses/NOTICE +8 -0
vectoramp/sources.py
ADDED
|
@@ -0,0 +1,506 @@
|
|
|
1
|
+
"""Typed ingestion source builders for the VectorAmp SDK."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from typing import Any, Literal, Mapping, Optional, Protocol, Sequence, Union, runtime_checkable
|
|
7
|
+
from urllib.parse import urlparse
|
|
8
|
+
|
|
9
|
+
from .types import JSON
|
|
10
|
+
|
|
11
|
+
KnownSourceType = Literal["s3", "web", "gcs", "gdrive", "file_upload", "jira", "confluence"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@runtime_checkable
|
|
15
|
+
class SourceBuilder(Protocol):
|
|
16
|
+
"""Object that can be converted to a source-create request."""
|
|
17
|
+
|
|
18
|
+
def to_create_request(self) -> JSON:
|
|
19
|
+
"""Return the body fields accepted by the source-create API."""
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class GenericSource:
|
|
24
|
+
"""Escape hatch for source types not yet modeled by the SDK.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
name: Source name. Defaults to the normalized ``source_type``.
|
|
28
|
+
source_type: API source type. Required.
|
|
29
|
+
config: Source-specific configuration.
|
|
30
|
+
description: Optional source description.
|
|
31
|
+
metadata: Optional source metadata.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
name: Optional[str] = None
|
|
35
|
+
source_type: str = ""
|
|
36
|
+
config: Mapping[str, Any] = field(default_factory=dict)
|
|
37
|
+
description: Optional[str] = None
|
|
38
|
+
metadata: Optional[Mapping[str, Any]] = None
|
|
39
|
+
|
|
40
|
+
def to_create_request(self) -> JSON:
|
|
41
|
+
"""Return source-create request fields for this generic source."""
|
|
42
|
+
if not self.source_type:
|
|
43
|
+
raise ValueError("GenericSource requires source_type.")
|
|
44
|
+
return _source_body(
|
|
45
|
+
name=self.name or _default_source_name(self.source_type),
|
|
46
|
+
source_type=self.source_type,
|
|
47
|
+
config=self.config,
|
|
48
|
+
description=self.description,
|
|
49
|
+
metadata=self.metadata,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True)
|
|
54
|
+
class WebSource:
|
|
55
|
+
"""Web crawler ingestion source.
|
|
56
|
+
|
|
57
|
+
Args:
|
|
58
|
+
name: Source name. Defaults to ``web-{host-or-path}`` from the first URL.
|
|
59
|
+
start_urls: Crawl entry URLs. At least one URL is required.
|
|
60
|
+
max_depth: Optional crawl depth limit.
|
|
61
|
+
max_pages: Optional page limit.
|
|
62
|
+
allowed_domains: Optional domain allow-list.
|
|
63
|
+
include_patterns: Optional URL include patterns.
|
|
64
|
+
exclude_patterns: Optional URL exclude patterns.
|
|
65
|
+
crawl_delay_seconds: Optional delay between requests.
|
|
66
|
+
sync_mode: Optional sync mode. Omitted from the request when ``None`` so
|
|
67
|
+
the server applies its default (``"incremental"``).
|
|
68
|
+
description: Optional source description.
|
|
69
|
+
metadata: Optional source metadata.
|
|
70
|
+
config_extra: Optional extra config fields merged into the request.
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
name: Optional[str] = None
|
|
74
|
+
start_urls: Sequence[str] = ()
|
|
75
|
+
max_depth: Optional[int] = None
|
|
76
|
+
max_pages: Optional[int] = None
|
|
77
|
+
allowed_domains: Optional[Sequence[str]] = None
|
|
78
|
+
include_patterns: Optional[Sequence[str]] = None
|
|
79
|
+
exclude_patterns: Optional[Sequence[str]] = None
|
|
80
|
+
crawl_delay_seconds: Optional[float] = None
|
|
81
|
+
include_assets: Optional[bool] = None
|
|
82
|
+
max_assets_per_page: Optional[int] = None
|
|
83
|
+
sync_mode: Optional[str] = None
|
|
84
|
+
description: Optional[str] = None
|
|
85
|
+
metadata: Optional[Mapping[str, Any]] = None
|
|
86
|
+
config_extra: Optional[Mapping[str, Any]] = None
|
|
87
|
+
|
|
88
|
+
def to_create_request(self) -> JSON:
|
|
89
|
+
"""Return source-create request fields for this web source."""
|
|
90
|
+
if not self.start_urls:
|
|
91
|
+
raise ValueError("WebSource requires at least one start URL.")
|
|
92
|
+
config: JSON = {"start_urls": list(self.start_urls)}
|
|
93
|
+
_set_optional(config, "sync_mode", self.sync_mode)
|
|
94
|
+
_set_optional(config, "max_depth", self.max_depth)
|
|
95
|
+
_set_optional(config, "max_pages", self.max_pages)
|
|
96
|
+
_set_optional_sequence(config, "allowed_domains", self.allowed_domains)
|
|
97
|
+
_set_optional_sequence(config, "include_patterns", self.include_patterns)
|
|
98
|
+
_set_optional_sequence(config, "exclude_patterns", self.exclude_patterns)
|
|
99
|
+
_set_optional(config, "crawl_delay_seconds", self.crawl_delay_seconds)
|
|
100
|
+
_set_optional(config, "include_assets", self.include_assets)
|
|
101
|
+
_set_optional(config, "max_assets_per_page", self.max_assets_per_page)
|
|
102
|
+
_merge_extra(config, self.config_extra)
|
|
103
|
+
return _source_body(
|
|
104
|
+
name=self.name or _default_web_source_name(self.start_urls),
|
|
105
|
+
source_type="web",
|
|
106
|
+
config=config,
|
|
107
|
+
description=self.description,
|
|
108
|
+
metadata=self.metadata,
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
@dataclass(frozen=True)
|
|
113
|
+
class S3Source:
|
|
114
|
+
"""Amazon S3 ingestion source.
|
|
115
|
+
|
|
116
|
+
Args:
|
|
117
|
+
name: Source name. Defaults to ``s3-{bucket}``.
|
|
118
|
+
bucket: S3 bucket name. Required.
|
|
119
|
+
region: AWS region. Defaults to ``"us-east-1"``.
|
|
120
|
+
prefix: Optional key prefix.
|
|
121
|
+
sync_mode: Optional sync mode. Omitted from the request when ``None`` so
|
|
122
|
+
the server applies its default (``"incremental"``).
|
|
123
|
+
access_key_id: Optional AWS access key id.
|
|
124
|
+
secret_access_key: Optional AWS secret access key.
|
|
125
|
+
role_arn: Optional role ARN.
|
|
126
|
+
endpoint_url: Optional S3-compatible endpoint URL.
|
|
127
|
+
file_patterns: Optional file pattern filters.
|
|
128
|
+
max_file_size_mb: Optional max file size in MB.
|
|
129
|
+
description: Optional source description.
|
|
130
|
+
metadata: Optional source metadata.
|
|
131
|
+
config_extra: Optional extra config fields merged into the request.
|
|
132
|
+
"""
|
|
133
|
+
|
|
134
|
+
name: Optional[str] = None
|
|
135
|
+
bucket: str = ""
|
|
136
|
+
region: str = "us-east-1"
|
|
137
|
+
prefix: Optional[str] = None
|
|
138
|
+
sync_mode: Optional[str] = None
|
|
139
|
+
access_key_id: Optional[str] = None
|
|
140
|
+
secret_access_key: Optional[str] = None
|
|
141
|
+
role_arn: Optional[str] = None
|
|
142
|
+
endpoint_url: Optional[str] = None
|
|
143
|
+
file_patterns: Optional[Sequence[str]] = None
|
|
144
|
+
max_file_size_mb: Optional[int] = None
|
|
145
|
+
description: Optional[str] = None
|
|
146
|
+
metadata: Optional[Mapping[str, Any]] = None
|
|
147
|
+
config_extra: Optional[Mapping[str, Any]] = None
|
|
148
|
+
|
|
149
|
+
def to_create_request(self) -> JSON:
|
|
150
|
+
"""Return source-create request fields for this S3 source."""
|
|
151
|
+
if not self.bucket:
|
|
152
|
+
raise ValueError("S3Source requires bucket.")
|
|
153
|
+
config: JSON = {"bucket": self.bucket, "region": self.region}
|
|
154
|
+
_set_optional(config, "sync_mode", self.sync_mode)
|
|
155
|
+
_set_optional(config, "prefix", self.prefix)
|
|
156
|
+
_set_optional(config, "access_key_id", self.access_key_id)
|
|
157
|
+
_set_optional(config, "secret_access_key", self.secret_access_key)
|
|
158
|
+
_set_optional(config, "role_arn", self.role_arn)
|
|
159
|
+
_set_optional(config, "endpoint_url", self.endpoint_url)
|
|
160
|
+
_set_optional_sequence(config, "file_patterns", self.file_patterns)
|
|
161
|
+
_set_optional(config, "max_file_size_mb", self.max_file_size_mb)
|
|
162
|
+
_merge_extra(config, self.config_extra)
|
|
163
|
+
return _source_body(
|
|
164
|
+
name=self.name or _default_source_name("s3", self.bucket),
|
|
165
|
+
source_type="s3",
|
|
166
|
+
config=config,
|
|
167
|
+
description=self.description,
|
|
168
|
+
metadata=self.metadata,
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
@dataclass(frozen=True)
|
|
173
|
+
class GCSSource:
|
|
174
|
+
"""Google Cloud Storage ingestion source."""
|
|
175
|
+
|
|
176
|
+
name: Optional[str] = None
|
|
177
|
+
bucket: str = ""
|
|
178
|
+
prefix: Optional[str] = None
|
|
179
|
+
project_id: Optional[str] = None
|
|
180
|
+
credentials_json: Optional[Mapping[str, Any]] = None
|
|
181
|
+
sync_mode: Optional[str] = None
|
|
182
|
+
file_patterns: Optional[Sequence[str]] = None
|
|
183
|
+
max_file_size_mb: Optional[int] = None
|
|
184
|
+
connection_id: Optional[str] = None
|
|
185
|
+
description: Optional[str] = None
|
|
186
|
+
metadata: Optional[Mapping[str, Any]] = None
|
|
187
|
+
config_extra: Optional[Mapping[str, Any]] = None
|
|
188
|
+
|
|
189
|
+
def to_create_request(self) -> JSON:
|
|
190
|
+
if not self.bucket:
|
|
191
|
+
raise ValueError("GCSSource requires bucket.")
|
|
192
|
+
config: JSON = {"bucket": self.bucket}
|
|
193
|
+
_set_optional(config, "sync_mode", self.sync_mode)
|
|
194
|
+
_set_optional(config, "prefix", self.prefix)
|
|
195
|
+
_set_optional(config, "project_id", self.project_id)
|
|
196
|
+
if self.credentials_json is not None:
|
|
197
|
+
config["credentials_json"] = dict(self.credentials_json)
|
|
198
|
+
_set_optional_sequence(config, "file_patterns", self.file_patterns)
|
|
199
|
+
_set_optional(config, "max_file_size_mb", self.max_file_size_mb)
|
|
200
|
+
_set_optional(config, "connection_id", self.connection_id)
|
|
201
|
+
_merge_extra(config, self.config_extra)
|
|
202
|
+
return _source_body(
|
|
203
|
+
name=self.name or _default_source_name("gcs", self.bucket),
|
|
204
|
+
source_type="gcs",
|
|
205
|
+
config=config,
|
|
206
|
+
description=self.description,
|
|
207
|
+
metadata=self.metadata,
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
@dataclass(frozen=True)
|
|
212
|
+
class GoogleDriveSource:
|
|
213
|
+
"""Google Drive ingestion source.
|
|
214
|
+
|
|
215
|
+
Args:
|
|
216
|
+
name: Source name. Defaults to ``gdrive-{first-folder-or-file-id}`` or
|
|
217
|
+
``gdrive``.
|
|
218
|
+
folder_ids: Optional folder ids to ingest.
|
|
219
|
+
file_ids: Optional file ids to ingest.
|
|
220
|
+
auth_mode: Auth mode. Defaults to ``"oauth"``.
|
|
221
|
+
oauth_credentials: Optional OAuth credential payload.
|
|
222
|
+
include_shared_drives: Optional shared-drive toggle.
|
|
223
|
+
sync_mode: Optional sync mode. Omitted from the request when ``None`` so
|
|
224
|
+
the server applies its default (``"incremental"``).
|
|
225
|
+
service_account_json: Optional service-account credential payload.
|
|
226
|
+
credentials_json: Optional generic credential payload.
|
|
227
|
+
connection_id: Optional id of a stored OAuth connection (see
|
|
228
|
+
``client.connections``) to authorize the source with.
|
|
229
|
+
description: Optional source description.
|
|
230
|
+
metadata: Optional source metadata.
|
|
231
|
+
config_extra: Optional extra config fields merged into the request.
|
|
232
|
+
"""
|
|
233
|
+
|
|
234
|
+
name: Optional[str] = None
|
|
235
|
+
folder_ids: Optional[Sequence[str]] = None
|
|
236
|
+
file_ids: Optional[Sequence[str]] = None
|
|
237
|
+
auth_mode: str = "oauth"
|
|
238
|
+
oauth_credentials: Optional[Mapping[str, Any]] = None
|
|
239
|
+
include_shared_drives: Optional[bool] = None
|
|
240
|
+
sync_mode: Optional[str] = None
|
|
241
|
+
service_account_json: Optional[Mapping[str, Any]] = None
|
|
242
|
+
credentials_json: Optional[Mapping[str, Any]] = None
|
|
243
|
+
connection_id: Optional[str] = None
|
|
244
|
+
description: Optional[str] = None
|
|
245
|
+
metadata: Optional[Mapping[str, Any]] = None
|
|
246
|
+
config_extra: Optional[Mapping[str, Any]] = None
|
|
247
|
+
|
|
248
|
+
def to_create_request(self) -> JSON:
|
|
249
|
+
"""Return source-create request fields for this Google Drive source."""
|
|
250
|
+
config: JSON = {"auth_mode": self.auth_mode}
|
|
251
|
+
_set_optional(config, "sync_mode", self.sync_mode)
|
|
252
|
+
_set_optional_sequence(config, "folder_ids", self.folder_ids)
|
|
253
|
+
_set_optional_sequence(config, "file_ids", self.file_ids)
|
|
254
|
+
_set_optional(config, "include_shared_drives", self.include_shared_drives)
|
|
255
|
+
if self.oauth_credentials is not None:
|
|
256
|
+
config["oauth_credentials"] = dict(self.oauth_credentials)
|
|
257
|
+
if self.service_account_json is not None:
|
|
258
|
+
config["service_account_json"] = dict(self.service_account_json)
|
|
259
|
+
if self.credentials_json is not None:
|
|
260
|
+
config["credentials_json"] = dict(self.credentials_json)
|
|
261
|
+
_set_optional(config, "connection_id", self.connection_id)
|
|
262
|
+
_merge_extra(config, self.config_extra)
|
|
263
|
+
return _source_body(
|
|
264
|
+
name=self.name or _default_google_drive_source_name(self.folder_ids, self.file_ids),
|
|
265
|
+
source_type="gdrive",
|
|
266
|
+
config=config,
|
|
267
|
+
description=self.description,
|
|
268
|
+
metadata=self.metadata,
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
@dataclass(frozen=True)
|
|
273
|
+
class FileUploadSource:
|
|
274
|
+
"""File-upload ingestion source.
|
|
275
|
+
|
|
276
|
+
This models the source record only. Use ``dataset.ingest_files`` or
|
|
277
|
+
``client.ingestion.ingest_files`` for the local presigned-upload flow; those
|
|
278
|
+
helpers auto-create a ``file_upload`` source when uploading files.
|
|
279
|
+
|
|
280
|
+
Args:
|
|
281
|
+
name: Source name. Defaults to ``"vectoramp-python-upload"``.
|
|
282
|
+
storage_provider: Upload storage provider. Defaults to ``"s3"``.
|
|
283
|
+
sync_mode: Sync mode. Defaults to ``"full"``.
|
|
284
|
+
description: Optional source description.
|
|
285
|
+
metadata: Optional source metadata.
|
|
286
|
+
config_extra: Optional extra config fields merged into the request.
|
|
287
|
+
"""
|
|
288
|
+
|
|
289
|
+
name: str = "vectoramp-python-upload"
|
|
290
|
+
storage_provider: str = "s3"
|
|
291
|
+
sync_mode: str = "full"
|
|
292
|
+
description: Optional[str] = None
|
|
293
|
+
metadata: Optional[Mapping[str, Any]] = None
|
|
294
|
+
config_extra: Optional[Mapping[str, Any]] = None
|
|
295
|
+
|
|
296
|
+
def to_create_request(self) -> JSON:
|
|
297
|
+
"""Return source-create request fields for this file-upload source."""
|
|
298
|
+
config: JSON = {
|
|
299
|
+
"storage_provider": self.storage_provider,
|
|
300
|
+
"sync_mode": self.sync_mode,
|
|
301
|
+
}
|
|
302
|
+
_merge_extra(config, self.config_extra)
|
|
303
|
+
return _source_body(
|
|
304
|
+
name=self.name,
|
|
305
|
+
source_type="file_upload",
|
|
306
|
+
config=config,
|
|
307
|
+
description=self.description,
|
|
308
|
+
metadata=self.metadata,
|
|
309
|
+
)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
@dataclass(frozen=True)
|
|
313
|
+
class JiraSource:
|
|
314
|
+
"""Jira ingestion source.
|
|
315
|
+
|
|
316
|
+
``include_comments`` defaults to true. ``sync_mode`` is omitted from the
|
|
317
|
+
request when ``None`` so the server applies its default (``"incremental"``).
|
|
318
|
+
"""
|
|
319
|
+
|
|
320
|
+
name: Optional[str] = None
|
|
321
|
+
cloud_id: str = ""
|
|
322
|
+
access_token: Optional[str] = None
|
|
323
|
+
project_keys: Optional[Sequence[str]] = None
|
|
324
|
+
jql: Optional[str] = None
|
|
325
|
+
include_comments: bool = True
|
|
326
|
+
sync_mode: Optional[str] = None
|
|
327
|
+
connection_id: Optional[str] = None
|
|
328
|
+
description: Optional[str] = None
|
|
329
|
+
metadata: Optional[Mapping[str, Any]] = None
|
|
330
|
+
config_extra: Optional[Mapping[str, Any]] = None
|
|
331
|
+
|
|
332
|
+
def to_create_request(self) -> JSON:
|
|
333
|
+
if not self.cloud_id:
|
|
334
|
+
raise ValueError("JiraSource requires cloud_id.")
|
|
335
|
+
config: JSON = {
|
|
336
|
+
"cloud_id": self.cloud_id,
|
|
337
|
+
"include_comments": self.include_comments,
|
|
338
|
+
}
|
|
339
|
+
_set_optional(config, "sync_mode", self.sync_mode)
|
|
340
|
+
_set_optional(config, "access_token", self.access_token)
|
|
341
|
+
_set_optional_sequence(config, "project_keys", self.project_keys)
|
|
342
|
+
_set_optional(config, "jql", self.jql)
|
|
343
|
+
_set_optional(config, "connection_id", self.connection_id)
|
|
344
|
+
_merge_extra(config, self.config_extra)
|
|
345
|
+
hint = self.project_keys[0] if self.project_keys else self.cloud_id
|
|
346
|
+
return _source_body(
|
|
347
|
+
name=self.name or _default_source_name("jira", hint),
|
|
348
|
+
source_type="jira",
|
|
349
|
+
config=config,
|
|
350
|
+
description=self.description,
|
|
351
|
+
metadata=self.metadata,
|
|
352
|
+
)
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
@dataclass(frozen=True)
|
|
356
|
+
class ConfluenceSource:
|
|
357
|
+
"""Confluence ingestion source.
|
|
358
|
+
|
|
359
|
+
Confluence spaces and pages from an Atlassian Cloud site. Provide either
|
|
360
|
+
``cloud_id`` (Atlassian OAuth cloud/site id) or ``base_url`` (e.g.
|
|
361
|
+
``"https://company.atlassian.net"``). For basic auth, set ``username`` and
|
|
362
|
+
``api_token``; for OAuth, set ``oauth_credentials``.
|
|
363
|
+
|
|
364
|
+
Args:
|
|
365
|
+
name: Source name. Defaults to ``confluence-{cloud-id-or-host}``.
|
|
366
|
+
cloud_id: Atlassian OAuth cloud/site id. Required unless ``base_url`` set.
|
|
367
|
+
base_url: Atlassian site base URL. Required unless ``cloud_id`` set.
|
|
368
|
+
auth_mode: Auth mode. Defaults to ``"basic"``.
|
|
369
|
+
username: Optional basic-auth username (typically an account email).
|
|
370
|
+
api_token: Optional basic-auth API token.
|
|
371
|
+
oauth_credentials: Optional Atlassian OAuth credential payload.
|
|
372
|
+
spaces: Optional space keys to ingest. Empty means all accessible spaces.
|
|
373
|
+
include_attachments: Whether to ingest attachments. Defaults to ``False``.
|
|
374
|
+
sync_mode: Optional sync mode. Omitted from the request when ``None`` so
|
|
375
|
+
the server applies its default (``"incremental"``).
|
|
376
|
+
connection_id: Optional id of a stored OAuth connection (see
|
|
377
|
+
``client.connections``) to authorize the source with.
|
|
378
|
+
description: Optional source description.
|
|
379
|
+
metadata: Optional source metadata.
|
|
380
|
+
config_extra: Optional extra config fields merged into the request.
|
|
381
|
+
"""
|
|
382
|
+
|
|
383
|
+
name: Optional[str] = None
|
|
384
|
+
cloud_id: Optional[str] = None
|
|
385
|
+
base_url: Optional[str] = None
|
|
386
|
+
auth_mode: str = "basic"
|
|
387
|
+
username: Optional[str] = None
|
|
388
|
+
api_token: Optional[str] = None
|
|
389
|
+
oauth_credentials: Optional[Mapping[str, Any]] = None
|
|
390
|
+
spaces: Optional[Sequence[str]] = None
|
|
391
|
+
include_attachments: bool = False
|
|
392
|
+
sync_mode: Optional[str] = None
|
|
393
|
+
connection_id: Optional[str] = None
|
|
394
|
+
description: Optional[str] = None
|
|
395
|
+
metadata: Optional[Mapping[str, Any]] = None
|
|
396
|
+
config_extra: Optional[Mapping[str, Any]] = None
|
|
397
|
+
|
|
398
|
+
def to_create_request(self) -> JSON:
|
|
399
|
+
"""Return source-create request fields for this Confluence source."""
|
|
400
|
+
if not self.cloud_id and not self.base_url:
|
|
401
|
+
raise ValueError("ConfluenceSource requires cloud_id or base_url.")
|
|
402
|
+
config: JSON = {
|
|
403
|
+
"auth_mode": self.auth_mode,
|
|
404
|
+
"include_attachments": self.include_attachments,
|
|
405
|
+
}
|
|
406
|
+
_set_optional(config, "cloud_id", self.cloud_id)
|
|
407
|
+
_set_optional(config, "base_url", self.base_url)
|
|
408
|
+
_set_optional(config, "username", self.username)
|
|
409
|
+
_set_optional(config, "api_token", self.api_token)
|
|
410
|
+
if self.oauth_credentials is not None:
|
|
411
|
+
config["oauth_credentials"] = dict(self.oauth_credentials)
|
|
412
|
+
_set_optional_sequence(config, "spaces", self.spaces)
|
|
413
|
+
_set_optional(config, "sync_mode", self.sync_mode)
|
|
414
|
+
_set_optional(config, "connection_id", self.connection_id)
|
|
415
|
+
_merge_extra(config, self.config_extra)
|
|
416
|
+
hint = self.cloud_id or _confluence_host_hint(self.base_url)
|
|
417
|
+
return _source_body(
|
|
418
|
+
name=self.name or _default_source_name("confluence", hint),
|
|
419
|
+
source_type="confluence",
|
|
420
|
+
config=config,
|
|
421
|
+
description=self.description,
|
|
422
|
+
metadata=self.metadata,
|
|
423
|
+
)
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
TypedSource = Union[
|
|
427
|
+
GenericSource,
|
|
428
|
+
WebSource,
|
|
429
|
+
S3Source,
|
|
430
|
+
GCSSource,
|
|
431
|
+
GoogleDriveSource,
|
|
432
|
+
FileUploadSource,
|
|
433
|
+
JiraSource,
|
|
434
|
+
ConfluenceSource,
|
|
435
|
+
]
|
|
436
|
+
SourceInput = Union[str, TypedSource]
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def _source_body(
|
|
440
|
+
*,
|
|
441
|
+
name: str,
|
|
442
|
+
source_type: str,
|
|
443
|
+
config: Mapping[str, Any],
|
|
444
|
+
description: Optional[str],
|
|
445
|
+
metadata: Optional[Mapping[str, Any]],
|
|
446
|
+
) -> JSON:
|
|
447
|
+
body: JSON = {"name": name, "source_type": source_type, "config": dict(config)}
|
|
448
|
+
if description is not None:
|
|
449
|
+
body["description"] = description
|
|
450
|
+
if metadata is not None:
|
|
451
|
+
body["metadata"] = dict(metadata)
|
|
452
|
+
return body
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def _default_web_source_name(start_urls: Sequence[str]) -> str:
|
|
456
|
+
first_url = start_urls[0]
|
|
457
|
+
parsed = urlparse(first_url)
|
|
458
|
+
hint = parsed.netloc or parsed.path or first_url
|
|
459
|
+
return _default_source_name("web", hint)
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def _default_google_drive_source_name(
|
|
463
|
+
folder_ids: Optional[Sequence[str]], file_ids: Optional[Sequence[str]]
|
|
464
|
+
) -> str:
|
|
465
|
+
hint = None
|
|
466
|
+
if folder_ids:
|
|
467
|
+
hint = folder_ids[0]
|
|
468
|
+
elif file_ids:
|
|
469
|
+
hint = file_ids[0]
|
|
470
|
+
return _default_source_name("gdrive", hint)
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def _confluence_host_hint(base_url: Optional[str]) -> Optional[str]:
|
|
474
|
+
if not base_url:
|
|
475
|
+
return None
|
|
476
|
+
parsed = urlparse(base_url)
|
|
477
|
+
return parsed.netloc or parsed.path or base_url
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
def _default_source_name(source_type: str, hint: Optional[str] = None) -> str:
|
|
481
|
+
parts = [source_type.replace("_", "-")]
|
|
482
|
+
if hint:
|
|
483
|
+
slug = _slugify(str(hint))
|
|
484
|
+
if slug:
|
|
485
|
+
parts.append(slug)
|
|
486
|
+
return "-".join(parts)
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def _slugify(value: str) -> str:
|
|
490
|
+
slug = "".join(char.lower() if char.isalnum() else "-" for char in value)
|
|
491
|
+
return "-".join(part for part in slug.split("-") if part)[:48]
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
def _set_optional(config: JSON, key: str, value: Any) -> None:
|
|
495
|
+
if value is not None:
|
|
496
|
+
config[key] = value
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def _set_optional_sequence(config: JSON, key: str, value: Optional[Sequence[str]]) -> None:
|
|
500
|
+
if value is not None:
|
|
501
|
+
config[key] = list(value)
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def _merge_extra(config: JSON, extra: Optional[Mapping[str, Any]]) -> None:
|
|
505
|
+
if extra is not None:
|
|
506
|
+
config.update(dict(extra))
|
vectoramp/transport.py
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
"""Transport abstraction for VectorAmp.
|
|
2
|
+
|
|
3
|
+
The public client depends on this small interface rather than directly on httpx. REST is the
|
|
4
|
+
current implementation; a future gRPC transport can implement the same request/stream surface.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
from abc import ABC, abstractmethod
|
|
11
|
+
from typing import Any, Dict, Iterator, Mapping, Optional
|
|
12
|
+
from urllib.parse import urljoin
|
|
13
|
+
|
|
14
|
+
import httpx
|
|
15
|
+
|
|
16
|
+
from .exceptions import APIError, AuthenticationError
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class BaseTransport(ABC):
|
|
20
|
+
"""Minimal transport contract used by resource clients."""
|
|
21
|
+
|
|
22
|
+
@abstractmethod
|
|
23
|
+
def request(
|
|
24
|
+
self,
|
|
25
|
+
method: str,
|
|
26
|
+
path: str,
|
|
27
|
+
*,
|
|
28
|
+
params: Optional[Mapping[str, Any]] = None,
|
|
29
|
+
json_body: Any = None,
|
|
30
|
+
headers: Optional[Mapping[str, str]] = None,
|
|
31
|
+
) -> Any:
|
|
32
|
+
"""Send one request and return decoded response content."""
|
|
33
|
+
|
|
34
|
+
@abstractmethod
|
|
35
|
+
def stream(
|
|
36
|
+
self,
|
|
37
|
+
method: str,
|
|
38
|
+
path: str,
|
|
39
|
+
*,
|
|
40
|
+
json_body: Any = None,
|
|
41
|
+
headers: Optional[Mapping[str, str]] = None,
|
|
42
|
+
) -> Iterator[Dict[str, Any]]:
|
|
43
|
+
"""Yield JSON-decoded Server-Sent Event data chunks."""
|
|
44
|
+
|
|
45
|
+
def download(
|
|
46
|
+
self,
|
|
47
|
+
method: str,
|
|
48
|
+
path: str,
|
|
49
|
+
*,
|
|
50
|
+
params: Optional[Mapping[str, Any]] = None,
|
|
51
|
+
headers: Optional[Mapping[str, str]] = None,
|
|
52
|
+
) -> bytes:
|
|
53
|
+
"""Send one request and return raw response bytes."""
|
|
54
|
+
raise NotImplementedError("transport does not support raw downloads")
|
|
55
|
+
|
|
56
|
+
@abstractmethod
|
|
57
|
+
def close(self) -> None:
|
|
58
|
+
"""Release transport resources."""
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class RestTransport(BaseTransport):
|
|
62
|
+
"""httpx-backed REST transport."""
|
|
63
|
+
|
|
64
|
+
def __init__(
|
|
65
|
+
self,
|
|
66
|
+
*,
|
|
67
|
+
api_key: str,
|
|
68
|
+
base_url: str = "https://api.vectoramp.com",
|
|
69
|
+
timeout: float = 30.0,
|
|
70
|
+
client: Optional[httpx.Client] = None,
|
|
71
|
+
) -> None:
|
|
72
|
+
if not api_key:
|
|
73
|
+
raise AuthenticationError(
|
|
74
|
+
"An API key is required. Pass api_key or set VECTORAMP_API_KEY."
|
|
75
|
+
)
|
|
76
|
+
self.api_key = api_key
|
|
77
|
+
self.base_url = base_url.rstrip("/") + "/"
|
|
78
|
+
self._owns_client = client is None
|
|
79
|
+
self.client = client or httpx.Client(timeout=timeout)
|
|
80
|
+
|
|
81
|
+
def request(
|
|
82
|
+
self,
|
|
83
|
+
method: str,
|
|
84
|
+
path: str,
|
|
85
|
+
*,
|
|
86
|
+
params: Optional[Mapping[str, Any]] = None,
|
|
87
|
+
json_body: Any = None,
|
|
88
|
+
headers: Optional[Mapping[str, str]] = None,
|
|
89
|
+
) -> Any:
|
|
90
|
+
response = self.client.request(
|
|
91
|
+
method,
|
|
92
|
+
self._url(path),
|
|
93
|
+
params=self._clean(params),
|
|
94
|
+
json=json_body,
|
|
95
|
+
headers=self._headers(headers),
|
|
96
|
+
)
|
|
97
|
+
return self._decode_response(response)
|
|
98
|
+
|
|
99
|
+
def stream(
|
|
100
|
+
self,
|
|
101
|
+
method: str,
|
|
102
|
+
path: str,
|
|
103
|
+
*,
|
|
104
|
+
json_body: Any = None,
|
|
105
|
+
headers: Optional[Mapping[str, str]] = None,
|
|
106
|
+
) -> Iterator[Dict[str, Any]]:
|
|
107
|
+
request_headers = self._headers({"Accept": "text/event-stream", **dict(headers or {})})
|
|
108
|
+
with self.client.stream(
|
|
109
|
+
method, self._url(path), json=json_body, headers=request_headers
|
|
110
|
+
) as response:
|
|
111
|
+
self._raise_for_status(response)
|
|
112
|
+
for line in response.iter_lines():
|
|
113
|
+
if not line or line.startswith(":"):
|
|
114
|
+
continue
|
|
115
|
+
if line.startswith("data:"):
|
|
116
|
+
payload = line[5:].strip()
|
|
117
|
+
if payload == "[DONE]":
|
|
118
|
+
break
|
|
119
|
+
yield json.loads(payload)
|
|
120
|
+
|
|
121
|
+
def download(
|
|
122
|
+
self,
|
|
123
|
+
method: str,
|
|
124
|
+
path: str,
|
|
125
|
+
*,
|
|
126
|
+
params: Optional[Mapping[str, Any]] = None,
|
|
127
|
+
headers: Optional[Mapping[str, str]] = None,
|
|
128
|
+
) -> bytes:
|
|
129
|
+
"""Send one request and return raw response bytes."""
|
|
130
|
+
response = self.client.request(
|
|
131
|
+
method,
|
|
132
|
+
self._url(path),
|
|
133
|
+
params=self._clean(params),
|
|
134
|
+
headers=self._headers(headers),
|
|
135
|
+
follow_redirects=True,
|
|
136
|
+
)
|
|
137
|
+
self._raise_for_status(response)
|
|
138
|
+
return response.content
|
|
139
|
+
|
|
140
|
+
def put_url(
|
|
141
|
+
self,
|
|
142
|
+
url: str,
|
|
143
|
+
*,
|
|
144
|
+
content: bytes,
|
|
145
|
+
content_type: Optional[str] = None,
|
|
146
|
+
) -> None:
|
|
147
|
+
"""Upload bytes to a presigned URL without VectorAmp auth headers."""
|
|
148
|
+
headers = {"Content-Type": content_type} if content_type else None
|
|
149
|
+
response = self.client.put(url, content=content, headers=headers)
|
|
150
|
+
self._raise_for_status(response)
|
|
151
|
+
|
|
152
|
+
def close(self) -> None:
|
|
153
|
+
if self._owns_client:
|
|
154
|
+
self.client.close()
|
|
155
|
+
|
|
156
|
+
def _url(self, path: str) -> str:
|
|
157
|
+
return urljoin(self.base_url, path.lstrip("/"))
|
|
158
|
+
|
|
159
|
+
def _headers(self, headers: Optional[Mapping[str, str]] = None) -> Dict[str, str]:
|
|
160
|
+
merged = {"X-API-Key": self.api_key, "User-Agent": "vectoramp-python/0.1.0"}
|
|
161
|
+
if headers:
|
|
162
|
+
merged.update(headers)
|
|
163
|
+
return merged
|
|
164
|
+
|
|
165
|
+
@staticmethod
|
|
166
|
+
def _clean(params: Optional[Mapping[str, Any]]) -> Optional[Dict[str, Any]]:
|
|
167
|
+
if params is None:
|
|
168
|
+
return None
|
|
169
|
+
return {key: value for key, value in params.items() if value is not None}
|
|
170
|
+
|
|
171
|
+
@classmethod
|
|
172
|
+
def _decode_response(cls, response: httpx.Response) -> Any:
|
|
173
|
+
cls._raise_for_status(response)
|
|
174
|
+
if response.status_code == 204 or not response.content:
|
|
175
|
+
return None
|
|
176
|
+
content_type = response.headers.get("content-type", "")
|
|
177
|
+
if "application/json" in content_type:
|
|
178
|
+
return response.json()
|
|
179
|
+
try:
|
|
180
|
+
return response.json()
|
|
181
|
+
except ValueError:
|
|
182
|
+
return response.text
|
|
183
|
+
|
|
184
|
+
@staticmethod
|
|
185
|
+
def _raise_for_status(response: httpx.Response) -> None:
|
|
186
|
+
if response.is_success:
|
|
187
|
+
return
|
|
188
|
+
try:
|
|
189
|
+
payload = response.json()
|
|
190
|
+
message = (
|
|
191
|
+
payload.get("message")
|
|
192
|
+
or payload.get("detail")
|
|
193
|
+
or payload.get("error")
|
|
194
|
+
or str(payload)
|
|
195
|
+
)
|
|
196
|
+
except ValueError:
|
|
197
|
+
message = response.text
|
|
198
|
+
raise APIError(response.status_code, message, response=response)
|