vectoramp 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vectoramp-0.3.0 → vectoramp-0.4.0}/CHANGELOG.md +7 -0
- {vectoramp-0.3.0 → vectoramp-0.4.0}/PKG-INFO +27 -3
- {vectoramp-0.3.0 → vectoramp-0.4.0}/README.md +25 -1
- {vectoramp-0.3.0 → vectoramp-0.4.0}/pyproject.toml +1 -1
- {vectoramp-0.3.0 → vectoramp-0.4.0}/src/vectoramp/__init__.py +4 -0
- {vectoramp-0.3.0 → vectoramp-0.4.0}/src/vectoramp/resources.py +120 -1
- {vectoramp-0.3.0 → vectoramp-0.4.0}/src/vectoramp/sources.py +195 -1
- {vectoramp-0.3.0 → vectoramp-0.4.0}/src/vectoramp/transport.py +1 -1
- {vectoramp-0.3.0 → vectoramp-0.4.0}/tests/test_client.py +2 -2
- {vectoramp-0.3.0 → vectoramp-0.4.0}/tests/test_connections_and_sources.py +216 -0
- {vectoramp-0.3.0 → vectoramp-0.4.0}/.github/images/logo-full-dark.svg +0 -0
- {vectoramp-0.3.0 → vectoramp-0.4.0}/.github/images/logo-full-light.svg +0 -0
- {vectoramp-0.3.0 → vectoramp-0.4.0}/.github/workflows/ci.yml +0 -0
- {vectoramp-0.3.0 → vectoramp-0.4.0}/.github/workflows/publish.yml +0 -0
- {vectoramp-0.3.0 → vectoramp-0.4.0}/.gitignore +0 -0
- {vectoramp-0.3.0 → vectoramp-0.4.0}/CONTRIBUTING.md +0 -0
- {vectoramp-0.3.0 → vectoramp-0.4.0}/LICENSE +0 -0
- {vectoramp-0.3.0 → vectoramp-0.4.0}/NOTICE +0 -0
- {vectoramp-0.3.0 → vectoramp-0.4.0}/SECURITY.md +0 -0
- {vectoramp-0.3.0 → vectoramp-0.4.0}/src/vectoramp/client.py +0 -0
- {vectoramp-0.3.0 → vectoramp-0.4.0}/src/vectoramp/connections.py +0 -0
- {vectoramp-0.3.0 → vectoramp-0.4.0}/src/vectoramp/embeddings.py +0 -0
- {vectoramp-0.3.0 → vectoramp-0.4.0}/src/vectoramp/exceptions.py +0 -0
- {vectoramp-0.3.0 → vectoramp-0.4.0}/src/vectoramp/py.typed +0 -0
- {vectoramp-0.3.0 → vectoramp-0.4.0}/src/vectoramp/types.py +0 -0
|
@@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file.
|
|
|
4
4
|
|
|
5
5
|
This project follows semantic versioning.
|
|
6
6
|
|
|
7
|
+
## [0.4.0] - 2026-08-20
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- Add `GitHubSource` and `GitLabSource` typed source builders.
|
|
12
|
+
- Add `client.sources.create_github(...)` and `client.sources.create_gitlab(...)`.
|
|
13
|
+
|
|
7
14
|
## [0.3.0] - 2026-07-20
|
|
8
15
|
|
|
9
16
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: vectoramp
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Python SDK for the VectorAmp API
|
|
5
5
|
Project-URL: Homepage, https://vectoramp.com
|
|
6
6
|
Project-URL: Documentation, https://docs.vectoramp.com
|
|
@@ -332,16 +332,40 @@ gdrive = client.sources.create_google_drive(
|
|
|
332
332
|
include_shared_drives=True,
|
|
333
333
|
)
|
|
334
334
|
|
|
335
|
+
github = client.sources.create_github(
|
|
336
|
+
installation_id=42, # from the VectorAmp GitHub App install
|
|
337
|
+
repositories=["acme/api", "acme/web"],
|
|
338
|
+
ref_mode="active", # server default: recently pushed branches
|
|
339
|
+
include_pull_requests=True, # default
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
gitlab = client.sources.create_gitlab(
|
|
343
|
+
projects=["mygroup/myproject"], # and/or groups=["mygroup"]
|
|
344
|
+
auth_mode="token", # defaults to "oauth"
|
|
345
|
+
access_token="…",
|
|
346
|
+
gitlab_url="https://gitlab.com", # override for self-managed
|
|
347
|
+
)
|
|
348
|
+
|
|
335
349
|
upload_source = client.sources.create_file_upload()
|
|
336
350
|
```
|
|
337
351
|
|
|
352
|
+
GitHub sources authenticate through the VectorAmp GitHub App, so the SDK never
|
|
353
|
+
takes a GitHub token. Install the app from the Sources page in the VectorAmp
|
|
354
|
+
app, then pass the `installation_id` it reports along with the `owner/repo`
|
|
355
|
+
names you want ingested. Both connectors ingest repository files plus active
|
|
356
|
+
branches, pull/merge requests, and review discussions; turn parts off with
|
|
357
|
+
`include_pull_requests` / `include_merge_requests`, `include_review_threads`,
|
|
358
|
+
and `include_direct_commits`, and narrow file selection with `include_globs`,
|
|
359
|
+
`exclude_globs`, and `max_file_size_bytes`.
|
|
360
|
+
|
|
338
361
|
`sync_mode` is omitted unless you set it, so the server applies its default of
|
|
339
362
|
`"incremental"` for the connectors that support it. Pass `sync_mode="full"` to
|
|
340
363
|
force a full re-sync.
|
|
341
364
|
|
|
342
365
|
The supported typed source classes are `WebSource`, `S3Source`, `GCSSource`,
|
|
343
366
|
`GoogleDriveSource` (`source_type="gdrive"`), `JiraSource`, `ConfluenceSource`,
|
|
344
|
-
|
|
367
|
+
`GitHubSource`, `GitLabSource`, and `FileUploadSource`
|
|
368
|
+
(`source_type="file_upload"`). Use `GenericSource` as an
|
|
345
369
|
escape hatch when the API supports a source type before the SDK has a dedicated
|
|
346
370
|
class:
|
|
347
371
|
|
|
@@ -302,16 +302,40 @@ gdrive = client.sources.create_google_drive(
|
|
|
302
302
|
include_shared_drives=True,
|
|
303
303
|
)
|
|
304
304
|
|
|
305
|
+
github = client.sources.create_github(
|
|
306
|
+
installation_id=42, # from the VectorAmp GitHub App install
|
|
307
|
+
repositories=["acme/api", "acme/web"],
|
|
308
|
+
ref_mode="active", # server default: recently pushed branches
|
|
309
|
+
include_pull_requests=True, # default
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
gitlab = client.sources.create_gitlab(
|
|
313
|
+
projects=["mygroup/myproject"], # and/or groups=["mygroup"]
|
|
314
|
+
auth_mode="token", # defaults to "oauth"
|
|
315
|
+
access_token="…",
|
|
316
|
+
gitlab_url="https://gitlab.com", # override for self-managed
|
|
317
|
+
)
|
|
318
|
+
|
|
305
319
|
upload_source = client.sources.create_file_upload()
|
|
306
320
|
```
|
|
307
321
|
|
|
322
|
+
GitHub sources authenticate through the VectorAmp GitHub App, so the SDK never
|
|
323
|
+
takes a GitHub token. Install the app from the Sources page in the VectorAmp
|
|
324
|
+
app, then pass the `installation_id` it reports along with the `owner/repo`
|
|
325
|
+
names you want ingested. Both connectors ingest repository files plus active
|
|
326
|
+
branches, pull/merge requests, and review discussions; turn parts off with
|
|
327
|
+
`include_pull_requests` / `include_merge_requests`, `include_review_threads`,
|
|
328
|
+
and `include_direct_commits`, and narrow file selection with `include_globs`,
|
|
329
|
+
`exclude_globs`, and `max_file_size_bytes`.
|
|
330
|
+
|
|
308
331
|
`sync_mode` is omitted unless you set it, so the server applies its default of
|
|
309
332
|
`"incremental"` for the connectors that support it. Pass `sync_mode="full"` to
|
|
310
333
|
force a full re-sync.
|
|
311
334
|
|
|
312
335
|
The supported typed source classes are `WebSource`, `S3Source`, `GCSSource`,
|
|
313
336
|
`GoogleDriveSource` (`source_type="gdrive"`), `JiraSource`, `ConfluenceSource`,
|
|
314
|
-
|
|
337
|
+
`GitHubSource`, `GitLabSource`, and `FileUploadSource`
|
|
338
|
+
(`source_type="file_upload"`). Use `GenericSource` as an
|
|
315
339
|
escape hatch when the API supports a source type before the SDK has a dedicated
|
|
316
340
|
class:
|
|
317
341
|
|
|
@@ -16,6 +16,8 @@ from .sources import (
|
|
|
16
16
|
FileUploadSource,
|
|
17
17
|
GCSSource,
|
|
18
18
|
GenericSource,
|
|
19
|
+
GitHubSource,
|
|
20
|
+
GitLabSource,
|
|
19
21
|
GoogleDriveSource,
|
|
20
22
|
JiraSource,
|
|
21
23
|
S3Source,
|
|
@@ -33,6 +35,8 @@ __all__ = [
|
|
|
33
35
|
"FileUploadSource",
|
|
34
36
|
"GCSSource",
|
|
35
37
|
"GenericSource",
|
|
38
|
+
"GitHubSource",
|
|
39
|
+
"GitLabSource",
|
|
36
40
|
"GoogleDriveSource",
|
|
37
41
|
"JiraSource",
|
|
38
42
|
"MetadataFieldType",
|
|
@@ -20,6 +20,8 @@ from .sources import (
|
|
|
20
20
|
ConfluenceSource,
|
|
21
21
|
FileUploadSource,
|
|
22
22
|
GCSSource,
|
|
23
|
+
GitHubSource,
|
|
24
|
+
GitLabSource,
|
|
23
25
|
GoogleDriveSource,
|
|
24
26
|
JiraSource,
|
|
25
27
|
S3Source,
|
|
@@ -710,7 +712,7 @@ class DatasetsResource:
|
|
|
710
712
|
if write_concern is not None:
|
|
711
713
|
body["write_concern"] = dict(write_concern)
|
|
712
714
|
return self._transport.request(
|
|
713
|
-
"DELETE", f"/
|
|
715
|
+
"DELETE", f"/datasets/{dataset_id}/vectors", json_body=body
|
|
714
716
|
)
|
|
715
717
|
|
|
716
718
|
def embed(
|
|
@@ -1243,6 +1245,123 @@ class IngestionResource:
|
|
|
1243
1245
|
)
|
|
1244
1246
|
)
|
|
1245
1247
|
|
|
1248
|
+
def create_github(
|
|
1249
|
+
self,
|
|
1250
|
+
*,
|
|
1251
|
+
installation_id: int,
|
|
1252
|
+
repositories: Sequence[str],
|
|
1253
|
+
name: Optional[str] = None,
|
|
1254
|
+
ref_mode: Optional[str] = None,
|
|
1255
|
+
refs: Optional[Sequence[str]] = None,
|
|
1256
|
+
excluded_refs: Optional[Sequence[str]] = None,
|
|
1257
|
+
active_branch_days: Optional[int] = None,
|
|
1258
|
+
include_pull_requests: Optional[bool] = None,
|
|
1259
|
+
include_review_threads: Optional[bool] = None,
|
|
1260
|
+
include_direct_commits: Optional[bool] = None,
|
|
1261
|
+
include_globs: Optional[Sequence[str]] = None,
|
|
1262
|
+
exclude_globs: Optional[Sequence[str]] = None,
|
|
1263
|
+
max_file_size_bytes: Optional[int] = None,
|
|
1264
|
+
sync_mode: Optional[str] = None,
|
|
1265
|
+
description: Optional[str] = None,
|
|
1266
|
+
metadata: Optional[Mapping[str, Any]] = None,
|
|
1267
|
+
config_extra: Optional[Mapping[str, Any]] = None,
|
|
1268
|
+
) -> JSON:
|
|
1269
|
+
"""Create a GitHub source backed by the VectorAmp GitHub App.
|
|
1270
|
+
|
|
1271
|
+
Install the VectorAmp GitHub App from the app's Sources page first and
|
|
1272
|
+
pass the resulting ``installation_id`` here; the SDK never handles a
|
|
1273
|
+
GitHub token. ``name`` defaults to ``github-{first-repository}``.
|
|
1274
|
+
Optional settings are omitted when ``None`` so the server applies its
|
|
1275
|
+
defaults (``ref_mode="active"``, pull requests, review threads, and
|
|
1276
|
+
direct commits included, ``sync_mode="incremental"``).
|
|
1277
|
+
|
|
1278
|
+
Returns:
|
|
1279
|
+
Created source JSON.
|
|
1280
|
+
"""
|
|
1281
|
+
return self.create_source(
|
|
1282
|
+
GitHubSource(
|
|
1283
|
+
name=name,
|
|
1284
|
+
installation_id=installation_id,
|
|
1285
|
+
repositories=repositories,
|
|
1286
|
+
ref_mode=ref_mode,
|
|
1287
|
+
refs=refs,
|
|
1288
|
+
excluded_refs=excluded_refs,
|
|
1289
|
+
active_branch_days=active_branch_days,
|
|
1290
|
+
include_pull_requests=include_pull_requests,
|
|
1291
|
+
include_review_threads=include_review_threads,
|
|
1292
|
+
include_direct_commits=include_direct_commits,
|
|
1293
|
+
include_globs=include_globs,
|
|
1294
|
+
exclude_globs=exclude_globs,
|
|
1295
|
+
max_file_size_bytes=max_file_size_bytes,
|
|
1296
|
+
sync_mode=sync_mode,
|
|
1297
|
+
description=description,
|
|
1298
|
+
metadata=metadata,
|
|
1299
|
+
config_extra=config_extra,
|
|
1300
|
+
)
|
|
1301
|
+
)
|
|
1302
|
+
|
|
1303
|
+
def create_gitlab(
|
|
1304
|
+
self,
|
|
1305
|
+
*,
|
|
1306
|
+
groups: Optional[Sequence[str]] = None,
|
|
1307
|
+
projects: Optional[Sequence[str]] = None,
|
|
1308
|
+
name: Optional[str] = None,
|
|
1309
|
+
auth_mode: str = "oauth",
|
|
1310
|
+
gitlab_url: str = "https://gitlab.com",
|
|
1311
|
+
access_token: Optional[str] = None,
|
|
1312
|
+
connection_id: Optional[str] = None,
|
|
1313
|
+
ref_mode: Optional[str] = None,
|
|
1314
|
+
refs: Optional[Sequence[str]] = None,
|
|
1315
|
+
excluded_refs: Optional[Sequence[str]] = None,
|
|
1316
|
+
active_branch_days: Optional[int] = None,
|
|
1317
|
+
include_merge_requests: Optional[bool] = None,
|
|
1318
|
+
include_review_threads: Optional[bool] = None,
|
|
1319
|
+
include_direct_commits: Optional[bool] = None,
|
|
1320
|
+
include_globs: Optional[Sequence[str]] = None,
|
|
1321
|
+
exclude_globs: Optional[Sequence[str]] = None,
|
|
1322
|
+
max_file_size_bytes: Optional[int] = None,
|
|
1323
|
+
sync_mode: Optional[str] = None,
|
|
1324
|
+
description: Optional[str] = None,
|
|
1325
|
+
metadata: Optional[Mapping[str, Any]] = None,
|
|
1326
|
+
config_extra: Optional[Mapping[str, Any]] = None,
|
|
1327
|
+
) -> JSON:
|
|
1328
|
+
"""Create a GitLab source for gitlab.com or a self-managed instance.
|
|
1329
|
+
|
|
1330
|
+
Pass at least one group or project. Authenticate with
|
|
1331
|
+
``auth_mode="token"`` plus ``access_token``, or with a stored OAuth
|
|
1332
|
+
``connection_id``. ``name`` defaults to
|
|
1333
|
+
``gitlab-{first-project-or-group}``. Optional settings are omitted when
|
|
1334
|
+
``None`` so the server applies its defaults.
|
|
1335
|
+
|
|
1336
|
+
Returns:
|
|
1337
|
+
Created source JSON.
|
|
1338
|
+
"""
|
|
1339
|
+
return self.create_source(
|
|
1340
|
+
GitLabSource(
|
|
1341
|
+
name=name,
|
|
1342
|
+
groups=groups,
|
|
1343
|
+
projects=projects,
|
|
1344
|
+
auth_mode=auth_mode,
|
|
1345
|
+
gitlab_url=gitlab_url,
|
|
1346
|
+
access_token=access_token,
|
|
1347
|
+
connection_id=connection_id,
|
|
1348
|
+
ref_mode=ref_mode,
|
|
1349
|
+
refs=refs,
|
|
1350
|
+
excluded_refs=excluded_refs,
|
|
1351
|
+
active_branch_days=active_branch_days,
|
|
1352
|
+
include_merge_requests=include_merge_requests,
|
|
1353
|
+
include_review_threads=include_review_threads,
|
|
1354
|
+
include_direct_commits=include_direct_commits,
|
|
1355
|
+
include_globs=include_globs,
|
|
1356
|
+
exclude_globs=exclude_globs,
|
|
1357
|
+
max_file_size_bytes=max_file_size_bytes,
|
|
1358
|
+
sync_mode=sync_mode,
|
|
1359
|
+
description=description,
|
|
1360
|
+
metadata=metadata,
|
|
1361
|
+
config_extra=config_extra,
|
|
1362
|
+
)
|
|
1363
|
+
)
|
|
1364
|
+
|
|
1246
1365
|
def create_file_upload(
|
|
1247
1366
|
self,
|
|
1248
1367
|
*,
|
|
@@ -8,7 +8,17 @@ from urllib.parse import urlparse
|
|
|
8
8
|
|
|
9
9
|
from .types import JSON
|
|
10
10
|
|
|
11
|
-
KnownSourceType = Literal[
|
|
11
|
+
KnownSourceType = Literal[
|
|
12
|
+
"s3",
|
|
13
|
+
"web",
|
|
14
|
+
"gcs",
|
|
15
|
+
"gdrive",
|
|
16
|
+
"file_upload",
|
|
17
|
+
"jira",
|
|
18
|
+
"confluence",
|
|
19
|
+
"github",
|
|
20
|
+
"gitlab",
|
|
21
|
+
]
|
|
12
22
|
|
|
13
23
|
|
|
14
24
|
@runtime_checkable
|
|
@@ -423,6 +433,188 @@ class ConfluenceSource:
|
|
|
423
433
|
)
|
|
424
434
|
|
|
425
435
|
|
|
436
|
+
@dataclass(frozen=True)
|
|
437
|
+
class GitHubSource:
|
|
438
|
+
"""GitHub ingestion source backed by the VectorAmp GitHub App.
|
|
439
|
+
|
|
440
|
+
Repositories, active branches, pull requests, and review discussions are
|
|
441
|
+
read through a GitHub App installation, so no token is passed to the SDK.
|
|
442
|
+
Install the VectorAmp GitHub App from the app's Sources page first; the
|
|
443
|
+
``installation_id`` shown there is the value this builder needs.
|
|
444
|
+
|
|
445
|
+
Args:
|
|
446
|
+
name: Source name. Defaults to ``github-{first-repository}``.
|
|
447
|
+
installation_id: GitHub App installation id. Required, must be positive.
|
|
448
|
+
repositories: ``owner/repo`` full names to ingest. At least one required.
|
|
449
|
+
ref_mode: Branch selection strategy — ``"active"`` (default on the
|
|
450
|
+
server), ``"default"``, or ``"explicit"``.
|
|
451
|
+
refs: Explicit branch names, used with ``ref_mode="explicit"``.
|
|
452
|
+
excluded_refs: Branch names to skip.
|
|
453
|
+
active_branch_days: How recently a branch must have been pushed to count
|
|
454
|
+
as active (1–90). Server default is 7.
|
|
455
|
+
include_pull_requests: Ingest pull requests. Server default is true.
|
|
456
|
+
include_review_threads: Ingest review discussions. Server default is true.
|
|
457
|
+
include_direct_commits: Ingest commits pushed outside a pull request.
|
|
458
|
+
Server default is true.
|
|
459
|
+
include_globs: Path globs to include. Server default is ``["**/*"]``.
|
|
460
|
+
exclude_globs: Path globs to skip.
|
|
461
|
+
max_file_size_bytes: Per-file size ceiling (1–25_000_000). Server
|
|
462
|
+
default is 1_000_000.
|
|
463
|
+
sync_mode: Optional sync mode. Omitted from the request when ``None`` so
|
|
464
|
+
the server applies its default (``"incremental"``).
|
|
465
|
+
description: Optional source description.
|
|
466
|
+
metadata: Optional source metadata.
|
|
467
|
+
config_extra: Optional extra config fields merged into the request.
|
|
468
|
+
"""
|
|
469
|
+
|
|
470
|
+
name: Optional[str] = None
|
|
471
|
+
installation_id: int = 0
|
|
472
|
+
repositories: Sequence[str] = ()
|
|
473
|
+
ref_mode: Optional[str] = None
|
|
474
|
+
refs: Optional[Sequence[str]] = None
|
|
475
|
+
excluded_refs: Optional[Sequence[str]] = None
|
|
476
|
+
active_branch_days: Optional[int] = None
|
|
477
|
+
include_pull_requests: Optional[bool] = None
|
|
478
|
+
include_review_threads: Optional[bool] = None
|
|
479
|
+
include_direct_commits: Optional[bool] = None
|
|
480
|
+
include_globs: Optional[Sequence[str]] = None
|
|
481
|
+
exclude_globs: Optional[Sequence[str]] = None
|
|
482
|
+
max_file_size_bytes: Optional[int] = None
|
|
483
|
+
sync_mode: Optional[str] = None
|
|
484
|
+
description: Optional[str] = None
|
|
485
|
+
metadata: Optional[Mapping[str, Any]] = None
|
|
486
|
+
config_extra: Optional[Mapping[str, Any]] = None
|
|
487
|
+
|
|
488
|
+
def to_create_request(self) -> JSON:
|
|
489
|
+
"""Return source-create request fields for this GitHub source."""
|
|
490
|
+
if not self.installation_id or self.installation_id <= 0:
|
|
491
|
+
raise ValueError("GitHubSource requires a positive installation_id.")
|
|
492
|
+
repositories = list(self.repositories)
|
|
493
|
+
if not repositories:
|
|
494
|
+
raise ValueError("GitHubSource requires at least one repository.")
|
|
495
|
+
config: JSON = {
|
|
496
|
+
"installation_id": int(self.installation_id),
|
|
497
|
+
"repositories": repositories,
|
|
498
|
+
}
|
|
499
|
+
_set_optional(config, "ref_mode", self.ref_mode)
|
|
500
|
+
_set_optional_sequence(config, "refs", self.refs)
|
|
501
|
+
_set_optional_sequence(config, "excluded_refs", self.excluded_refs)
|
|
502
|
+
_set_optional(config, "active_branch_days", self.active_branch_days)
|
|
503
|
+
_set_optional(config, "include_pull_requests", self.include_pull_requests)
|
|
504
|
+
_set_optional(config, "include_review_threads", self.include_review_threads)
|
|
505
|
+
_set_optional(config, "include_direct_commits", self.include_direct_commits)
|
|
506
|
+
_set_optional_sequence(config, "include_globs", self.include_globs)
|
|
507
|
+
_set_optional_sequence(config, "exclude_globs", self.exclude_globs)
|
|
508
|
+
_set_optional(config, "max_file_size_bytes", self.max_file_size_bytes)
|
|
509
|
+
_set_optional(config, "sync_mode", self.sync_mode)
|
|
510
|
+
_merge_extra(config, self.config_extra)
|
|
511
|
+
return _source_body(
|
|
512
|
+
name=self.name or _default_source_name("github", repositories[0]),
|
|
513
|
+
source_type="github",
|
|
514
|
+
config=config,
|
|
515
|
+
description=self.description,
|
|
516
|
+
metadata=self.metadata,
|
|
517
|
+
)
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
@dataclass(frozen=True)
|
|
521
|
+
class GitLabSource:
|
|
522
|
+
"""GitLab ingestion source for gitlab.com or a self-managed instance.
|
|
523
|
+
|
|
524
|
+
Projects, active branches, merge requests, and discussions are read with
|
|
525
|
+
either an access token (``auth_mode="token"`` plus ``access_token``) or a
|
|
526
|
+
stored OAuth connection (``connection_id``). At least one group or project
|
|
527
|
+
must be given.
|
|
528
|
+
|
|
529
|
+
Args:
|
|
530
|
+
name: Source name. Defaults to ``gitlab-{first-project-or-group}``.
|
|
531
|
+
groups: Group paths to ingest, e.g. ``["mygroup"]``. Required unless
|
|
532
|
+
``projects`` is given.
|
|
533
|
+
projects: Project paths with namespace, e.g. ``["mygroup/myproject"]``.
|
|
534
|
+
Required unless ``groups`` is given.
|
|
535
|
+
auth_mode: ``"oauth"`` (default) or ``"token"``.
|
|
536
|
+
gitlab_url: Instance base URL. Defaults to ``"https://gitlab.com"``.
|
|
537
|
+
access_token: Personal or group access token for ``auth_mode="token"``.
|
|
538
|
+
connection_id: Id of a stored OAuth connection to authorize the source
|
|
539
|
+
with, used instead of an inline token.
|
|
540
|
+
ref_mode: Branch selection strategy — ``"active"`` (default on the
|
|
541
|
+
server), ``"default"``, or ``"explicit"``.
|
|
542
|
+
refs: Explicit branch names, used with ``ref_mode="explicit"``.
|
|
543
|
+
excluded_refs: Branch names to skip.
|
|
544
|
+
active_branch_days: How recently a branch must have been pushed to count
|
|
545
|
+
as active (1–90). Server default is 7.
|
|
546
|
+
include_merge_requests: Ingest merge requests. Server default is true.
|
|
547
|
+
include_review_threads: Ingest review discussions. Server default is true.
|
|
548
|
+
include_direct_commits: Ingest commits pushed outside a merge request.
|
|
549
|
+
Server default is true.
|
|
550
|
+
include_globs: Path globs to include. Server default is ``["**/*"]``.
|
|
551
|
+
exclude_globs: Path globs to skip.
|
|
552
|
+
max_file_size_bytes: Per-file size ceiling (1–25_000_000). Server
|
|
553
|
+
default is 1_000_000.
|
|
554
|
+
sync_mode: Optional sync mode. Omitted from the request when ``None`` so
|
|
555
|
+
the server applies its default (``"incremental"``).
|
|
556
|
+
description: Optional source description.
|
|
557
|
+
metadata: Optional source metadata.
|
|
558
|
+
config_extra: Optional extra config fields merged into the request.
|
|
559
|
+
"""
|
|
560
|
+
|
|
561
|
+
name: Optional[str] = None
|
|
562
|
+
groups: Optional[Sequence[str]] = None
|
|
563
|
+
projects: Optional[Sequence[str]] = None
|
|
564
|
+
auth_mode: str = "oauth"
|
|
565
|
+
gitlab_url: str = "https://gitlab.com"
|
|
566
|
+
access_token: Optional[str] = None
|
|
567
|
+
connection_id: Optional[str] = None
|
|
568
|
+
ref_mode: Optional[str] = None
|
|
569
|
+
refs: Optional[Sequence[str]] = None
|
|
570
|
+
excluded_refs: Optional[Sequence[str]] = None
|
|
571
|
+
active_branch_days: Optional[int] = None
|
|
572
|
+
include_merge_requests: Optional[bool] = None
|
|
573
|
+
include_review_threads: Optional[bool] = None
|
|
574
|
+
include_direct_commits: Optional[bool] = None
|
|
575
|
+
include_globs: Optional[Sequence[str]] = None
|
|
576
|
+
exclude_globs: Optional[Sequence[str]] = None
|
|
577
|
+
max_file_size_bytes: Optional[int] = None
|
|
578
|
+
sync_mode: Optional[str] = None
|
|
579
|
+
description: Optional[str] = None
|
|
580
|
+
metadata: Optional[Mapping[str, Any]] = None
|
|
581
|
+
config_extra: Optional[Mapping[str, Any]] = None
|
|
582
|
+
|
|
583
|
+
def to_create_request(self) -> JSON:
|
|
584
|
+
"""Return source-create request fields for this GitLab source."""
|
|
585
|
+
groups = list(self.groups or ())
|
|
586
|
+
projects = list(self.projects or ())
|
|
587
|
+
if not groups and not projects:
|
|
588
|
+
raise ValueError("GitLabSource requires at least one group or project.")
|
|
589
|
+
config: JSON = {"auth_mode": self.auth_mode, "gitlab_url": self.gitlab_url}
|
|
590
|
+
if groups:
|
|
591
|
+
config["groups"] = groups
|
|
592
|
+
if projects:
|
|
593
|
+
config["projects"] = projects
|
|
594
|
+
_set_optional(config, "access_token", self.access_token)
|
|
595
|
+
_set_optional(config, "connection_id", self.connection_id)
|
|
596
|
+
_set_optional(config, "ref_mode", self.ref_mode)
|
|
597
|
+
_set_optional_sequence(config, "refs", self.refs)
|
|
598
|
+
_set_optional_sequence(config, "excluded_refs", self.excluded_refs)
|
|
599
|
+
_set_optional(config, "active_branch_days", self.active_branch_days)
|
|
600
|
+
_set_optional(config, "include_merge_requests", self.include_merge_requests)
|
|
601
|
+
_set_optional(config, "include_review_threads", self.include_review_threads)
|
|
602
|
+
_set_optional(config, "include_direct_commits", self.include_direct_commits)
|
|
603
|
+
_set_optional_sequence(config, "include_globs", self.include_globs)
|
|
604
|
+
_set_optional_sequence(config, "exclude_globs", self.exclude_globs)
|
|
605
|
+
_set_optional(config, "max_file_size_bytes", self.max_file_size_bytes)
|
|
606
|
+
_set_optional(config, "sync_mode", self.sync_mode)
|
|
607
|
+
_merge_extra(config, self.config_extra)
|
|
608
|
+
hint = projects[0] if projects else groups[0]
|
|
609
|
+
return _source_body(
|
|
610
|
+
name=self.name or _default_source_name("gitlab", hint),
|
|
611
|
+
source_type="gitlab",
|
|
612
|
+
config=config,
|
|
613
|
+
description=self.description,
|
|
614
|
+
metadata=self.metadata,
|
|
615
|
+
)
|
|
616
|
+
|
|
617
|
+
|
|
426
618
|
TypedSource = Union[
|
|
427
619
|
GenericSource,
|
|
428
620
|
WebSource,
|
|
@@ -432,6 +624,8 @@ TypedSource = Union[
|
|
|
432
624
|
FileUploadSource,
|
|
433
625
|
JiraSource,
|
|
434
626
|
ConfluenceSource,
|
|
627
|
+
GitHubSource,
|
|
628
|
+
GitLabSource,
|
|
435
629
|
]
|
|
436
630
|
SourceInput = Union[str, TypedSource]
|
|
437
631
|
|
|
@@ -157,7 +157,7 @@ class RestTransport(BaseTransport):
|
|
|
157
157
|
return urljoin(self.base_url, path.lstrip("/"))
|
|
158
158
|
|
|
159
159
|
def _headers(self, headers: Optional[Mapping[str, str]] = None) -> Dict[str, str]:
|
|
160
|
-
merged = {"X-API-Key": self.api_key, "User-Agent": "vectoramp-python/0.3.
|
|
160
|
+
merged = {"X-API-Key": self.api_key, "User-Agent": "vectoramp-python/0.3.1"}
|
|
161
161
|
if headers:
|
|
162
162
|
merged.update(headers)
|
|
163
163
|
return merged
|
|
@@ -251,10 +251,10 @@ def test_delete_vectors_payload_and_dataset_helper() -> None:
|
|
|
251
251
|
assert calls == [
|
|
252
252
|
(
|
|
253
253
|
"DELETE",
|
|
254
|
-
"/
|
|
254
|
+
"/datasets/ds_1/vectors",
|
|
255
255
|
{"ids": ["vec-a", 42], "write_concern": {"wait": True}},
|
|
256
256
|
),
|
|
257
|
-
("DELETE", "/
|
|
257
|
+
("DELETE", "/datasets/ds_1/vectors", {"ids": ["vec-c"]}),
|
|
258
258
|
]
|
|
259
259
|
|
|
260
260
|
|
|
@@ -9,6 +9,8 @@ import pytest
|
|
|
9
9
|
from vectoramp import (
|
|
10
10
|
ConfluenceSource,
|
|
11
11
|
GCSSource,
|
|
12
|
+
GitHubSource,
|
|
13
|
+
GitLabSource,
|
|
12
14
|
GoogleDriveSource,
|
|
13
15
|
JiraSource,
|
|
14
16
|
VectorAmp,
|
|
@@ -307,3 +309,217 @@ def test_connect_requires_id() -> None:
|
|
|
307
309
|
client = make_client(handler)
|
|
308
310
|
with pytest.raises(ValueError):
|
|
309
311
|
client.connections.connect("google_drive", poll_interval=0, timeout=5)
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
# --------------------------------------------------------------------------- #
|
|
315
|
+
# Source-control source builders (GitHub / GitLab)
|
|
316
|
+
# --------------------------------------------------------------------------- #
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def test_github_source_builds_minimal_config() -> None:
|
|
320
|
+
body = GitHubSource(installation_id=42, repositories=["octo/hello-world"]).to_create_request()
|
|
321
|
+
|
|
322
|
+
assert body == {
|
|
323
|
+
"name": "github-octo-hello-world",
|
|
324
|
+
"source_type": "github",
|
|
325
|
+
"config": {"installation_id": 42, "repositories": ["octo/hello-world"]},
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def test_github_source_serializes_all_options() -> None:
|
|
330
|
+
body = GitHubSource(
|
|
331
|
+
name="Platform repos",
|
|
332
|
+
installation_id=7,
|
|
333
|
+
repositories=["acme/api", "acme/web"],
|
|
334
|
+
ref_mode="explicit",
|
|
335
|
+
refs=["main", "release"],
|
|
336
|
+
excluded_refs=["wip"],
|
|
337
|
+
active_branch_days=30,
|
|
338
|
+
include_pull_requests=False,
|
|
339
|
+
include_review_threads=False,
|
|
340
|
+
include_direct_commits=False,
|
|
341
|
+
include_globs=["docs/**"],
|
|
342
|
+
exclude_globs=["**/*.lock"],
|
|
343
|
+
max_file_size_bytes=2_000_000,
|
|
344
|
+
sync_mode="full",
|
|
345
|
+
description="Platform code",
|
|
346
|
+
metadata={"team": "platform"},
|
|
347
|
+
config_extra={"experimental": True},
|
|
348
|
+
).to_create_request()
|
|
349
|
+
|
|
350
|
+
assert body == {
|
|
351
|
+
"name": "Platform repos",
|
|
352
|
+
"source_type": "github",
|
|
353
|
+
"config": {
|
|
354
|
+
"installation_id": 7,
|
|
355
|
+
"repositories": ["acme/api", "acme/web"],
|
|
356
|
+
"ref_mode": "explicit",
|
|
357
|
+
"refs": ["main", "release"],
|
|
358
|
+
"excluded_refs": ["wip"],
|
|
359
|
+
"active_branch_days": 30,
|
|
360
|
+
"include_pull_requests": False,
|
|
361
|
+
"include_review_threads": False,
|
|
362
|
+
"include_direct_commits": False,
|
|
363
|
+
"include_globs": ["docs/**"],
|
|
364
|
+
"exclude_globs": ["**/*.lock"],
|
|
365
|
+
"max_file_size_bytes": 2_000_000,
|
|
366
|
+
"sync_mode": "full",
|
|
367
|
+
"experimental": True,
|
|
368
|
+
},
|
|
369
|
+
"description": "Platform code",
|
|
370
|
+
"metadata": {"team": "platform"},
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
@pytest.mark.parametrize(
|
|
375
|
+
"source",
|
|
376
|
+
[
|
|
377
|
+
GitHubSource(repositories=["octo/hello-world"]),
|
|
378
|
+
GitHubSource(installation_id=-1, repositories=["octo/hello-world"]),
|
|
379
|
+
GitHubSource(installation_id=42),
|
|
380
|
+
],
|
|
381
|
+
)
|
|
382
|
+
def test_github_source_requires_installation_and_repositories(source: GitHubSource) -> None:
|
|
383
|
+
with pytest.raises(ValueError):
|
|
384
|
+
source.to_create_request()
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def test_gitlab_source_builds_minimal_config() -> None:
|
|
388
|
+
body = GitLabSource(projects=["mygroup/myproject"]).to_create_request()
|
|
389
|
+
|
|
390
|
+
assert body == {
|
|
391
|
+
"name": "gitlab-mygroup-myproject",
|
|
392
|
+
"source_type": "gitlab",
|
|
393
|
+
"config": {
|
|
394
|
+
"auth_mode": "oauth",
|
|
395
|
+
"gitlab_url": "https://gitlab.com",
|
|
396
|
+
"projects": ["mygroup/myproject"],
|
|
397
|
+
},
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def test_gitlab_source_names_from_group_when_no_project() -> None:
|
|
402
|
+
body = GitLabSource(groups=["mygroup"]).to_create_request()
|
|
403
|
+
|
|
404
|
+
assert body["name"] == "gitlab-mygroup"
|
|
405
|
+
assert body["config"]["groups"] == ["mygroup"]
|
|
406
|
+
assert "projects" not in body["config"]
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def test_gitlab_source_serializes_token_auth_and_options() -> None:
|
|
410
|
+
body = GitLabSource(
|
|
411
|
+
name="Self-managed",
|
|
412
|
+
groups=["platform"],
|
|
413
|
+
projects=["platform/api"],
|
|
414
|
+
auth_mode="token",
|
|
415
|
+
gitlab_url="https://gitlab.example.com",
|
|
416
|
+
access_token="glpat-secret",
|
|
417
|
+
ref_mode="active",
|
|
418
|
+
refs=["main"],
|
|
419
|
+
excluded_refs=["scratch"],
|
|
420
|
+
active_branch_days=14,
|
|
421
|
+
include_merge_requests=False,
|
|
422
|
+
include_review_threads=False,
|
|
423
|
+
include_direct_commits=False,
|
|
424
|
+
include_globs=["src/**"],
|
|
425
|
+
exclude_globs=["**/*.png"],
|
|
426
|
+
max_file_size_bytes=500_000,
|
|
427
|
+
sync_mode="full",
|
|
428
|
+
).to_create_request()
|
|
429
|
+
|
|
430
|
+
assert body["config"] == {
|
|
431
|
+
"auth_mode": "token",
|
|
432
|
+
"gitlab_url": "https://gitlab.example.com",
|
|
433
|
+
"groups": ["platform"],
|
|
434
|
+
"projects": ["platform/api"],
|
|
435
|
+
"access_token": "glpat-secret",
|
|
436
|
+
"ref_mode": "active",
|
|
437
|
+
"refs": ["main"],
|
|
438
|
+
"excluded_refs": ["scratch"],
|
|
439
|
+
"active_branch_days": 14,
|
|
440
|
+
"include_merge_requests": False,
|
|
441
|
+
"include_review_threads": False,
|
|
442
|
+
"include_direct_commits": False,
|
|
443
|
+
"include_globs": ["src/**"],
|
|
444
|
+
"exclude_globs": ["**/*.png"],
|
|
445
|
+
"max_file_size_bytes": 500_000,
|
|
446
|
+
"sync_mode": "full",
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def test_gitlab_source_supports_connection_id() -> None:
|
|
451
|
+
config = GitLabSource(projects=["g/p"], connection_id="conn_gl").to_create_request()["config"]
|
|
452
|
+
|
|
453
|
+
assert config["connection_id"] == "conn_gl"
|
|
454
|
+
assert "access_token" not in config
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def test_gitlab_source_requires_group_or_project() -> None:
|
|
458
|
+
with pytest.raises(ValueError):
|
|
459
|
+
GitLabSource().to_create_request()
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def test_create_github_and_gitlab_post_to_sources() -> None:
|
|
463
|
+
calls = []
|
|
464
|
+
|
|
465
|
+
def handler(request: httpx.Request) -> httpx.Response:
|
|
466
|
+
calls.append((request.url.path, json.loads(request.content)))
|
|
467
|
+
return json_response({"id": "src_new"}, status_code=201)
|
|
468
|
+
|
|
469
|
+
client = make_client(handler)
|
|
470
|
+
|
|
471
|
+
assert client.sources.create_github(
|
|
472
|
+
installation_id=42, repositories=["octo/hello-world"]
|
|
473
|
+
) == {"id": "src_new"}
|
|
474
|
+
assert client.sources.create_gitlab(
|
|
475
|
+
projects=["mygroup/myproject"], auth_mode="token", access_token="glpat-secret"
|
|
476
|
+
) == {"id": "src_new"}
|
|
477
|
+
|
|
478
|
+
assert calls[0] == (
|
|
479
|
+
"/ingestion/sources",
|
|
480
|
+
{
|
|
481
|
+
"name": "github-octo-hello-world",
|
|
482
|
+
"source_type": "github",
|
|
483
|
+
"config": {"installation_id": 42, "repositories": ["octo/hello-world"]},
|
|
484
|
+
},
|
|
485
|
+
)
|
|
486
|
+
assert calls[1] == (
|
|
487
|
+
"/ingestion/sources",
|
|
488
|
+
{
|
|
489
|
+
"name": "gitlab-mygroup-myproject",
|
|
490
|
+
"source_type": "gitlab",
|
|
491
|
+
"config": {
|
|
492
|
+
"auth_mode": "token",
|
|
493
|
+
"gitlab_url": "https://gitlab.com",
|
|
494
|
+
"projects": ["mygroup/myproject"],
|
|
495
|
+
"access_token": "glpat-secret",
|
|
496
|
+
},
|
|
497
|
+
},
|
|
498
|
+
)
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def test_validate_source_accepts_source_control_builders() -> None:
|
|
502
|
+
calls = []
|
|
503
|
+
|
|
504
|
+
def handler(request: httpx.Request) -> httpx.Response:
|
|
505
|
+
calls.append(json.loads(request.content))
|
|
506
|
+
return json_response({"valid": True})
|
|
507
|
+
|
|
508
|
+
client = make_client(handler)
|
|
509
|
+
|
|
510
|
+
client.sources.validate_source(GitHubSource(installation_id=1, repositories=["o/r"]))
|
|
511
|
+
client.sources.validate_source(GitLabSource(groups=["g"]))
|
|
512
|
+
|
|
513
|
+
# Validation bodies carry source_type + config only (no name).
|
|
514
|
+
assert calls[0] == {
|
|
515
|
+
"source_type": "github",
|
|
516
|
+
"config": {"installation_id": 1, "repositories": ["o/r"]},
|
|
517
|
+
}
|
|
518
|
+
assert calls[1] == {
|
|
519
|
+
"source_type": "gitlab",
|
|
520
|
+
"config": {
|
|
521
|
+
"auth_mode": "oauth",
|
|
522
|
+
"gitlab_url": "https://gitlab.com",
|
|
523
|
+
"groups": ["g"],
|
|
524
|
+
},
|
|
525
|
+
}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|