prefect-gitlab 0.2.1__tar.gz → 0.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect_gitlab-0.2.3/PKG-INFO +124 -0
- prefect_gitlab-0.2.3/README.md +84 -0
- {prefect-gitlab-0.2.1 → prefect_gitlab-0.2.3}/prefect_gitlab/__init__.py +1 -1
- prefect_gitlab-0.2.3/prefect_gitlab/_version.py +16 -0
- {prefect-gitlab-0.2.1 → prefect_gitlab-0.2.3}/prefect_gitlab/credentials.py +2 -1
- {prefect-gitlab-0.2.1 → prefect_gitlab-0.2.3}/prefect_gitlab/repositories.py +27 -2
- prefect_gitlab-0.2.3/prefect_gitlab.egg-info/PKG-INFO +124 -0
- {prefect-gitlab-0.2.1 → prefect_gitlab-0.2.3}/prefect_gitlab.egg-info/SOURCES.txt +4 -8
- {prefect-gitlab-0.2.1 → prefect_gitlab-0.2.3}/prefect_gitlab.egg-info/requires.txt +10 -10
- prefect_gitlab-0.2.3/pyproject.toml +81 -0
- prefect_gitlab-0.2.3/setup.cfg +4 -0
- prefect_gitlab-0.2.3/tests/conftest.py +23 -0
- {prefect-gitlab-0.2.1 → prefect_gitlab-0.2.3}/tests/test_repositories.py +46 -5
- prefect_gitlab-0.2.3/tests/test_version.py +9 -0
- prefect-gitlab-0.2.1/LICENSE +0 -202
- prefect-gitlab-0.2.1/MANIFEST.in +0 -14
- prefect-gitlab-0.2.1/PKG-INFO +0 -158
- prefect-gitlab-0.2.1/README.md +0 -117
- prefect-gitlab-0.2.1/prefect_gitlab/_version.py +0 -21
- prefect-gitlab-0.2.1/prefect_gitlab.egg-info/PKG-INFO +0 -158
- prefect-gitlab-0.2.1/requirements-dev.txt +0 -15
- prefect-gitlab-0.2.1/requirements.txt +0 -2
- prefect-gitlab-0.2.1/setup.cfg +0 -43
- prefect-gitlab-0.2.1/setup.py +0 -47
- prefect-gitlab-0.2.1/versioneer.py +0 -2163
- {prefect-gitlab-0.2.1 → prefect_gitlab-0.2.3}/prefect_gitlab.egg-info/dependency_links.txt +0 -0
- {prefect-gitlab-0.2.1 → prefect_gitlab-0.2.3}/prefect_gitlab.egg-info/entry_points.txt +0 -0
- {prefect-gitlab-0.2.1 → prefect_gitlab-0.2.3}/prefect_gitlab.egg-info/top_level.txt +0 -0
- {prefect-gitlab-0.2.1 → prefect_gitlab-0.2.3}/tests/test_credentials.py +0 -0
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: prefect-gitlab
|
|
3
|
+
Version: 0.2.3
|
|
4
|
+
Summary: A Prefect collection for working with GitLab repositories.
|
|
5
|
+
Author-email: "Prefect Technologies, Inc." <help@prefect.io>
|
|
6
|
+
License: Apache License 2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/PrefectHQ/prefect/tree/main/src/integrations/prefect-gitlab
|
|
8
|
+
Keywords: prefect
|
|
9
|
+
Classifier: Natural Language :: English
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Intended Audience :: System Administrators
|
|
12
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
20
|
+
Requires-Python: >=3.8
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
Requires-Dist: prefect>=2.13.5
|
|
23
|
+
Requires-Dist: python-gitlab>=3.12.0
|
|
24
|
+
Requires-Dist: tenacity>=8.2.3
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: aiohttp; extra == "dev"
|
|
27
|
+
Requires-Dist: coverage; extra == "dev"
|
|
28
|
+
Requires-Dist: interrogate; extra == "dev"
|
|
29
|
+
Requires-Dist: mkdocs-gen-files; extra == "dev"
|
|
30
|
+
Requires-Dist: mkdocs-material; extra == "dev"
|
|
31
|
+
Requires-Dist: mkdocs; extra == "dev"
|
|
32
|
+
Requires-Dist: mkdocstrings[python]; extra == "dev"
|
|
33
|
+
Requires-Dist: mock; python_version < "3.8" and extra == "dev"
|
|
34
|
+
Requires-Dist: mypy; extra == "dev"
|
|
35
|
+
Requires-Dist: pillow; extra == "dev"
|
|
36
|
+
Requires-Dist: pre-commit; extra == "dev"
|
|
37
|
+
Requires-Dist: pytest-asyncio; extra == "dev"
|
|
38
|
+
Requires-Dist: pytest; extra == "dev"
|
|
39
|
+
Requires-Dist: pytest-xdist; extra == "dev"
|
|
40
|
+
|
|
41
|
+
# prefect-gitlab
|
|
42
|
+
|
|
43
|
+
<p align="center">
|
|
44
|
+
<a href="https://pypi.python.org/pypi/prefect-gitlab/" alt="PyPI version">
|
|
45
|
+
<img alt="PyPI" src="https://img.shields.io/pypi/v/prefect-gitlab?color=26272B&labelColor=090422"></a>
|
|
46
|
+
<a href="https://pepy.tech/badge/prefect-gitlab/" alt="Downloads">
|
|
47
|
+
<img src="https://img.shields.io/pypi/dm/prefect-gitlab?color=26272B&labelColor=090422" /></a>
|
|
48
|
+
</p>
|
|
49
|
+
|
|
50
|
+
## Welcome!
|
|
51
|
+
|
|
52
|
+
`prefect-gitlab` is a Prefect collection for working with GitLab repositories.
|
|
53
|
+
|
|
54
|
+
## Getting Started
|
|
55
|
+
|
|
56
|
+
### Python setup
|
|
57
|
+
|
|
58
|
+
Requires an installation of Python 3.8 or higher.
|
|
59
|
+
|
|
60
|
+
We recommend using a Python virtual environment manager such as pipenv, conda, or virtualenv.
|
|
61
|
+
|
|
62
|
+
This integration is designed to work with Prefect 2.3.0 or higher. For more information about how to use Prefect, please refer to the [Prefect documentation](https://docs.prefect.io/).
|
|
63
|
+
|
|
64
|
+
### Installation
|
|
65
|
+
|
|
66
|
+
Install `prefect-gitlab` with `pip`:
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
pip install prefect-gitlab
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Then, register the [block types](https://docs.prefect.io/concepts/blocks/)) in this integration to view the storage block type on Prefect Cloud:
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
prefect block register -m prefect_gitlab
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Note, to use the `load` method on a block, you must already have a block document [saved](https://docs.prefect.io/concepts/blocks/).
|
|
79
|
+
|
|
80
|
+
## Creating a GitLab storage block
|
|
81
|
+
|
|
82
|
+
### In Python
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
from prefect_gitlab import GitLabRepository
|
|
86
|
+
|
|
87
|
+
# public GitLab repository
|
|
88
|
+
public_gitlab_block = GitLabRepository(
|
|
89
|
+
name="my-gitlab-block",
|
|
90
|
+
repository="https://gitlab.com/testing/my-repository.git"
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
public_gitlab_block.save()
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# specific branch or tag of a GitLab repository
|
|
97
|
+
branch_gitlab_block = GitLabRepository(
|
|
98
|
+
name="my-gitlab-block",
|
|
99
|
+
reference="branch-or-tag-name",
|
|
100
|
+
repository="https://gitlab.com/testing/my-repository.git"
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
branch_gitlab_block.save()
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
# Get all history of a specific branch or tag of a GitLab repository
|
|
107
|
+
branch_gitlab_block = GitLabRepository(
|
|
108
|
+
name="my-gitlab-block",
|
|
109
|
+
reference="branch-or-tag-name",
|
|
110
|
+
git_depth=None,
|
|
111
|
+
repository="https://gitlab.com/testing/my-repository.git"
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
branch_gitlab_block.save()
|
|
115
|
+
|
|
116
|
+
# private GitLab repository
|
|
117
|
+
private_gitlab_block = GitLabRepository(
|
|
118
|
+
name="my-private-gitlab-block",
|
|
119
|
+
repository="https://gitlab.com/testing/my-repository.git",
|
|
120
|
+
access_token="YOUR_GITLAB_PERSONAL_ACCESS_TOKEN"
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
private_gitlab_block.save()
|
|
124
|
+
```
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# prefect-gitlab
|
|
2
|
+
|
|
3
|
+
<p align="center">
|
|
4
|
+
<a href="https://pypi.python.org/pypi/prefect-gitlab/" alt="PyPI version">
|
|
5
|
+
<img alt="PyPI" src="https://img.shields.io/pypi/v/prefect-gitlab?color=26272B&labelColor=090422"></a>
|
|
6
|
+
<a href="https://pepy.tech/badge/prefect-gitlab/" alt="Downloads">
|
|
7
|
+
<img src="https://img.shields.io/pypi/dm/prefect-gitlab?color=26272B&labelColor=090422" /></a>
|
|
8
|
+
</p>
|
|
9
|
+
|
|
10
|
+
## Welcome!
|
|
11
|
+
|
|
12
|
+
`prefect-gitlab` is a Prefect collection for working with GitLab repositories.
|
|
13
|
+
|
|
14
|
+
## Getting Started
|
|
15
|
+
|
|
16
|
+
### Python setup
|
|
17
|
+
|
|
18
|
+
Requires an installation of Python 3.8 or higher.
|
|
19
|
+
|
|
20
|
+
We recommend using a Python virtual environment manager such as pipenv, conda, or virtualenv.
|
|
21
|
+
|
|
22
|
+
This integration is designed to work with Prefect 2.3.0 or higher. For more information about how to use Prefect, please refer to the [Prefect documentation](https://docs.prefect.io/).
|
|
23
|
+
|
|
24
|
+
### Installation
|
|
25
|
+
|
|
26
|
+
Install `prefect-gitlab` with `pip`:
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install prefect-gitlab
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Then, register the [block types](https://docs.prefect.io/concepts/blocks/)) in this integration to view the storage block type on Prefect Cloud:
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
prefect block register -m prefect_gitlab
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Note, to use the `load` method on a block, you must already have a block document [saved](https://docs.prefect.io/concepts/blocks/).
|
|
39
|
+
|
|
40
|
+
## Creating a GitLab storage block
|
|
41
|
+
|
|
42
|
+
### In Python
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
from prefect_gitlab import GitLabRepository
|
|
46
|
+
|
|
47
|
+
# public GitLab repository
|
|
48
|
+
public_gitlab_block = GitLabRepository(
|
|
49
|
+
name="my-gitlab-block",
|
|
50
|
+
repository="https://gitlab.com/testing/my-repository.git"
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
public_gitlab_block.save()
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
# specific branch or tag of a GitLab repository
|
|
57
|
+
branch_gitlab_block = GitLabRepository(
|
|
58
|
+
name="my-gitlab-block",
|
|
59
|
+
reference="branch-or-tag-name",
|
|
60
|
+
repository="https://gitlab.com/testing/my-repository.git"
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
branch_gitlab_block.save()
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
# Get all history of a specific branch or tag of a GitLab repository
|
|
67
|
+
branch_gitlab_block = GitLabRepository(
|
|
68
|
+
name="my-gitlab-block",
|
|
69
|
+
reference="branch-or-tag-name",
|
|
70
|
+
git_depth=None,
|
|
71
|
+
repository="https://gitlab.com/testing/my-repository.git"
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
branch_gitlab_block.save()
|
|
75
|
+
|
|
76
|
+
# private GitLab repository
|
|
77
|
+
private_gitlab_block = GitLabRepository(
|
|
78
|
+
name="my-private-gitlab-block",
|
|
79
|
+
repository="https://gitlab.com/testing/my-repository.git",
|
|
80
|
+
access_token="YOUR_GITLAB_PERSONAL_ACCESS_TOKEN"
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
private_gitlab_block.save()
|
|
84
|
+
```
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# file generated by setuptools_scm
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
TYPE_CHECKING = False
|
|
4
|
+
if TYPE_CHECKING:
|
|
5
|
+
from typing import Tuple, Union
|
|
6
|
+
VERSION_TUPLE = Tuple[Union[int, str], ...]
|
|
7
|
+
else:
|
|
8
|
+
VERSION_TUPLE = object
|
|
9
|
+
|
|
10
|
+
version: str
|
|
11
|
+
__version__: str
|
|
12
|
+
__version_tuple__: VERSION_TUPLE
|
|
13
|
+
version_tuple: VERSION_TUPLE
|
|
14
|
+
|
|
15
|
+
__version__ = version = '0.2.3'
|
|
16
|
+
__version_tuple__ = version_tuple = (0, 2, 3)
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
"""Module used to enable authenticated interactions with GitLab"""
|
|
2
2
|
|
|
3
3
|
from gitlab import Gitlab
|
|
4
|
-
from prefect.blocks.core import Block
|
|
5
4
|
from pydantic import VERSION as PYDANTIC_VERSION
|
|
6
5
|
|
|
6
|
+
from prefect.blocks.core import Block
|
|
7
|
+
|
|
7
8
|
if PYDANTIC_VERSION.startswith("2."):
|
|
8
9
|
from pydantic.v1 import Field, HttpUrl, SecretStr
|
|
9
10
|
else:
|
|
@@ -47,11 +47,13 @@ from pathlib import Path
|
|
|
47
47
|
from tempfile import TemporaryDirectory
|
|
48
48
|
from typing import Optional, Tuple, Union
|
|
49
49
|
|
|
50
|
+
from pydantic import VERSION as PYDANTIC_VERSION
|
|
51
|
+
from tenacity import retry, stop_after_attempt, wait_fixed, wait_random
|
|
52
|
+
|
|
50
53
|
from prefect.exceptions import InvalidRepositoryURLError
|
|
51
54
|
from prefect.filesystems import ReadableDeploymentStorage
|
|
52
55
|
from prefect.utilities.asyncutils import sync_compatible
|
|
53
56
|
from prefect.utilities.processutils import run_process
|
|
54
|
-
from pydantic import VERSION as PYDANTIC_VERSION
|
|
55
57
|
|
|
56
58
|
if PYDANTIC_VERSION.startswith("2."):
|
|
57
59
|
from pydantic.v1 import Field, HttpUrl, validator
|
|
@@ -60,6 +62,13 @@ else:
|
|
|
60
62
|
|
|
61
63
|
from prefect_gitlab.credentials import GitLabCredentials
|
|
62
64
|
|
|
65
|
+
# Create get_directory retry settings
|
|
66
|
+
|
|
67
|
+
MAX_CLONE_ATTEMPTS = 3
|
|
68
|
+
CLONE_RETRY_MIN_DELAY_SECONDS = 1
|
|
69
|
+
CLONE_RETRY_MIN_DELAY_JITTER_SECONDS = 0
|
|
70
|
+
CLONE_RETRY_MAX_DELAY_JITTER_SECONDS = 3
|
|
71
|
+
|
|
63
72
|
|
|
64
73
|
class GitLabRepository(ReadableDeploymentStorage):
|
|
65
74
|
"""
|
|
@@ -86,6 +95,12 @@ class GitLabRepository(ReadableDeploymentStorage):
|
|
|
86
95
|
default=None,
|
|
87
96
|
description="An optional reference to pin to; can be a branch name or tag.",
|
|
88
97
|
)
|
|
98
|
+
git_depth: Optional[int] = Field(
|
|
99
|
+
default=1,
|
|
100
|
+
gte=1,
|
|
101
|
+
description="The number of commits that Git history is truncated to "
|
|
102
|
+
"during cloning. Set to None to fetch the entire history.",
|
|
103
|
+
)
|
|
89
104
|
credentials: Optional[GitLabCredentials] = Field(
|
|
90
105
|
default=None,
|
|
91
106
|
description="An optional GitLab Credentials block for authenticating with "
|
|
@@ -153,6 +168,15 @@ class GitLabRepository(ReadableDeploymentStorage):
|
|
|
153
168
|
return str(content_source), str(content_destination)
|
|
154
169
|
|
|
155
170
|
@sync_compatible
|
|
171
|
+
@retry(
|
|
172
|
+
stop=stop_after_attempt(MAX_CLONE_ATTEMPTS),
|
|
173
|
+
wait=wait_fixed(CLONE_RETRY_MIN_DELAY_SECONDS)
|
|
174
|
+
+ wait_random(
|
|
175
|
+
CLONE_RETRY_MIN_DELAY_JITTER_SECONDS,
|
|
176
|
+
CLONE_RETRY_MAX_DELAY_JITTER_SECONDS,
|
|
177
|
+
),
|
|
178
|
+
reraise=True,
|
|
179
|
+
)
|
|
156
180
|
async def get_directory(
|
|
157
181
|
self, from_path: Optional[str] = None, local_path: Optional[str] = None
|
|
158
182
|
) -> None:
|
|
@@ -171,7 +195,8 @@ class GitLabRepository(ReadableDeploymentStorage):
|
|
|
171
195
|
cmd += ["-b", self.reference]
|
|
172
196
|
|
|
173
197
|
# Limit git history
|
|
174
|
-
|
|
198
|
+
if self.git_depth is not None:
|
|
199
|
+
cmd += ["--depth", f"{self.git_depth}"]
|
|
175
200
|
|
|
176
201
|
# Clone to a temporary directory and move the subdirectory over
|
|
177
202
|
with TemporaryDirectory(suffix="prefect") as tmp_dir:
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: prefect-gitlab
|
|
3
|
+
Version: 0.2.3
|
|
4
|
+
Summary: A Prefect collection for working with GitLab repositories.
|
|
5
|
+
Author-email: "Prefect Technologies, Inc." <help@prefect.io>
|
|
6
|
+
License: Apache License 2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/PrefectHQ/prefect/tree/main/src/integrations/prefect-gitlab
|
|
8
|
+
Keywords: prefect
|
|
9
|
+
Classifier: Natural Language :: English
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Intended Audience :: System Administrators
|
|
12
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
20
|
+
Requires-Python: >=3.8
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
Requires-Dist: prefect>=2.13.5
|
|
23
|
+
Requires-Dist: python-gitlab>=3.12.0
|
|
24
|
+
Requires-Dist: tenacity>=8.2.3
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: aiohttp; extra == "dev"
|
|
27
|
+
Requires-Dist: coverage; extra == "dev"
|
|
28
|
+
Requires-Dist: interrogate; extra == "dev"
|
|
29
|
+
Requires-Dist: mkdocs-gen-files; extra == "dev"
|
|
30
|
+
Requires-Dist: mkdocs-material; extra == "dev"
|
|
31
|
+
Requires-Dist: mkdocs; extra == "dev"
|
|
32
|
+
Requires-Dist: mkdocstrings[python]; extra == "dev"
|
|
33
|
+
Requires-Dist: mock; python_version < "3.8" and extra == "dev"
|
|
34
|
+
Requires-Dist: mypy; extra == "dev"
|
|
35
|
+
Requires-Dist: pillow; extra == "dev"
|
|
36
|
+
Requires-Dist: pre-commit; extra == "dev"
|
|
37
|
+
Requires-Dist: pytest-asyncio; extra == "dev"
|
|
38
|
+
Requires-Dist: pytest; extra == "dev"
|
|
39
|
+
Requires-Dist: pytest-xdist; extra == "dev"
|
|
40
|
+
|
|
41
|
+
# prefect-gitlab
|
|
42
|
+
|
|
43
|
+
<p align="center">
|
|
44
|
+
<a href="https://pypi.python.org/pypi/prefect-gitlab/" alt="PyPI version">
|
|
45
|
+
<img alt="PyPI" src="https://img.shields.io/pypi/v/prefect-gitlab?color=26272B&labelColor=090422"></a>
|
|
46
|
+
<a href="https://pepy.tech/badge/prefect-gitlab/" alt="Downloads">
|
|
47
|
+
<img src="https://img.shields.io/pypi/dm/prefect-gitlab?color=26272B&labelColor=090422" /></a>
|
|
48
|
+
</p>
|
|
49
|
+
|
|
50
|
+
## Welcome!
|
|
51
|
+
|
|
52
|
+
`prefect-gitlab` is a Prefect collection for working with GitLab repositories.
|
|
53
|
+
|
|
54
|
+
## Getting Started
|
|
55
|
+
|
|
56
|
+
### Python setup
|
|
57
|
+
|
|
58
|
+
Requires an installation of Python 3.8 or higher.
|
|
59
|
+
|
|
60
|
+
We recommend using a Python virtual environment manager such as pipenv, conda, or virtualenv.
|
|
61
|
+
|
|
62
|
+
This integration is designed to work with Prefect 2.3.0 or higher. For more information about how to use Prefect, please refer to the [Prefect documentation](https://docs.prefect.io/).
|
|
63
|
+
|
|
64
|
+
### Installation
|
|
65
|
+
|
|
66
|
+
Install `prefect-gitlab` with `pip`:
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
pip install prefect-gitlab
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Then, register the [block types](https://docs.prefect.io/concepts/blocks/)) in this integration to view the storage block type on Prefect Cloud:
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
prefect block register -m prefect_gitlab
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Note, to use the `load` method on a block, you must already have a block document [saved](https://docs.prefect.io/concepts/blocks/).
|
|
79
|
+
|
|
80
|
+
## Creating a GitLab storage block
|
|
81
|
+
|
|
82
|
+
### In Python
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
from prefect_gitlab import GitLabRepository
|
|
86
|
+
|
|
87
|
+
# public GitLab repository
|
|
88
|
+
public_gitlab_block = GitLabRepository(
|
|
89
|
+
name="my-gitlab-block",
|
|
90
|
+
repository="https://gitlab.com/testing/my-repository.git"
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
public_gitlab_block.save()
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# specific branch or tag of a GitLab repository
|
|
97
|
+
branch_gitlab_block = GitLabRepository(
|
|
98
|
+
name="my-gitlab-block",
|
|
99
|
+
reference="branch-or-tag-name",
|
|
100
|
+
repository="https://gitlab.com/testing/my-repository.git"
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
branch_gitlab_block.save()
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
# Get all history of a specific branch or tag of a GitLab repository
|
|
107
|
+
branch_gitlab_block = GitLabRepository(
|
|
108
|
+
name="my-gitlab-block",
|
|
109
|
+
reference="branch-or-tag-name",
|
|
110
|
+
git_depth=None,
|
|
111
|
+
repository="https://gitlab.com/testing/my-repository.git"
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
branch_gitlab_block.save()
|
|
115
|
+
|
|
116
|
+
# private GitLab repository
|
|
117
|
+
private_gitlab_block = GitLabRepository(
|
|
118
|
+
name="my-private-gitlab-block",
|
|
119
|
+
repository="https://gitlab.com/testing/my-repository.git",
|
|
120
|
+
access_token="YOUR_GITLAB_PERSONAL_ACCESS_TOKEN"
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
private_gitlab_block.save()
|
|
124
|
+
```
|
|
@@ -1,11 +1,5 @@
|
|
|
1
|
-
LICENSE
|
|
2
|
-
MANIFEST.in
|
|
3
1
|
README.md
|
|
4
|
-
|
|
5
|
-
requirements.txt
|
|
6
|
-
setup.cfg
|
|
7
|
-
setup.py
|
|
8
|
-
versioneer.py
|
|
2
|
+
pyproject.toml
|
|
9
3
|
prefect_gitlab/__init__.py
|
|
10
4
|
prefect_gitlab/_version.py
|
|
11
5
|
prefect_gitlab/credentials.py
|
|
@@ -16,5 +10,7 @@ prefect_gitlab.egg-info/dependency_links.txt
|
|
|
16
10
|
prefect_gitlab.egg-info/entry_points.txt
|
|
17
11
|
prefect_gitlab.egg-info/requires.txt
|
|
18
12
|
prefect_gitlab.egg-info/top_level.txt
|
|
13
|
+
tests/conftest.py
|
|
19
14
|
tests/test_credentials.py
|
|
20
|
-
tests/test_repositories.py
|
|
15
|
+
tests/test_repositories.py
|
|
16
|
+
tests/test_version.py
|
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
prefect>=2.13.5
|
|
2
2
|
python-gitlab>=3.12.0
|
|
3
|
+
tenacity>=8.2.3
|
|
3
4
|
|
|
4
5
|
[dev]
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
mkdocs
|
|
6
|
+
aiohttp
|
|
7
|
+
coverage
|
|
8
|
+
interrogate
|
|
9
|
+
mkdocs-gen-files
|
|
10
10
|
mkdocs-material
|
|
11
|
+
mkdocs
|
|
11
12
|
mkdocstrings[python]
|
|
12
|
-
|
|
13
|
+
mypy
|
|
14
|
+
pillow
|
|
13
15
|
pre-commit
|
|
14
16
|
pytest-asyncio
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
coverage
|
|
18
|
-
pillow
|
|
17
|
+
pytest
|
|
18
|
+
pytest-xdist
|
|
19
19
|
|
|
20
20
|
[dev:python_version < "3.8"]
|
|
21
21
|
mock
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "prefect-gitlab"
|
|
7
|
+
description = "A Prefect collection for working with GitLab repositories."
|
|
8
|
+
readme = "README.md"
|
|
9
|
+
requires-python = ">=3.8"
|
|
10
|
+
license = {text = "Apache License 2.0"}
|
|
11
|
+
keywords = ["prefect"]
|
|
12
|
+
authors = [
|
|
13
|
+
{name = "Prefect Technologies, Inc.", email = "help@prefect.io"}
|
|
14
|
+
]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Natural Language :: English",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Intended Audience :: System Administrators",
|
|
19
|
+
"License :: OSI Approved :: Apache Software License",
|
|
20
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
21
|
+
"Programming Language :: Python :: 3.8",
|
|
22
|
+
"Programming Language :: Python :: 3.9",
|
|
23
|
+
"Programming Language :: Python :: 3.10",
|
|
24
|
+
"Programming Language :: Python :: 3.11",
|
|
25
|
+
"Programming Language :: Python :: 3.12",
|
|
26
|
+
"Topic :: Software Development :: Libraries",
|
|
27
|
+
]
|
|
28
|
+
dependencies = [
|
|
29
|
+
"prefect>=2.13.5",
|
|
30
|
+
"python-gitlab>=3.12.0",
|
|
31
|
+
"tenacity>=8.2.3",
|
|
32
|
+
]
|
|
33
|
+
dynamic = ["version"]
|
|
34
|
+
|
|
35
|
+
[project.optional-dependencies]
|
|
36
|
+
dev = [
|
|
37
|
+
"aiohttp",
|
|
38
|
+
"coverage",
|
|
39
|
+
"interrogate",
|
|
40
|
+
"mkdocs-gen-files",
|
|
41
|
+
"mkdocs-material",
|
|
42
|
+
"mkdocs",
|
|
43
|
+
"mkdocstrings[python]",
|
|
44
|
+
"mock; python_version < '3.8'",
|
|
45
|
+
"mypy",
|
|
46
|
+
"pillow",
|
|
47
|
+
"pre-commit",
|
|
48
|
+
"pytest-asyncio",
|
|
49
|
+
"pytest",
|
|
50
|
+
"pytest-xdist",
|
|
51
|
+
]
|
|
52
|
+
|
|
53
|
+
[project.urls]
|
|
54
|
+
Homepage = "https://github.com/PrefectHQ/prefect/tree/main/src/integrations/prefect-gitlab"
|
|
55
|
+
|
|
56
|
+
[project.entry-points."prefect.collections"]
|
|
57
|
+
prefect_gitlab = "prefect_gitlab"
|
|
58
|
+
|
|
59
|
+
[tool.setuptools_scm]
|
|
60
|
+
version_file = "prefect_gitlab/_version.py"
|
|
61
|
+
root = "../../.."
|
|
62
|
+
tag_regex = "^prefect-gitlab-(?P<version>\\d+\\.\\d+\\.\\d+)$"
|
|
63
|
+
fallback_version = "0.0.0"
|
|
64
|
+
git_describe_command = 'git describe --dirty --tags --long --match "prefect-gitlab-*[0-9]*"'
|
|
65
|
+
|
|
66
|
+
[tool.interrogate]
|
|
67
|
+
ignore-init-module = true
|
|
68
|
+
ignore_init_method = true
|
|
69
|
+
exclude = ["prefect_gitlab/_version.py", "tests"]
|
|
70
|
+
fail-under = 95
|
|
71
|
+
omit-covered-files = true
|
|
72
|
+
|
|
73
|
+
[tool.coverage.run]
|
|
74
|
+
omit = ["tests/*", "prefect_gitlab/_version.py"]
|
|
75
|
+
|
|
76
|
+
[tool.coverage.report]
|
|
77
|
+
fail_under = 80
|
|
78
|
+
show_missing = true
|
|
79
|
+
|
|
80
|
+
[tool.pytest.ini_options]
|
|
81
|
+
asyncio_mode = "auto"
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
from prefect.testing.utilities import prefect_test_harness
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@pytest.fixture(scope="session", autouse=True)
|
|
7
|
+
def prefect_db():
|
|
8
|
+
"""
|
|
9
|
+
Sets up test harness for temporary DB during test runs.
|
|
10
|
+
"""
|
|
11
|
+
with prefect_test_harness():
|
|
12
|
+
yield
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@pytest.fixture(autouse=True)
|
|
16
|
+
def reset_object_registry():
|
|
17
|
+
"""
|
|
18
|
+
Ensures each test has a clean object registry.
|
|
19
|
+
"""
|
|
20
|
+
from prefect.context import PrefectObjectRegistry
|
|
21
|
+
|
|
22
|
+
with PrefectObjectRegistry():
|
|
23
|
+
yield
|
|
@@ -4,9 +4,10 @@ from tempfile import TemporaryDirectory
|
|
|
4
4
|
from typing import Set, Tuple
|
|
5
5
|
|
|
6
6
|
import pytest
|
|
7
|
+
from pydantic import VERSION as PYDANTIC_VERSION
|
|
8
|
+
|
|
7
9
|
from prefect.exceptions import InvalidRepositoryURLError
|
|
8
10
|
from prefect.testing.utilities import AsyncMock
|
|
9
|
-
from pydantic import VERSION as PYDANTIC_VERSION
|
|
10
11
|
|
|
11
12
|
if PYDANTIC_VERSION.startswith("2."):
|
|
12
13
|
from pydantic.v1 import SecretStr
|
|
@@ -15,7 +16,7 @@ else:
|
|
|
15
16
|
|
|
16
17
|
import prefect_gitlab
|
|
17
18
|
from prefect_gitlab.credentials import GitLabCredentials
|
|
18
|
-
from prefect_gitlab.repositories import GitLabRepository
|
|
19
|
+
from prefect_gitlab.repositories import GitLabRepository # noqa: E402
|
|
19
20
|
|
|
20
21
|
|
|
21
22
|
class TestGitLab:
|
|
@@ -146,6 +147,29 @@ class TestGitLab:
|
|
|
146
147
|
]
|
|
147
148
|
assert mock.await_args[0][0][: len(expected_cmd)] == expected_cmd
|
|
148
149
|
|
|
150
|
+
async def test_cloning_with_custom_depth(self, monkeypatch):
|
|
151
|
+
"""Ensure that we can retrieve the whole history, i.e. support true git clone""" # noqa: E501
|
|
152
|
+
|
|
153
|
+
class p:
|
|
154
|
+
returncode = 0
|
|
155
|
+
|
|
156
|
+
mock = AsyncMock(return_value=p())
|
|
157
|
+
monkeypatch.setattr(prefect_gitlab.repositories, "run_process", mock)
|
|
158
|
+
repo = "git@gitlab.com:PrefectHQ/prefect.git"
|
|
159
|
+
depth = None
|
|
160
|
+
g = GitLabRepository(
|
|
161
|
+
repository=repo,
|
|
162
|
+
git_depth=depth,
|
|
163
|
+
)
|
|
164
|
+
await g.get_directory()
|
|
165
|
+
assert mock.await_count == 1
|
|
166
|
+
expected_cmd = [
|
|
167
|
+
"git",
|
|
168
|
+
"clone",
|
|
169
|
+
repo,
|
|
170
|
+
]
|
|
171
|
+
assert mock.await_args[0][0][: len(expected_cmd)] == expected_cmd
|
|
172
|
+
|
|
149
173
|
async def test_ssh_fails_with_credential(self, monkeypatch):
|
|
150
174
|
"""Ensure that credentials cannot be passed in if the URL is not in the HTTPS/HTTP
|
|
151
175
|
format.
|
|
@@ -170,9 +194,7 @@ class TestGitLab:
|
|
|
170
194
|
credentials=GitLabCredentials(token=SecretStr(credential)),
|
|
171
195
|
)
|
|
172
196
|
|
|
173
|
-
async def test_dir_contents_copied_correctly_with_get_directory(
|
|
174
|
-
self, monkeypatch
|
|
175
|
-
): # noqa
|
|
197
|
+
async def test_dir_contents_copied_correctly_with_get_directory(self, monkeypatch): # noqa
|
|
176
198
|
"""Check that `get_directory` is able to correctly copy contents from src->dst""" # noqa
|
|
177
199
|
|
|
178
200
|
class p:
|
|
@@ -243,3 +265,22 @@ class TestGitLab:
|
|
|
243
265
|
|
|
244
266
|
assert set(os.listdir(tmp_dst)) == set([sub_dir_name])
|
|
245
267
|
assert set(os.listdir(Path(tmp_dst) / sub_dir_name)) == child_contents
|
|
268
|
+
|
|
269
|
+
async def test_get_directory_retries(self, monkeypatch):
|
|
270
|
+
# Constants for the retry decorator
|
|
271
|
+
MAX_CLONE_ATTEMPTS = 3
|
|
272
|
+
|
|
273
|
+
# Create an instance of GitLabRepository
|
|
274
|
+
g = GitLabRepository(repository="https://gitlab.com/prefectHQ/prefect.git")
|
|
275
|
+
|
|
276
|
+
# Prepare a MagicMock to simulate the process call within get_directory
|
|
277
|
+
mock = AsyncMock()
|
|
278
|
+
mock.return_value = AsyncMock(returncode=1) # Simulate failure
|
|
279
|
+
monkeypatch.setattr(prefect_gitlab.repositories, "run_process", mock)
|
|
280
|
+
|
|
281
|
+
# Call get_directory and expect it to raise a RetryError after maximum attempts
|
|
282
|
+
with pytest.raises(OSError):
|
|
283
|
+
await g.get_directory()
|
|
284
|
+
print(mock.call_count)
|
|
285
|
+
# Verify that the function retried the expected number of times
|
|
286
|
+
assert mock.call_count == MAX_CLONE_ATTEMPTS
|