glpkg 1.0.0__tar.gz → 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {glpkg-1.0.0 → glpkg-1.2.0}/PKG-INFO +18 -15
- {glpkg-1.0.0 → glpkg-1.2.0}/README.md +17 -8
- {glpkg-1.0.0 → glpkg-1.2.0}/pyproject.toml +1 -10
- {glpkg-1.0.0 → glpkg-1.2.0}/src/gitlab/__init__.py +1 -1
- {glpkg-1.0.0 → glpkg-1.2.0}/src/gitlab/cli_handler.py +50 -9
- glpkg-1.2.0/src/gitlab/packages.py +176 -0
- {glpkg-1.0.0 → glpkg-1.2.0}/src/glpkg.egg-info/PKG-INFO +18 -15
- {glpkg-1.0.0 → glpkg-1.2.0}/src/glpkg.egg-info/SOURCES.txt +2 -2
- glpkg-1.2.0/test/test_cli_handler.py +45 -0
- glpkg-1.0.0/test/test_gitlab.py → glpkg-1.2.0/test/test_packages.py +34 -38
- glpkg-1.0.0/src/gitlab/packages.py +0 -130
- glpkg-1.0.0/src/glpkg.egg-info/requires.txt +0 -8
- {glpkg-1.0.0 → glpkg-1.2.0}/LICENSE.md +0 -0
- {glpkg-1.0.0 → glpkg-1.2.0}/setup.cfg +0 -0
- {glpkg-1.0.0 → glpkg-1.2.0}/src/gitlab/__main__.py +0 -0
- {glpkg-1.0.0 → glpkg-1.2.0}/src/glpkg.egg-info/dependency_links.txt +0 -0
- {glpkg-1.0.0 → glpkg-1.2.0}/src/glpkg.egg-info/entry_points.txt +0 -0
- {glpkg-1.0.0 → glpkg-1.2.0}/src/glpkg.egg-info/top_level.txt +0 -0
|
@@ -1,25 +1,19 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: glpkg
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.2.0
|
|
4
4
|
Summary: Tool to make GitLab generic package registry operations easy.
|
|
5
5
|
Author-email: bugproduction <bugproduction@outlook.com>
|
|
6
6
|
License-Expression: MIT
|
|
7
7
|
Project-URL: Repository, https://gitlab.com/bugproduction/glpkg.git
|
|
8
8
|
Keywords: GitLab,packages,registry,generic
|
|
9
|
+
Requires-Python: >=3.9
|
|
9
10
|
Description-Content-Type: text/markdown
|
|
10
11
|
License-File: LICENSE.md
|
|
11
|
-
Provides-Extra: dev
|
|
12
|
-
Requires-Dist: black; extra == "dev"
|
|
13
|
-
Requires-Dist: build; extra == "dev"
|
|
14
|
-
Requires-Dist: pytest; extra == "dev"
|
|
15
|
-
Requires-Dist: pytest-cov; extra == "dev"
|
|
16
|
-
Requires-Dist: setuptools; extra == "dev"
|
|
17
|
-
Requires-Dist: twine; extra == "dev"
|
|
18
12
|
Dynamic: license-file
|
|
19
13
|
|
|
20
|
-
# glpkg - GitLab Generic Package
|
|
14
|
+
# glpkg - GitLab Generic Package tools
|
|
21
15
|
|
|
22
|
-
glpkg is a tool that makes it easy to work with [GitLab generic
|
|
16
|
+
glpkg is a tool that makes it easy to work with [GitLab generic packages](https://docs.gitlab.com/user/packages/generic_packages/).
|
|
23
17
|
|
|
24
18
|
|
|
25
19
|
## Installation
|
|
@@ -47,7 +41,11 @@ By default, the used GitLab host is gitlab.com. If you use a self-hosted GitLab,
|
|
|
47
41
|
|
|
48
42
|
To authenticate with the package registry in any of the commands below, use `--token readapitoken123` argument where the `readapitoken123` is a [personal](https://docs.gitlab.com/user/profile/personal_access_tokens/#create-a-personal-access-token) or [project](https://docs.gitlab.com/user/project/settings/project_access_tokens/#create-a-project-access-token) access token, with read API scope. In case the package registry is public, you can omit this argument.
|
|
49
43
|
|
|
50
|
-
|
|
44
|
+
Alternatively you can use a token stored in your `.netrc` file by setting `--netrc` argument.
|
|
45
|
+
|
|
46
|
+
> If you use the tool in GitLab CI, read [below](#Use-in-GitLab-pipelines) on how to use the `CI_JOB_TOKEN`.
|
|
47
|
+
|
|
48
|
+
The arguments related to the GitLab host or authentication (`--token`, `--netrc`, and `--ci`) are omitted in the examples below to focus on the commands.
|
|
51
49
|
|
|
52
50
|
In general, run `glpkg --help` when needed.
|
|
53
51
|
|
|
@@ -74,7 +72,7 @@ mypackagename 2.0
|
|
|
74
72
|
|
|
75
73
|
### Download generic package
|
|
76
74
|
|
|
77
|
-
To download
|
|
75
|
+
To download all files from a specific version of a generic package, run
|
|
78
76
|
|
|
79
77
|
```bash
|
|
80
78
|
glpkg download --project 12345 --name mypackagename --version 1.0
|
|
@@ -85,7 +83,13 @@ Where:
|
|
|
85
83
|
- `mypackagename` is the name of the generic package
|
|
86
84
|
- `1.0` is the version of the generic package from which the files are downloaded
|
|
87
85
|
|
|
88
|
-
|
|
86
|
+
By default the files will be downloaded in the current working directory. To download the files to another directory, add argument `--destination` to the command. In all cases, as long as you have permissions to the destination directory, any pre-existing files will be overridden without warning.
|
|
87
|
+
|
|
88
|
+
To download only a specific file from the package, add `--file` argument.
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
glpkg download --project 12345 --name mypackagename --version 1.5 --file the_only_one --destination /temp
|
|
92
|
+
```
|
|
89
93
|
|
|
90
94
|
> If a package has multiple files with the same filename, the tool can only download the newest file. This is a restriction of GitLab API.
|
|
91
95
|
|
|
@@ -107,7 +111,7 @@ Where:
|
|
|
107
111
|
|
|
108
112
|
### Use in GitLab pipelines
|
|
109
113
|
|
|
110
|
-
If you use the tool in a GitLab pipeline,
|
|
114
|
+
If you use the tool in a GitLab pipeline, setting argument `--ci` uses [GitLab predefined variables](https://docs.gitlab.com/ci/variables/predefined_variables/) to configure the tool. In this case `CI_SERVER_HOST`, `CI_PROJECT_ID`, and `CI_JOB_TOKEN` environment variables are used. The `--project`, and `--token` arguments can still be used to override the project ID or to use a personal or project access token instead of `CI_JOB_TOKEN`.
|
|
111
115
|
|
|
112
116
|
In other words, you don't need to give the `--host`, `--project`, or `--token` arguments if you are interacting with the package registry of the project where the pipeline is running. Example: uploading `my-file.txt` to generic package `mypackagename` version `1.0` in the project package registry in CI:
|
|
113
117
|
|
|
@@ -124,4 +128,3 @@ The tool is not perfect (yet) and has limitations. The following limitations are
|
|
|
124
128
|
|
|
125
129
|
- Uploading files must be done one-by-one.
|
|
126
130
|
- Only project registries are supported for now.
|
|
127
|
-
- Pagination is not supported for now - in case you have more than 100 versions of a package, not all will be shown.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
# glpkg - GitLab Generic Package
|
|
1
|
+
# glpkg - GitLab Generic Package tools
|
|
2
2
|
|
|
3
|
-
glpkg is a tool that makes it easy to work with [GitLab generic
|
|
3
|
+
glpkg is a tool that makes it easy to work with [GitLab generic packages](https://docs.gitlab.com/user/packages/generic_packages/).
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
## Installation
|
|
@@ -28,7 +28,11 @@ By default, the used GitLab host is gitlab.com. If you use a self-hosted GitLab,
|
|
|
28
28
|
|
|
29
29
|
To authenticate with the package registry in any of the commands below, use `--token readapitoken123` argument where the `readapitoken123` is a [personal](https://docs.gitlab.com/user/profile/personal_access_tokens/#create-a-personal-access-token) or [project](https://docs.gitlab.com/user/project/settings/project_access_tokens/#create-a-project-access-token) access token, with read API scope. In case the package registry is public, you can omit this argument.
|
|
30
30
|
|
|
31
|
-
|
|
31
|
+
Alternatively you can use a token stored in your `.netrc` file by setting `--netrc` argument.
|
|
32
|
+
|
|
33
|
+
> If you use the tool in GitLab CI, read [below](#Use-in-GitLab-pipelines) on how to use the `CI_JOB_TOKEN`.
|
|
34
|
+
|
|
35
|
+
The arguments related to the GitLab host or authentication (`--token`, `--netrc`, and `--ci`) are omitted in the examples below to focus on the commands.
|
|
32
36
|
|
|
33
37
|
In general, run `glpkg --help` when needed.
|
|
34
38
|
|
|
@@ -55,7 +59,7 @@ mypackagename 2.0
|
|
|
55
59
|
|
|
56
60
|
### Download generic package
|
|
57
61
|
|
|
58
|
-
To download
|
|
62
|
+
To download all files from a specific version of a generic package, run
|
|
59
63
|
|
|
60
64
|
```bash
|
|
61
65
|
glpkg download --project 12345 --name mypackagename --version 1.0
|
|
@@ -66,7 +70,13 @@ Where:
|
|
|
66
70
|
- `mypackagename` is the name of the generic package
|
|
67
71
|
- `1.0` is the version of the generic package from which the files are downloaded
|
|
68
72
|
|
|
69
|
-
|
|
73
|
+
By default the files will be downloaded in the current working directory. To download the files to another directory, add argument `--destination` to the command. In all cases, as long as you have permissions to the destination directory, any pre-existing files will be overridden without warning.
|
|
74
|
+
|
|
75
|
+
To download only a specific file from the package, add `--file` argument.
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
glpkg download --project 12345 --name mypackagename --version 1.5 --file the_only_one --destination /temp
|
|
79
|
+
```
|
|
70
80
|
|
|
71
81
|
> If a package has multiple files with the same filename, the tool can only download the newest file. This is a restriction of GitLab API.
|
|
72
82
|
|
|
@@ -88,7 +98,7 @@ Where:
|
|
|
88
98
|
|
|
89
99
|
### Use in GitLab pipelines
|
|
90
100
|
|
|
91
|
-
If you use the tool in a GitLab pipeline,
|
|
101
|
+
If you use the tool in a GitLab pipeline, setting argument `--ci` uses [GitLab predefined variables](https://docs.gitlab.com/ci/variables/predefined_variables/) to configure the tool. In this case `CI_SERVER_HOST`, `CI_PROJECT_ID`, and `CI_JOB_TOKEN` environment variables are used. The `--project`, and `--token` arguments can still be used to override the project ID or to use a personal or project access token instead of `CI_JOB_TOKEN`.
|
|
92
102
|
|
|
93
103
|
In other words, you don't need to give the `--host`, `--project`, or `--token` arguments if you are interacting with the package registry of the project where the pipeline is running. Example: uploading `my-file.txt` to generic package `mypackagename` version `1.0` in the project package registry in CI:
|
|
94
104
|
|
|
@@ -104,5 +114,4 @@ To use the `CI_JOB_TOKEN` with package registry of another projects, add `--proj
|
|
|
104
114
|
The tool is not perfect (yet) and has limitations. The following limitations are known, but more can exist:
|
|
105
115
|
|
|
106
116
|
- Uploading files must be done one-by-one.
|
|
107
|
-
- Only project registries are supported for now.
|
|
108
|
-
- Pagination is not supported for now - in case you have more than 100 versions of a package, not all will be shown.
|
|
117
|
+
- Only project registries are supported for now.
|
|
@@ -12,21 +12,12 @@ license-files = ["LICENSE.md"]
|
|
|
12
12
|
authors = [
|
|
13
13
|
{ name="bugproduction", email="bugproduction@outlook.com" }
|
|
14
14
|
]
|
|
15
|
+
requires-python = ">= 3.9"
|
|
15
16
|
dependencies = []
|
|
16
17
|
|
|
17
18
|
[project.urls]
|
|
18
19
|
Repository = "https://gitlab.com/bugproduction/glpkg.git"
|
|
19
20
|
|
|
20
|
-
[project.optional-dependencies]
|
|
21
|
-
dev = [
|
|
22
|
-
"black",
|
|
23
|
-
"build",
|
|
24
|
-
"pytest",
|
|
25
|
-
"pytest-cov",
|
|
26
|
-
"setuptools",
|
|
27
|
-
"twine"
|
|
28
|
-
]
|
|
29
|
-
|
|
30
21
|
[project.scripts]
|
|
31
22
|
glpkg = "gitlab.__main__:cli"
|
|
32
23
|
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import argparse
|
|
2
2
|
import netrc
|
|
3
3
|
import os
|
|
4
|
+
import sys
|
|
5
|
+
import urllib
|
|
4
6
|
from gitlab import Packages, __version__
|
|
5
7
|
|
|
6
8
|
|
|
@@ -33,7 +35,20 @@ class CLIHandler:
|
|
|
33
35
|
return 0
|
|
34
36
|
|
|
35
37
|
def do_it(self) -> int:
|
|
36
|
-
|
|
38
|
+
ret = 1
|
|
39
|
+
try:
|
|
40
|
+
ret = self.args.action(self.args)
|
|
41
|
+
except urllib.error.HTTPError as e:
|
|
42
|
+
# GitLab API returns 404 when a resource is not found
|
|
43
|
+
# but also when the user has no access to the resource
|
|
44
|
+
print("Oops! Something did go wrong.", file=sys.stderr)
|
|
45
|
+
print(e, file=sys.stderr)
|
|
46
|
+
print(
|
|
47
|
+
"Note that Error 404 may also indicate authentication issues with GitLab API.",
|
|
48
|
+
file=sys.stderr,
|
|
49
|
+
)
|
|
50
|
+
print("Check your arguments and credentials.", file=sys.stderr)
|
|
51
|
+
return ret
|
|
37
52
|
|
|
38
53
|
def _register_common_arguments(self, parser) -> None:
|
|
39
54
|
group = parser.add_mutually_exclusive_group()
|
|
@@ -73,6 +88,19 @@ class CLIHandler:
|
|
|
73
88
|
def _register_download_parser(self, parser):
|
|
74
89
|
self._register_common_arguments(parser)
|
|
75
90
|
parser.add_argument("-v", "--version", type=str, help="The package version.")
|
|
91
|
+
parser.add_argument(
|
|
92
|
+
"-f",
|
|
93
|
+
"--file",
|
|
94
|
+
type=str,
|
|
95
|
+
help="The file to download from the package. If not defined, all files are downloaded.",
|
|
96
|
+
)
|
|
97
|
+
parser.add_argument(
|
|
98
|
+
"-d",
|
|
99
|
+
"--destination",
|
|
100
|
+
default="",
|
|
101
|
+
type=str,
|
|
102
|
+
help="The path where the file(s) are downloaded. If not defined, the current working directory is used.",
|
|
103
|
+
)
|
|
76
104
|
parser.set_defaults(action=self._download_handler)
|
|
77
105
|
|
|
78
106
|
def _args(self, args):
|
|
@@ -98,16 +126,25 @@ class CLIHandler:
|
|
|
98
126
|
return host, project, name, token_user, token
|
|
99
127
|
|
|
100
128
|
def _download_handler(self, args) -> int:
|
|
129
|
+
ret = 1
|
|
101
130
|
host, project, name, token_user, token = self._args(args)
|
|
102
131
|
version = args.version
|
|
132
|
+
destination = args.destination
|
|
103
133
|
gitlab = Packages(host, token_user, token)
|
|
104
134
|
package_id = gitlab.get_package_id(project, name, version)
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
135
|
+
if package_id:
|
|
136
|
+
files = []
|
|
137
|
+
if args.file:
|
|
138
|
+
files.append(args.file)
|
|
139
|
+
else:
|
|
140
|
+
files = gitlab.list_files(project, package_id)
|
|
141
|
+
for file in files:
|
|
142
|
+
ret = gitlab.download_file(project, name, version, file, destination)
|
|
143
|
+
if ret:
|
|
144
|
+
print("Failed to download file " + file)
|
|
145
|
+
break
|
|
146
|
+
else:
|
|
147
|
+
print("No package " + name + " version " + version + " found!")
|
|
111
148
|
return ret
|
|
112
149
|
|
|
113
150
|
def _register_list_parser(self, parser):
|
|
@@ -134,9 +171,13 @@ class CLIHandler:
|
|
|
134
171
|
parser.set_defaults(action=self._upload)
|
|
135
172
|
|
|
136
173
|
def _upload(self, args) -> int:
|
|
174
|
+
ret = 1
|
|
137
175
|
host, project, name, token_user, token = self._args(args)
|
|
138
176
|
version = args.version
|
|
139
177
|
file = args.file
|
|
140
|
-
|
|
141
|
-
|
|
178
|
+
if os.path.isfile(file):
|
|
179
|
+
gitlab = Packages(host, token_user, token)
|
|
180
|
+
ret = gitlab.upload_file(project, name, version, file)
|
|
181
|
+
else:
|
|
182
|
+
print("File " + file + " does not exist!")
|
|
142
183
|
return ret
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
from http.client import HTTPMessage
|
|
2
|
+
import json
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
from urllib import request, parse
|
|
6
|
+
|
|
7
|
+
logger = logging.getLogger(__name__)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Packages:
|
|
11
|
+
def __init__(self, host: str, token_type: str, token: str):
|
|
12
|
+
self.host = host
|
|
13
|
+
self.token_type = token_type
|
|
14
|
+
self.token = token
|
|
15
|
+
|
|
16
|
+
def api_url(self) -> str:
|
|
17
|
+
return "https://{}/api/v4/".format(parse.quote(self.host))
|
|
18
|
+
|
|
19
|
+
def project_api_url(self, project: str) -> str:
|
|
20
|
+
return self.api_url() + "projects/{}/".format(parse.quote_plus(project))
|
|
21
|
+
|
|
22
|
+
def get_headers(self) -> dict:
|
|
23
|
+
headers = {}
|
|
24
|
+
if self.token_type and self.token:
|
|
25
|
+
headers = {self.token_type: self.token}
|
|
26
|
+
return headers
|
|
27
|
+
|
|
28
|
+
def _request(self, url: str) -> tuple[int, bytes, HTTPMessage]:
|
|
29
|
+
logger.debug("Requesting " + url)
|
|
30
|
+
req = request.Request(url, headers=self.get_headers())
|
|
31
|
+
with request.urlopen(req) as response:
|
|
32
|
+
return response.status, response.read(), response.headers
|
|
33
|
+
|
|
34
|
+
def _get_next_page(self, headers: HTTPMessage) -> int:
|
|
35
|
+
ret = 0
|
|
36
|
+
if headers:
|
|
37
|
+
next_page = headers.get("x-next-page")
|
|
38
|
+
if next_page:
|
|
39
|
+
ret = int(next_page)
|
|
40
|
+
logger.debug("Response incomplete, next page is " + next_page)
|
|
41
|
+
else:
|
|
42
|
+
logger.debug("Response complete")
|
|
43
|
+
return ret
|
|
44
|
+
|
|
45
|
+
def _build_query(self, arg: str, page: int) -> str:
|
|
46
|
+
query = ""
|
|
47
|
+
if arg or page:
|
|
48
|
+
if page:
|
|
49
|
+
page = "page=" + str(page)
|
|
50
|
+
query = "?{}".format("&".join(filter(None, (arg, page))))
|
|
51
|
+
return query
|
|
52
|
+
|
|
53
|
+
def gl_project_api(self, project: str, path: str, arg: str = None) -> list:
|
|
54
|
+
data = []
|
|
55
|
+
more = True
|
|
56
|
+
page = None
|
|
57
|
+
while more:
|
|
58
|
+
more = False
|
|
59
|
+
query = self._build_query(arg, page)
|
|
60
|
+
url = self.project_api_url(project) + path + query
|
|
61
|
+
status, res_data, headers = self._request(url)
|
|
62
|
+
logger.debug("Response status: " + str(status))
|
|
63
|
+
res_data = json.loads(res_data)
|
|
64
|
+
logger.debug("Response data: " + str(res_data))
|
|
65
|
+
data = data + res_data
|
|
66
|
+
page = self._get_next_page(headers)
|
|
67
|
+
if page:
|
|
68
|
+
more = True
|
|
69
|
+
return data
|
|
70
|
+
|
|
71
|
+
def list_packages(self, project: str, package_name: str) -> list:
|
|
72
|
+
packages = []
|
|
73
|
+
logger.debug("Listing packages with name " + package_name)
|
|
74
|
+
data = self.gl_project_api(
|
|
75
|
+
project, "packages", "package_name=" + parse.quote_plus(package_name)
|
|
76
|
+
)
|
|
77
|
+
for package in data:
|
|
78
|
+
name = parse.unquote(package["name"])
|
|
79
|
+
version = parse.unquote(package["version"])
|
|
80
|
+
# GitLab API returns packages that have some match to the filter;
|
|
81
|
+
# let's filter out non-exact matches
|
|
82
|
+
if package_name != name:
|
|
83
|
+
continue
|
|
84
|
+
packages.append({"name": name, "version": version})
|
|
85
|
+
return packages
|
|
86
|
+
|
|
87
|
+
def list_files(self, project: str, package_id: int) -> list:
|
|
88
|
+
files = []
|
|
89
|
+
logger.debug("Listing package " + str(package_id) + " files")
|
|
90
|
+
path = "packages/" + parse.quote_plus(str(package_id)) + "/package_files"
|
|
91
|
+
data = self.gl_project_api(project, path)
|
|
92
|
+
for package in data:
|
|
93
|
+
# Only append the filename once to the list of files
|
|
94
|
+
# as there's no way to download them separately through
|
|
95
|
+
# the API
|
|
96
|
+
filename = parse.unquote(package["file_name"])
|
|
97
|
+
if filename not in files:
|
|
98
|
+
files.append(filename)
|
|
99
|
+
return files
|
|
100
|
+
|
|
101
|
+
def get_package_id(
|
|
102
|
+
self, project: str, package_name: str, package_version: str
|
|
103
|
+
) -> int:
|
|
104
|
+
id = 0
|
|
105
|
+
logger.debug(
|
|
106
|
+
"Fetching package " + package_name + " (" + package_version + ") ID"
|
|
107
|
+
)
|
|
108
|
+
path = "packages"
|
|
109
|
+
arg = (
|
|
110
|
+
"package_name="
|
|
111
|
+
+ parse.quote_plus(package_name)
|
|
112
|
+
+ "&package_version="
|
|
113
|
+
+ parse.quote_plus(package_version)
|
|
114
|
+
)
|
|
115
|
+
data = self.gl_project_api(project, path, arg)
|
|
116
|
+
if len(data) == 1:
|
|
117
|
+
package = data.pop()
|
|
118
|
+
id = package["id"]
|
|
119
|
+
return id
|
|
120
|
+
|
|
121
|
+
def download_file(
|
|
122
|
+
self,
|
|
123
|
+
project: str,
|
|
124
|
+
package_name: str,
|
|
125
|
+
package_version: str,
|
|
126
|
+
filename: str,
|
|
127
|
+
destination: str = "",
|
|
128
|
+
) -> int:
|
|
129
|
+
ret = 1
|
|
130
|
+
logger.debug("Downloading file " + filename)
|
|
131
|
+
url = (
|
|
132
|
+
self.project_api_url(project)
|
|
133
|
+
+ "packages/generic/"
|
|
134
|
+
+ parse.quote_plus(package_name)
|
|
135
|
+
+ "/"
|
|
136
|
+
+ parse.quote_plus(package_version)
|
|
137
|
+
+ "/"
|
|
138
|
+
+ parse.quote(filename)
|
|
139
|
+
)
|
|
140
|
+
status, data, _ = self._request(url)
|
|
141
|
+
if status == 200:
|
|
142
|
+
path = os.path.join(destination, filename)
|
|
143
|
+
parent = os.path.dirname(path)
|
|
144
|
+
if parent:
|
|
145
|
+
# Create missing directories if needed
|
|
146
|
+
# In case path has no parent, current
|
|
147
|
+
# workind directory is used
|
|
148
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
149
|
+
with open(path, "wb") as file:
|
|
150
|
+
file.write(data)
|
|
151
|
+
ret = 0
|
|
152
|
+
return ret
|
|
153
|
+
|
|
154
|
+
def upload_file(
|
|
155
|
+
self, project: str, package_name: str, package_version: str, file: str
|
|
156
|
+
) -> int:
|
|
157
|
+
ret = 1
|
|
158
|
+
logger.debug("Uploading file " + file)
|
|
159
|
+
with open(str(file), "rb") as data:
|
|
160
|
+
url = (
|
|
161
|
+
self.project_api_url(project)
|
|
162
|
+
+ "packages/generic/"
|
|
163
|
+
+ parse.quote_plus(package_name)
|
|
164
|
+
+ "/"
|
|
165
|
+
+ parse.quote_plus(package_version)
|
|
166
|
+
+ "/"
|
|
167
|
+
+ parse.quote(str(file))
|
|
168
|
+
)
|
|
169
|
+
res = request.urlopen(
|
|
170
|
+
request.Request(
|
|
171
|
+
url, method="PUT", data=data, headers=self.get_headers()
|
|
172
|
+
)
|
|
173
|
+
)
|
|
174
|
+
if res.status == 201: # 201 is created
|
|
175
|
+
ret = 0
|
|
176
|
+
return ret
|
|
@@ -1,25 +1,19 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: glpkg
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.2.0
|
|
4
4
|
Summary: Tool to make GitLab generic package registry operations easy.
|
|
5
5
|
Author-email: bugproduction <bugproduction@outlook.com>
|
|
6
6
|
License-Expression: MIT
|
|
7
7
|
Project-URL: Repository, https://gitlab.com/bugproduction/glpkg.git
|
|
8
8
|
Keywords: GitLab,packages,registry,generic
|
|
9
|
+
Requires-Python: >=3.9
|
|
9
10
|
Description-Content-Type: text/markdown
|
|
10
11
|
License-File: LICENSE.md
|
|
11
|
-
Provides-Extra: dev
|
|
12
|
-
Requires-Dist: black; extra == "dev"
|
|
13
|
-
Requires-Dist: build; extra == "dev"
|
|
14
|
-
Requires-Dist: pytest; extra == "dev"
|
|
15
|
-
Requires-Dist: pytest-cov; extra == "dev"
|
|
16
|
-
Requires-Dist: setuptools; extra == "dev"
|
|
17
|
-
Requires-Dist: twine; extra == "dev"
|
|
18
12
|
Dynamic: license-file
|
|
19
13
|
|
|
20
|
-
# glpkg - GitLab Generic Package
|
|
14
|
+
# glpkg - GitLab Generic Package tools
|
|
21
15
|
|
|
22
|
-
glpkg is a tool that makes it easy to work with [GitLab generic
|
|
16
|
+
glpkg is a tool that makes it easy to work with [GitLab generic packages](https://docs.gitlab.com/user/packages/generic_packages/).
|
|
23
17
|
|
|
24
18
|
|
|
25
19
|
## Installation
|
|
@@ -47,7 +41,11 @@ By default, the used GitLab host is gitlab.com. If you use a self-hosted GitLab,
|
|
|
47
41
|
|
|
48
42
|
To authenticate with the package registry in any of the commands below, use `--token readapitoken123` argument where the `readapitoken123` is a [personal](https://docs.gitlab.com/user/profile/personal_access_tokens/#create-a-personal-access-token) or [project](https://docs.gitlab.com/user/project/settings/project_access_tokens/#create-a-project-access-token) access token, with read API scope. In case the package registry is public, you can omit this argument.
|
|
49
43
|
|
|
50
|
-
|
|
44
|
+
Alternatively you can use a token stored in your `.netrc` file by setting `--netrc` argument.
|
|
45
|
+
|
|
46
|
+
> If you use the tool in GitLab CI, read [below](#Use-in-GitLab-pipelines) on how to use the `CI_JOB_TOKEN`.
|
|
47
|
+
|
|
48
|
+
The arguments related to the GitLab host or authentication (`--token`, `--netrc`, and `--ci`) are omitted in the examples below to focus on the commands.
|
|
51
49
|
|
|
52
50
|
In general, run `glpkg --help` when needed.
|
|
53
51
|
|
|
@@ -74,7 +72,7 @@ mypackagename 2.0
|
|
|
74
72
|
|
|
75
73
|
### Download generic package
|
|
76
74
|
|
|
77
|
-
To download
|
|
75
|
+
To download all files from a specific version of a generic package, run
|
|
78
76
|
|
|
79
77
|
```bash
|
|
80
78
|
glpkg download --project 12345 --name mypackagename --version 1.0
|
|
@@ -85,7 +83,13 @@ Where:
|
|
|
85
83
|
- `mypackagename` is the name of the generic package
|
|
86
84
|
- `1.0` is the version of the generic package from which the files are downloaded
|
|
87
85
|
|
|
88
|
-
|
|
86
|
+
By default the files will be downloaded in the current working directory. To download the files to another directory, add argument `--destination` to the command. In all cases, as long as you have permissions to the destination directory, any pre-existing files will be overridden without warning.
|
|
87
|
+
|
|
88
|
+
To download only a specific file from the package, add `--file` argument.
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
glpkg download --project 12345 --name mypackagename --version 1.5 --file the_only_one --destination /temp
|
|
92
|
+
```
|
|
89
93
|
|
|
90
94
|
> If a package has multiple files with the same filename, the tool can only download the newest file. This is a restriction of GitLab API.
|
|
91
95
|
|
|
@@ -107,7 +111,7 @@ Where:
|
|
|
107
111
|
|
|
108
112
|
### Use in GitLab pipelines
|
|
109
113
|
|
|
110
|
-
If you use the tool in a GitLab pipeline,
|
|
114
|
+
If you use the tool in a GitLab pipeline, setting argument `--ci` uses [GitLab predefined variables](https://docs.gitlab.com/ci/variables/predefined_variables/) to configure the tool. In this case `CI_SERVER_HOST`, `CI_PROJECT_ID`, and `CI_JOB_TOKEN` environment variables are used. The `--project`, and `--token` arguments can still be used to override the project ID or to use a personal or project access token instead of `CI_JOB_TOKEN`.
|
|
111
115
|
|
|
112
116
|
In other words, you don't need to give the `--host`, `--project`, or `--token` arguments if you are interacting with the package registry of the project where the pipeline is running. Example: uploading `my-file.txt` to generic package `mypackagename` version `1.0` in the project package registry in CI:
|
|
113
117
|
|
|
@@ -124,4 +128,3 @@ The tool is not perfect (yet) and has limitations. The following limitations are
|
|
|
124
128
|
|
|
125
129
|
- Uploading files must be done one-by-one.
|
|
126
130
|
- Only project registries are supported for now.
|
|
127
|
-
- Pagination is not supported for now - in case you have more than 100 versions of a package, not all will be shown.
|
|
@@ -9,6 +9,6 @@ src/glpkg.egg-info/PKG-INFO
|
|
|
9
9
|
src/glpkg.egg-info/SOURCES.txt
|
|
10
10
|
src/glpkg.egg-info/dependency_links.txt
|
|
11
11
|
src/glpkg.egg-info/entry_points.txt
|
|
12
|
-
src/glpkg.egg-info/requires.txt
|
|
13
12
|
src/glpkg.egg-info/top_level.txt
|
|
14
|
-
test/
|
|
13
|
+
test/test_cli_handler.py
|
|
14
|
+
test/test_packages.py
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
from unittest.mock import patch
|
|
2
|
+
|
|
3
|
+
import urllib
|
|
4
|
+
|
|
5
|
+
from gitlab import __version__
|
|
6
|
+
from gitlab.cli_handler import CLIHandler
|
|
7
|
+
import sys
|
|
8
|
+
|
|
9
|
+
from utils import mock_empty_response, mock_one_response
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class TestCliHandler:
|
|
13
|
+
|
|
14
|
+
def test_version(self, capsys):
|
|
15
|
+
args = ["glpkg", "-v"]
|
|
16
|
+
with patch.object(sys, "argv", args):
|
|
17
|
+
handler = CLIHandler()
|
|
18
|
+
handler.do_it()
|
|
19
|
+
out, err = capsys.readouterr()
|
|
20
|
+
assert out == __version__ + "\n"
|
|
21
|
+
assert err == ""
|
|
22
|
+
|
|
23
|
+
def test_list_empty(self, mock_empty_response, capsys):
|
|
24
|
+
args = ["glpkg", "list", "--project", "18105942", "--name", "AABCComponent"]
|
|
25
|
+
with patch.object(sys, "argv", args):
|
|
26
|
+
with patch.object(
|
|
27
|
+
urllib.request, "urlopen", return_value=mock_empty_response
|
|
28
|
+
):
|
|
29
|
+
handler = CLIHandler()
|
|
30
|
+
handler.do_it()
|
|
31
|
+
out, err = capsys.readouterr()
|
|
32
|
+
assert out == "Name\t\tVersion\n"
|
|
33
|
+
assert err == ""
|
|
34
|
+
|
|
35
|
+
def test_list_one(self, mock_one_response, capsys):
|
|
36
|
+
args = ["glpkg", "list", "--project", "18105942", "--name", "ABCComponent"]
|
|
37
|
+
with patch.object(sys, "argv", args):
|
|
38
|
+
with patch.object(
|
|
39
|
+
urllib.request, "urlopen", return_value=mock_one_response
|
|
40
|
+
):
|
|
41
|
+
handler = CLIHandler()
|
|
42
|
+
handler.do_it()
|
|
43
|
+
out, err = capsys.readouterr()
|
|
44
|
+
assert out == "Name\t\tVersion\nABCComponent\t0.0.1\n"
|
|
45
|
+
assert err == ""
|
|
@@ -1,14 +1,17 @@
|
|
|
1
|
-
import io
|
|
2
|
-
import pytest
|
|
3
1
|
import urllib
|
|
4
2
|
from gitlab.packages import *
|
|
5
3
|
from unittest.mock import mock_open, patch
|
|
6
4
|
|
|
5
|
+
from utils import (
|
|
6
|
+
ResponseMock,
|
|
7
|
+
test_gitlab,
|
|
8
|
+
mock_empty_response,
|
|
9
|
+
mock_one_response,
|
|
10
|
+
mock_five_response,
|
|
11
|
+
)
|
|
7
12
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def test_gitlab(self):
|
|
11
|
-
return Packages("gl-host", "token-name", "token-value")
|
|
13
|
+
|
|
14
|
+
class TestPackages:
|
|
12
15
|
|
|
13
16
|
def test_api_url(self, test_gitlab):
|
|
14
17
|
assert test_gitlab.api_url() == "https://gl-host/api/v4/"
|
|
@@ -33,79 +36,72 @@ class TestGitLab:
|
|
|
33
36
|
test_gitlab = Packages("gl-host", "", "")
|
|
34
37
|
assert test_gitlab.get_headers() == {}
|
|
35
38
|
|
|
36
|
-
def test_list_packages_none(self, test_gitlab):
|
|
37
|
-
|
|
38
|
-
with patch.object(urllib.request, "urlopen", return_value=data):
|
|
39
|
+
def test_list_packages_none(self, test_gitlab, mock_empty_response):
|
|
40
|
+
with patch.object(urllib.request, "urlopen", return_value=mock_empty_response):
|
|
39
41
|
packages = test_gitlab.list_packages("24", "package-name")
|
|
40
42
|
assert len(packages) == 0
|
|
41
43
|
|
|
42
|
-
def test_list_packages_one(self, test_gitlab):
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
packages = test_gitlab.list_packages("24", "package-name")
|
|
44
|
+
def test_list_packages_one(self, test_gitlab, mock_one_response):
|
|
45
|
+
with patch.object(urllib.request, "urlopen", return_value=mock_one_response):
|
|
46
|
+
packages = test_gitlab.list_packages("18105942", "ABCComponent")
|
|
46
47
|
assert len(packages) == 1
|
|
47
48
|
|
|
48
49
|
def test_list_name_packages_filter(self, test_gitlab):
|
|
49
|
-
data =
|
|
50
|
-
|
|
50
|
+
data = ResponseMock(
|
|
51
|
+
200,
|
|
52
|
+
"",
|
|
53
|
+
'[{"name": "package-name", "version": "0.1.2"}, {"name": "package-name-something", "version": "0.1.2"}]',
|
|
51
54
|
)
|
|
52
55
|
with patch.object(urllib.request, "urlopen", return_value=data):
|
|
53
56
|
packages = test_gitlab.list_packages("24", "package-name")
|
|
54
57
|
assert len(packages) == 1
|
|
55
58
|
|
|
56
|
-
def test_list_name_packages_five(self, test_gitlab):
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
)
|
|
60
|
-
with patch.object(urllib.request, "urlopen", return_value=data):
|
|
61
|
-
packages = test_gitlab.list_packages("24", "package-name")
|
|
59
|
+
def test_list_name_packages_five(self, test_gitlab, mock_five_response):
|
|
60
|
+
with patch.object(urllib.request, "urlopen", return_value=mock_five_response):
|
|
61
|
+
packages = test_gitlab.list_packages("18105942", "ABCComponent")
|
|
62
62
|
assert len(packages) == 5
|
|
63
63
|
|
|
64
|
-
def test_list_files_none(self, test_gitlab):
|
|
65
|
-
|
|
66
|
-
with patch.object(urllib.request, "urlopen", return_value=data):
|
|
64
|
+
def test_list_files_none(self, test_gitlab, mock_empty_response):
|
|
65
|
+
with patch.object(urllib.request, "urlopen", return_value=mock_empty_response):
|
|
67
66
|
packages = test_gitlab.list_files("24", "123")
|
|
68
67
|
assert len(packages) == 0
|
|
69
68
|
|
|
70
69
|
def test_list_files_one(self, test_gitlab):
|
|
71
|
-
data =
|
|
70
|
+
data = ResponseMock(200, "", '[{"file_name": "filea.txt"}]')
|
|
72
71
|
with patch.object(urllib.request, "urlopen", return_value=data):
|
|
73
72
|
packages = test_gitlab.list_files("24", "123")
|
|
74
73
|
assert len(packages) == 1
|
|
75
74
|
|
|
76
75
|
def test_list_files_five(self, test_gitlab):
|
|
77
|
-
data =
|
|
78
|
-
|
|
76
|
+
data = ResponseMock(
|
|
77
|
+
200,
|
|
78
|
+
"",
|
|
79
|
+
'[{"file_name": "filea.txt"}, {"file_name": "fileb.txt"}, {"file_name": "filec.txt"}, {"file_name": "filed.txt"}, {"file_name": "filee.txt"}]',
|
|
79
80
|
)
|
|
80
81
|
with patch.object(urllib.request, "urlopen", return_value=data):
|
|
81
82
|
packages = test_gitlab.list_files("24", "123")
|
|
82
83
|
assert len(packages) == 5
|
|
83
84
|
|
|
84
|
-
def test_package_id_none(self, test_gitlab):
|
|
85
|
-
|
|
86
|
-
with patch.object(urllib.request, "urlopen", return_value=data):
|
|
85
|
+
def test_package_id_none(self, test_gitlab, mock_empty_response):
|
|
86
|
+
with patch.object(urllib.request, "urlopen", return_value=mock_empty_response):
|
|
87
87
|
packages = test_gitlab.get_package_id("24", "package-name", "0.1")
|
|
88
88
|
assert packages == 0
|
|
89
89
|
|
|
90
90
|
def test_package_id_one(self, test_gitlab):
|
|
91
|
-
data =
|
|
91
|
+
data = ResponseMock(200, "", '[{"id": 123}]')
|
|
92
92
|
with patch.object(urllib.request, "urlopen", return_value=data):
|
|
93
93
|
packages = test_gitlab.get_package_id("24", "package-name", "0.1")
|
|
94
94
|
assert packages == 123
|
|
95
95
|
|
|
96
96
|
def test_upload_file(self, test_gitlab):
|
|
97
|
-
|
|
98
|
-
def getcode():
|
|
99
|
-
return 201
|
|
100
|
-
|
|
97
|
+
data = ResponseMock(201, "", "[]")
|
|
101
98
|
with patch("builtins.open", mock_open(read_data="data")):
|
|
102
|
-
with patch.object(urllib.request, "urlopen", return_value=
|
|
99
|
+
with patch.object(urllib.request, "urlopen", return_value=data):
|
|
103
100
|
packages = test_gitlab.upload_file("24", "package-name", "0.1", "file")
|
|
104
101
|
assert packages == 0
|
|
105
102
|
|
|
106
103
|
def test_download_file(self, test_gitlab):
|
|
107
|
-
data =
|
|
108
|
-
m = mock_open()
|
|
104
|
+
data = ResponseMock(200, "", "file-content")
|
|
109
105
|
with patch("builtins.open", mock_open()) as file_mock:
|
|
110
106
|
# mock_open.write.return_value = 0
|
|
111
107
|
with patch.object(urllib.request, "urlopen", return_value=data):
|
|
@@ -1,130 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
from urllib import request, parse
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
class Packages:
|
|
6
|
-
def __init__(self, host: str, token_type: str, token: str):
|
|
7
|
-
self.host = host
|
|
8
|
-
self.token_type = token_type
|
|
9
|
-
self.token = token
|
|
10
|
-
|
|
11
|
-
def api_url(self) -> str:
|
|
12
|
-
return "https://{}/api/v4/".format(parse.quote(self.host))
|
|
13
|
-
|
|
14
|
-
def project_api_url(self, project: str) -> str:
|
|
15
|
-
return self.api_url() + "projects/{}/".format(parse.quote_plus(project))
|
|
16
|
-
|
|
17
|
-
def get_headers(self):
|
|
18
|
-
headers = {}
|
|
19
|
-
if self.token_type and self.token:
|
|
20
|
-
headers = {self.token_type: self.token}
|
|
21
|
-
return headers
|
|
22
|
-
|
|
23
|
-
def list_packages(self, project: str, package_name: str) -> list:
|
|
24
|
-
packages = []
|
|
25
|
-
with request.urlopen(
|
|
26
|
-
request.Request(
|
|
27
|
-
self.project_api_url(project)
|
|
28
|
-
+ "packages?package_name="
|
|
29
|
-
+ parse.quote_plus(package_name),
|
|
30
|
-
headers=self.get_headers(),
|
|
31
|
-
)
|
|
32
|
-
) as res:
|
|
33
|
-
data = res.read()
|
|
34
|
-
for package in json.loads(data):
|
|
35
|
-
name = parse.unquote(package["name"])
|
|
36
|
-
version = parse.unquote(package["version"])
|
|
37
|
-
# The GitLab API returns packages that have some match to the filter;
|
|
38
|
-
# let's filter out non-exact matches
|
|
39
|
-
if package_name != name:
|
|
40
|
-
continue
|
|
41
|
-
packages.append({"name": name, "version": version})
|
|
42
|
-
return packages
|
|
43
|
-
|
|
44
|
-
def list_files(self, project: str, package_id: int) -> list:
|
|
45
|
-
files = []
|
|
46
|
-
with request.urlopen(
|
|
47
|
-
request.Request(
|
|
48
|
-
self.project_api_url(project)
|
|
49
|
-
+ "packages/"
|
|
50
|
-
+ parse.quote_plus(str(package_id))
|
|
51
|
-
+ "/package_files",
|
|
52
|
-
headers=self.get_headers(),
|
|
53
|
-
)
|
|
54
|
-
) as x:
|
|
55
|
-
data = x.read()
|
|
56
|
-
for package in json.loads(
|
|
57
|
-
data,
|
|
58
|
-
):
|
|
59
|
-
# Only append the filename once to the list of files
|
|
60
|
-
# as there's no way to download them separately through
|
|
61
|
-
# the API
|
|
62
|
-
filename = parse.unquote(package["file_name"])
|
|
63
|
-
if filename not in files:
|
|
64
|
-
files.append(filename)
|
|
65
|
-
return files
|
|
66
|
-
|
|
67
|
-
def get_package_id(
|
|
68
|
-
self, project: str, package_name: str, package_version: str
|
|
69
|
-
) -> int:
|
|
70
|
-
id = 0
|
|
71
|
-
with request.urlopen(
|
|
72
|
-
request.Request(
|
|
73
|
-
self.project_api_url(project)
|
|
74
|
-
+ "packages?package_name="
|
|
75
|
-
+ parse.quote_plus(package_name)
|
|
76
|
-
+ "&package_version="
|
|
77
|
-
+ parse.quote_plus(package_version),
|
|
78
|
-
headers=self.get_headers(),
|
|
79
|
-
)
|
|
80
|
-
) as res:
|
|
81
|
-
data = res.read()
|
|
82
|
-
package = json.loads(data)
|
|
83
|
-
if len(package) == 1:
|
|
84
|
-
package = package.pop()
|
|
85
|
-
id = package["id"]
|
|
86
|
-
return id
|
|
87
|
-
|
|
88
|
-
def download_file(
|
|
89
|
-
self, project: str, package_name: str, package_version: str, file: str
|
|
90
|
-
) -> int:
|
|
91
|
-
ret = 1
|
|
92
|
-
with request.urlopen(
|
|
93
|
-
request.Request(
|
|
94
|
-
self.project_api_url(project)
|
|
95
|
-
+ "packages/generic/"
|
|
96
|
-
+ parse.quote_plus(package_name)
|
|
97
|
-
+ "/"
|
|
98
|
-
+ parse.quote_plus(package_version)
|
|
99
|
-
+ "/"
|
|
100
|
-
+ parse.quote(str(file)),
|
|
101
|
-
headers=self.get_headers(),
|
|
102
|
-
)
|
|
103
|
-
) as req:
|
|
104
|
-
with open(str(file), "wb") as file:
|
|
105
|
-
file.write(req.read())
|
|
106
|
-
ret = 0
|
|
107
|
-
return ret
|
|
108
|
-
|
|
109
|
-
def upload_file(
|
|
110
|
-
self, project: str, package_name: str, package_version: str, file: str
|
|
111
|
-
) -> int:
|
|
112
|
-
ret = 1
|
|
113
|
-
with open(str(file), "rb") as data:
|
|
114
|
-
res = request.urlopen(
|
|
115
|
-
request.Request(
|
|
116
|
-
self.project_api_url(project)
|
|
117
|
-
+ "packages/generic/"
|
|
118
|
-
+ parse.quote_plus(package_name)
|
|
119
|
-
+ "/"
|
|
120
|
-
+ parse.quote_plus(package_version)
|
|
121
|
-
+ "/"
|
|
122
|
-
+ parse.quote(str(file)),
|
|
123
|
-
method="PUT",
|
|
124
|
-
data=data,
|
|
125
|
-
headers=self.get_headers(),
|
|
126
|
-
)
|
|
127
|
-
)
|
|
128
|
-
if res.getcode() == 201: # 201 is created
|
|
129
|
-
ret = 0
|
|
130
|
-
return ret
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|