glpkg 0.0.1__tar.gz → 1.1.0__tar.gz

{glpkg-0.0.1 → glpkg-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: glpkg
-Version: 0.0.1
+Version: 1.1.0
 Summary: Tool to make GitLab generic package registry operations easy.
 Author-email: bugproduction <bugproduction@outlook.com>
 License-Expression: MIT
@@ -124,4 +124,3 @@ The tool is not perfect (yet) and has limitations. The following limitations are
 
 - Uploading files must be done one-by-one.
 - Only project registries are supported for now.
-- Pagination is not supported for now - in case you have more than 100 versions of a package, not all will be shown.

{glpkg-0.0.1 → glpkg-1.1.0}/README.md

@@ -104,5 +104,4 @@ To use the `CI_JOB_TOKEN` with package registry of another projects, add `--proj
 The tool is not perfect (yet) and has limitations. The following limitations are known, but more can exist:
 
 - Uploading files must be done one-by-one.
-- Only project registries are supported for now.
-- Pagination is not supported for now - in case you have more than 100 versions of a package, not all will be shown.
+- Only project registries are supported for now.

{glpkg-0.0.1 → glpkg-1.1.0}/src/gitlab/__init__.py

@@ -1,3 +1,3 @@
 from gitlab.packages import Packages
 
-__version__ = "0.0.1"
+__version__ = "1.1.0"

{glpkg-0.0.1 → glpkg-1.1.0}/src/gitlab/cli_handler.py

@@ -1,4 +1,5 @@
 import argparse
+import netrc
 import os
 from gitlab import Packages, __version__
 
@@ -47,7 +48,7 @@ class CLIHandler:
             "-c",
             "--ci",
             action="store_true",
-            help="Use this to run the tool in GitLab pipelines. In this case CI_SERVER_HOST, CI_PROJECT_ID, and CI_JOB_TOKEN variables from the environment are used. --project and --token can be used to override project ID and the CI_JOB_TOKEN to a personal or project access token.",
+            help="Use this in GitLab jobs. In this case CI_SERVER_HOST, CI_PROJECT_ID, and CI_JOB_TOKEN variables from the environment are used. --project and --token can be used to override project ID and the CI_JOB_TOKEN to a personal or project access token.",
         )
         parser.add_argument(
             "-p",
@@ -56,12 +57,18 @@ class CLIHandler:
             help="The project ID or path. For example 123456 or namespace/project.",
         )
         parser.add_argument("-n", "--name", type=str, help="The package name.")
-        parser.add_argument(
+        group2 = parser.add_mutually_exclusive_group()
+        group2.add_argument(
             "-t",
             "--token",
             type=str,
             help="Private or project access token that is used to authenticate with the package registry. Leave empty if the registry is public. The token must have 'read API' or 'API' scope.",
         )
+        group2.add_argument(
+            "--netrc",
+            action="store_true",
+            help="Set to use a token from .netrc file (~/.netrc) for the host. The .netrc username is ignored due to API restrictions. PRIVATE-TOKEN is used instead. Note that .netrc file access rights must be correct.",
+        )
 
     def _register_download_parser(self, parser):
         self._register_common_arguments(parser)
@@ -71,35 +78,40 @@ class CLIHandler:
     def _args(self, args):
         if args.ci:
             host = os.environ["CI_SERVER_HOST"]
+            project = os.environ["CI_PROJECT_ID"]
+            token = os.environ["CI_JOB_TOKEN"]
+            token_user = "JOB-TOKEN"
             if args.project:
                 project = args.project
-            else:
-                project = os.environ["CI_PROJECT_ID"]
             if args.token:
-                token_user = "PRIVATE-TOKEN"
                 token = args.token
-            else:
-                token_user = "JOB-TOKEN"
-                token = os.environ["CI_JOB_TOKEN"]
+                token_user = "PRIVATE-TOKEN"
         else:
             host = args.host
             project = args.project
-            token_user = "PRIVATE-TOKEN"
             token = args.token
+            token_user = "PRIVATE-TOKEN"
+        if args.netrc:
+            _, _, token = netrc.netrc().authenticators(host)
+            token_user = "PRIVATE-TOKEN"
         name = args.name
         return host, project, name, token_user, token
 
     def _download_handler(self, args) -> int:
+        ret = 1
         host, project, name, token_user, token = self._args(args)
         version = args.version
         gitlab = Packages(host, token_user, token)
         package_id = gitlab.get_package_id(project, name, version)
-        files = gitlab.list_files(project, package_id)
-        ret = 1
-        for file in files:
-            ret = gitlab.download_file(project, name, version, file)
-            if not ret:
-                break
+        if package_id:
+            files = gitlab.list_files(project, package_id)
+            for file in files:
+                ret = gitlab.download_file(project, name, version, file)
+                if ret:
+                    print("Failed to download file " + file)
+                    break
+        else:
+            print("No package " + name + " version " + version + " found!")
         return ret
 
     def _register_list_parser(self, parser):
@@ -126,9 +138,13 @@ class CLIHandler:
         parser.set_defaults(action=self._upload)
 
     def _upload(self, args) -> int:
+        ret = 1
         host, project, name, token_user, token = self._args(args)
         version = args.version
         file = args.file
-        gitlab = Packages(host, token_user, token)
-        ret = gitlab.upload_file(project, name, version, file)
+        if os.path.isfile(file):
+            gitlab = Packages(host, token_user, token)
+            ret = gitlab.upload_file(project, name, version, file)
+        else:
+            print("File " + file + " does not exist!")
         return ret

glpkg-1.1.0/src/gitlab/packages.py

@@ -0,0 +1,163 @@
+from http.client import HTTPMessage
+import json
+import logging
+from urllib import request, parse
+
+logger = logging.getLogger(__name__)
+
+
+class Packages:
+    def __init__(self, host: str, token_type: str, token: str):
+        self.host = host
+        self.token_type = token_type
+        self.token = token
+
+    def api_url(self) -> str:
+        return "https://{}/api/v4/".format(parse.quote(self.host))
+
+    def project_api_url(self, project: str) -> str:
+        return self.api_url() + "projects/{}/".format(parse.quote_plus(project))
+
+    def get_headers(self) -> dict:
+        headers = {}
+        if self.token_type and self.token:
+            headers = {self.token_type: self.token}
+        return headers
+
+    def _request(self, url: str) -> tuple[int, bytes, HTTPMessage]:
+        logger.debug("Requesting " + url)
+        req = request.Request(url, headers=self.get_headers())
+        with request.urlopen(req) as response:
+            return response.status, response.read(), response.headers
+
+    def _get_next_page(self, headers: HTTPMessage) -> int:
+        ret = 0
+        if headers:
+            next_page = headers.get("x-next-page")
+            if next_page:
+                ret = int(next_page)
+                logger.debug("Response incomplete, next page is " + next_page)
+            else:
+                logger.debug("Response complete")
+        return ret
+
+    def _build_query(self, arg: str, page: int) -> str:
+        query = ""
+        if arg or page:
+            if page:
+                page = "page=" + str(page)
+            query = "?{}".format("&".join(filter(None, (arg, page))))
+        return query
+
+    def gl_project_api(self, project: str, path: str, arg: str = None) -> list:
+        data = []
+        more = True
+        page = None
+        while more:
+            more = False
+            query = self._build_query(arg, page)
+            url = self.project_api_url(project) + path + query
+            status, res_data, headers = self._request(url)
+            logger.debug("Response status: " + str(status))
+            res_data = json.loads(res_data)
+            logger.debug("Response data: " + str(res_data))
+            data = data + res_data
+            page = self._get_next_page(headers)
+            if page:
+                more = True
+        return data
+
+    def list_packages(self, project: str, package_name: str) -> list:
+        packages = []
+        logger.debug("Listing packages with name " + package_name)
+        data = self.gl_project_api(
+            project, "packages", "package_name=" + parse.quote_plus(package_name)
+        )
+        for package in data:
+            name = parse.unquote(package["name"])
+            version = parse.unquote(package["version"])
+            # GitLab API returns packages that have some match to the filter;
+            # let's filter out non-exact matches
+            if package_name != name:
+                continue
+            packages.append({"name": name, "version": version})
+        return packages
+
+    def list_files(self, project: str, package_id: int) -> list:
+        files = []
+        logger.debug("Listing package " + str(package_id) + " files")
+        path = "packages/" + parse.quote_plus(str(package_id)) + "/package_files"
+        data = self.gl_project_api(project, path)
+        for package in data:
+            # Only append the filename once to the list of files
+            # as there's no way to download them separately through
+            # the API
+            filename = parse.unquote(package["file_name"])
+            if filename not in files:
+                files.append(filename)
+        return files
+
+    def get_package_id(
+        self, project: str, package_name: str, package_version: str
+    ) -> int:
+        id = 0
+        logger.debug(
+            "Fetching package " + package_name + " (" + package_version + ") ID"
+        )
+        path = "packages"
+        arg = (
+            "package_name="
+            + parse.quote_plus(package_name)
+            + "&package_version="
+            + parse.quote_plus(package_version)
+        )
+        data = self.gl_project_api(project, path, arg)
+        if len(data) == 1:
+            package = data.pop()
+            id = package["id"]
+        return id
+
+    def download_file(
+        self, project: str, package_name: str, package_version: str, file: str
+    ) -> int:
+        ret = 1
+        logger.debug("Downloading file " + file)
+        url = (
+            self.project_api_url(project)
+            + "packages/generic/"
+            + parse.quote_plus(package_name)
+            + "/"
+            + parse.quote_plus(package_version)
+            + "/"
+            + parse.quote(str(file))
+        )
+        status, data, _ = self._request(url)
+        if status == 200:
+            with open(str(file), "wb") as file:
+                file.write(data)
+                ret = 0
+        return ret
+
+    def upload_file(
+        self, project: str, package_name: str, package_version: str, file: str
+    ) -> int:
+        ret = 1
+        logger.debug("Uploading file " + file)
+        with open(str(file), "rb") as data:
+            url = (
+                self.project_api_url(project)
+                + "packages/generic/"
+                + parse.quote_plus(package_name)
+                + "/"
+                + parse.quote_plus(package_version)
+                + "/"
+                + parse.quote(str(file))
+            )
+            res = request.urlopen(
+                request.Request(
+                    url, method="PUT", data=data, headers=self.get_headers()
+                )
+            )
+            if res.status == 201:  # 201 is created
+                ret = 0
+        return ret

{glpkg-0.0.1 → glpkg-1.1.0}/src/glpkg.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: glpkg
-Version: 0.0.1
+Version: 1.1.0
 Summary: Tool to make GitLab generic package registry operations easy.
 Author-email: bugproduction <bugproduction@outlook.com>
 License-Expression: MIT
@@ -124,4 +124,3 @@ The tool is not perfect (yet) and has limitations. The following limitations are
 
 - Uploading files must be done one-by-one.
 - Only project registries are supported for now.
-- Pagination is not supported for now - in case you have more than 100 versions of a package, not all will be shown.

{glpkg-0.0.1 → glpkg-1.1.0}/src/glpkg.egg-info/SOURCES.txt

@@ -11,4 +11,5 @@ src/glpkg.egg-info/dependency_links.txt
 src/glpkg.egg-info/entry_points.txt
 src/glpkg.egg-info/requires.txt
 src/glpkg.egg-info/top_level.txt
-test/test_gitlab.py
+test/test_cli_handler.py
+test/test_packages.py

glpkg-1.1.0/test/test_cli_handler.py

@@ -0,0 +1,45 @@
+from unittest.mock import patch
+
+import urllib
+
+from gitlab import __version__
+from gitlab.cli_handler import CLIHandler
+import sys
+
+from utils import mock_empty_response, mock_one_response
+
+
+class TestCliHandler:
+
+    def test_version(self, capsys):
+        args = ["glpkg", "-v"]
+        with patch.object(sys, "argv", args):
+            handler = CLIHandler()
+            handler.do_it()
+            out, err = capsys.readouterr()
+            assert out == __version__ + "\n"
+            assert err == ""
+
+    def test_list_empty(self, mock_empty_response, capsys):
+        args = ["glpkg", "list", "--project", "18105942", "--name", "AABCComponent"]
+        with patch.object(sys, "argv", args):
+            with patch.object(
+                urllib.request, "urlopen", return_value=mock_empty_response
+            ):
+                handler = CLIHandler()
+                handler.do_it()
+                out, err = capsys.readouterr()
+                assert out == "Name\t\tVersion\n"
+                assert err == ""
+
+    def test_list_one(self, mock_one_response, capsys):
+        args = ["glpkg", "list", "--project", "18105942", "--name", "ABCComponent"]
+        with patch.object(sys, "argv", args):
+            with patch.object(
+                urllib.request, "urlopen", return_value=mock_one_response
+            ):
+                handler = CLIHandler()
+                handler.do_it()
+                out, err = capsys.readouterr()
+                assert out == "Name\t\tVersion\nABCComponent\t0.0.1\n"
+                assert err == ""

glpkg-0.0.1/test/test_gitlab.py → glpkg-1.1.0/test/test_packages.py

@@ -1,14 +1,17 @@
-import io
-import pytest
 import urllib
 from gitlab.packages import *
 from unittest.mock import mock_open, patch
 
+from utils import (
+    ResponseMock,
+    test_gitlab,
+    mock_empty_response,
+    mock_one_response,
+    mock_five_response,
+)
 
-class TestGitLab:
-    @pytest.fixture
-    def test_gitlab(self):
-        return Packages("gl-host", "token-name", "token-value")
+
+class TestPackages:
 
     def test_api_url(self, test_gitlab):
         assert test_gitlab.api_url() == "https://gl-host/api/v4/"
@@ -33,79 +36,72 @@ class TestGitLab:
         test_gitlab = Packages("gl-host", "", "")
         assert test_gitlab.get_headers() == {}
 
-    def test_list_packages_none(self, test_gitlab):
-        data = io.StringIO("[]")
-        with patch.object(urllib.request, "urlopen", return_value=data):
+    def test_list_packages_none(self, test_gitlab, mock_empty_response):
+        with patch.object(urllib.request, "urlopen", return_value=mock_empty_response):
             packages = test_gitlab.list_packages("24", "package-name")
             assert len(packages) == 0
 
-    def test_list_packages_one(self, test_gitlab):
-        data = io.StringIO('[{"name": "package-name", "version": "0.1.2"}]')
-        with patch.object(urllib.request, "urlopen", return_value=data):
-            packages = test_gitlab.list_packages("24", "package-name")
+    def test_list_packages_one(self, test_gitlab, mock_one_response):
+        with patch.object(urllib.request, "urlopen", return_value=mock_one_response):
+            packages = test_gitlab.list_packages("18105942", "ABCComponent")
             assert len(packages) == 1
 
     def test_list_name_packages_filter(self, test_gitlab):
-        data = io.StringIO(
-            '[{"name": "package-name", "version": "0.1.2"}, {"name": "package-name-something", "version": "0.1.2"}]'
+        data = ResponseMock(
+            200,
+            "",
+            '[{"name": "package-name", "version": "0.1.2"}, {"name": "package-name-something", "version": "0.1.2"}]',
         )
         with patch.object(urllib.request, "urlopen", return_value=data):
             packages = test_gitlab.list_packages("24", "package-name")
             assert len(packages) == 1
 
-    def test_list_name_packages_five(self, test_gitlab):
-        data = io.StringIO(
-            '[{"name": "package-name", "version": "0.1"}, {"name": "package-name", "version": "0.2"}, {"name": "package-name", "version": "0.3"}, {"name": "package-name", "version": "0.4"}, {"name": "package-name", "version": "0.5"}]'
-        )
-        with patch.object(urllib.request, "urlopen", return_value=data):
-            packages = test_gitlab.list_packages("24", "package-name")
+    def test_list_name_packages_five(self, test_gitlab, mock_five_response):
+        with patch.object(urllib.request, "urlopen", return_value=mock_five_response):
+            packages = test_gitlab.list_packages("18105942", "ABCComponent")
             assert len(packages) == 5
 
-    def test_list_files_none(self, test_gitlab):
-        data = io.StringIO("[]")
-        with patch.object(urllib.request, "urlopen", return_value=data):
+    def test_list_files_none(self, test_gitlab, mock_empty_response):
+        with patch.object(urllib.request, "urlopen", return_value=mock_empty_response):
             packages = test_gitlab.list_files("24", "123")
             assert len(packages) == 0
 
     def test_list_files_one(self, test_gitlab):
-        data = io.StringIO('[{"file_name": "filea.txt"}]')
+        data = ResponseMock(200, "", '[{"file_name": "filea.txt"}]')
         with patch.object(urllib.request, "urlopen", return_value=data):
             packages = test_gitlab.list_files("24", "123")
             assert len(packages) == 1
 
     def test_list_files_five(self, test_gitlab):
-        data = io.StringIO(
-            '[{"file_name": "filea.txt"}, {"file_name": "fileb.txt"}, {"file_name": "filec.txt"}, {"file_name": "filed.txt"}, {"file_name": "filee.txt"}]'
+        data = ResponseMock(
+            200,
+            "",
+            '[{"file_name": "filea.txt"}, {"file_name": "fileb.txt"}, {"file_name": "filec.txt"}, {"file_name": "filed.txt"}, {"file_name": "filee.txt"}]',
         )
         with patch.object(urllib.request, "urlopen", return_value=data):
             packages = test_gitlab.list_files("24", "123")
             assert len(packages) == 5
 
-    def test_package_id_none(self, test_gitlab):
-        data = io.StringIO("[]")
-        with patch.object(urllib.request, "urlopen", return_value=data):
+    def test_package_id_none(self, test_gitlab, mock_empty_response):
+        with patch.object(urllib.request, "urlopen", return_value=mock_empty_response):
             packages = test_gitlab.get_package_id("24", "package-name", "0.1")
             assert packages == 0
 
     def test_package_id_one(self, test_gitlab):
-        data = io.StringIO('[{"id": 123}]')
+        data = ResponseMock(200, "", '[{"id": 123}]')
         with patch.object(urllib.request, "urlopen", return_value=data):
             packages = test_gitlab.get_package_id("24", "package-name", "0.1")
             assert packages == 123
 
     def test_upload_file(self, test_gitlab):
-        class rmock:
-            def getcode():
-                return 201
-
+        data = ResponseMock(201, "", "[]")
         with patch("builtins.open", mock_open(read_data="data")):
-            with patch.object(urllib.request, "urlopen", return_value=rmock):
+            with patch.object(urllib.request, "urlopen", return_value=data):
                 packages = test_gitlab.upload_file("24", "package-name", "0.1", "file")
                 assert packages == 0
 
     def test_download_file(self, test_gitlab):
-        data = io.StringIO("file-content")
-        m = mock_open()
+        data = ResponseMock(200, "", "file-content")
         with patch("builtins.open", mock_open()) as file_mock:
             # mock_open.write.return_value = 0
             with patch.object(urllib.request, "urlopen", return_value=data):

glpkg-0.0.1/src/gitlab/packages.py

@@ -1,130 +0,0 @@
-import json
-from urllib import request, parse
-
-
-class Packages:
-    def __init__(self, host: str, token_type: str, token: str):
-        self.host = host
-        self.token_type = token_type
-        self.token = token
-
-    def api_url(self) -> str:
-        return "https://{}/api/v4/".format(parse.quote(self.host))
-
-    def project_api_url(self, project: str) -> str:
-        return self.api_url() + "projects/{}/".format(parse.quote_plus(project))
-
-    def get_headers(self):
-        headers = {}
-        if self.token_type and self.token:
-            headers = {self.token_type: self.token}
-        return headers
-
-    def list_packages(self, project: str, package_name: str) -> list:
-        packages = []
-        with request.urlopen(
-            request.Request(
-                self.project_api_url(project)
-                + "packages?package_name="
-                + parse.quote_plus(package_name),
-                headers=self.get_headers(),
-            )
-        ) as res:
-            data = res.read()
-            for package in json.loads(data):
-                name = parse.unquote(package["name"])
-                version = parse.unquote(package["version"])
-                # The GitLab API returns packages that have some match to the filter;
-                # let's filter out non-exact matches
-                if package_name != name:
-                    continue
-                packages.append({"name": name, "version": version})
-        return packages
-
-    def list_files(self, project: str, package_id: int) -> list:
-        files = []
-        with request.urlopen(
-            request.Request(
-                self.project_api_url(project)
-                + "packages/"
-                + parse.quote_plus(str(package_id))
-                + "/package_files",
-                headers=self.get_headers(),
-            )
-        ) as x:
-            data = x.read()
-            for package in json.loads(
-                data,
-            ):
-                # Only append the filename once to the list of files
-                # as there's no way to download them separately through
-                # the API
-                filename = parse.unquote(package["file_name"])
-                if filename not in files:
-                    files.append(filename)
-        return files
-
-    def get_package_id(
-        self, project: str, package_name: str, package_version: str
-    ) -> int:
-        id = 0
-        with request.urlopen(
-            request.Request(
-                self.project_api_url(project)
-                + "packages?package_name="
-                + parse.quote_plus(package_name)
-                + "&package_version="
-                + parse.quote_plus(package_version),
-                headers=self.get_headers(),
-            )
-        ) as res:
-            data = res.read()
-            package = json.loads(data)
-            if len(package) == 1:
-                package = package.pop()
-                id = package["id"]
-        return id
-
-    def download_file(
-        self, project: str, package_name: str, package_version: str, file: str
-    ) -> int:
-        ret = 1
-        with request.urlopen(
-            request.Request(
-                self.project_api_url(project)
-                + "packages/generic/"
-                + parse.quote_plus(package_name)
-                + "/"
-                + parse.quote_plus(package_version)
-                + "/"
-                + parse.quote(str(file)),
-                headers=self.get_headers(),
-            )
-        ) as req:
-            with open(str(file), "wb") as file:
-                file.write(req.read())
-                ret = 0
-        return ret
-
-    def upload_file(
-        self, project: str, package_name: str, package_version: str, file: str
-    ) -> int:
-        ret = 1
-        with open(str(file), "rb") as data:
-            res = request.urlopen(
-                request.Request(
-                    self.project_api_url(project)
-                    + "packages/generic/"
-                    + parse.quote_plus(package_name)
-                    + "/"
-                    + parse.quote_plus(package_version)
-                    + "/"
-                    + parse.quote(str(file)),
-                    method="PUT",
-                    data=data,
-                    headers=self.get_headers(),
-                )
-            )
-            if res.getcode() == 201:  # 201 is created
-                ret = 0
-        return ret