gitlabber 1.2.2__tar.gz → 1.2.4__tar.gz

{gitlabber-1.2.2/gitlabber.egg-info → gitlabber-1.2.4}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: gitlabber
-Version: 1.2.2
+Version: 1.2.4
 Summary: A Gitlab clone/pull utility for backing up or cloning Gitlab groups
 Home-page: https://github.com/ezbz/gitlabber
-Download-URL: https://github.com/ezbz/gitlabber/archive/master.zip
+Download-URL: https://github.com/ezbz/gitlabber/archive/main.zip
 Author: Erez Mazor
 Author-email: erezmazor@gmail.com
 License: MIT
@@ -21,20 +21,13 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3
-Description-Content-Type: text/markdown
+Description-Content-Type: text/x-rst
 License-File: LICENSE.txt
-Requires-Dist: docopt
-Requires-Dist: anytree
-Requires-Dist: globre
-Requires-Dist: pyyaml
-Requires-Dist: tqdm
-Requires-Dist: GitPython
-Requires-Dist: python-gitlab
 
 .. image:: https://github.com/ezbz/gitlabber/actions/workflows/python-app.yml/badge.svg?branch=master
     :target: https://github.com/ezbz/gitlabber/actions/workflows/python-app.yml
 
-.. image:: https://codecov.io/gh/ezbz/gitlabber/branch/master/graph/badge.svg
+.. image:: https://codecov.io/gh/ezbz/gitlabber/branch/main/graph/badge.svg
   :target: https://codecov.io/gh/ezbz/gitlabber
   
 .. image:: https://badge.fury.io/py/gitlabber.svg
@@ -62,7 +55,6 @@ Purpose
 Gitlabber clones or pulls all projects under a subset of groups / subgroups by building a tree from the GitLab API and allowing you to specify which subset of the tree you want to clone using glob patterns and/or regex expressions.
 
 
-
 Installation
 ------------
 
@@ -73,12 +65,13 @@ Installation
     pip install gitlabber
 
 * You'll need to create an `access token <https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html>`_ from GitLab with API scopes `read_repository`
-  and `read_api` (or `api`, for GitLab versions <12.0)
+  and ``read_api`` (or ``api``, for GitLab versions <12.0)
 
 Usage
 -----
 
-* Arguments can be provided via the CLI arguments directly or via environment variables::
+* Arguments can be provided via the CLI arguments directly or via environment variables:
+
     +---------------+---------------+---------------------------+
     | Argument      | Flag          | Environment Variable      |
     +===============+===============+===========================+
@@ -95,7 +88,7 @@ Usage
     | exclude       | -x            | `GITLABBER_EXCLUDE`       |
     +---------------+---------------+---------------------------+
 
-* To view the tree run the command with your includes/excludes and the `-p` flag. It will print your tree like so:
+* To view the tree run the command with your includes/excludes and the ``-p`` flag. It will print your tree like so:
 
 .. code-block:: bash
 
@@ -111,72 +104,78 @@ Usage
 
 * To see how to use glob patterns and regex to filter tree nodes, see the `globre project page <https://pypi.org/project/globre/#details>`_.
 
+* Include/Exclude patterns do not work at the API level but work on the results returned from the API, for large Gitlab installations this can take a lot of time, if you need to reduce the amound of API calls for such projects use the ``--group-search`` parameter to search only for the top level groups the interest you using the `Gitlab Group Search API <https://docs.gitlab.com/ee/api/groups.html#search-for-group>` which allows you to do a partial like query for a Group's path or name
+
 * Cloning vs Pulling: when running Gitlabber consecutively with the same parameters, it will scan the local tree structure; if the project directory exists and is a valid git repository (has .git folder in it) Gitlabber will perform a git pull in the directory, otherwise the project directory will be created and the GitLab project will be cloned into it.
 
-* Cloning submodules: use the `-r` flag to recurse git submodules, uses the `--recursive` for cloning and utilizes `GitPython's smart update method <https://github.com/gitpython-developers/GitPython/blob/20f4a9d49b466a18f1af1fdfb480bc4520a4cdc2/git/objects/submodule/root.py#L67>`_ for updating cloned repositories
+* Cloning submodules: use the ``-r`` flag to recurse git submodules, uses the ``--recursive`` for cloning and utilizes `GitPython's smart update method <https://github.com/gitpython-developers/GitPython/blob/20f4a9d49b466a18f1af1fdfb480bc4520a4cdc2/git/objects/submodule/root.py#L67>`_ for updating cloned repositories
 
 * Printed Usage:
 
 .. code-block:: bash
 
- usage: gitlabber [-h] [-t token] [-T] [-u url] [--verbose] [-p] [--print-format {json,yaml,tree}] [-n {name,path}] [-m {ssh,http}] [-a {include,exclude,only}] [-i csv] [-x csv] [-r] [-F] [-d] [-s] [--version]
-                 [dest]
+    usage: gitlabber [-h] [-t token] [-T] [-u url] [--verbose] [-p] [--print-format {json,yaml,tree}] [-n {name,path}] [-m {ssh,http}]
+                    [-a {include,exclude,only}] [-i csv] [-x csv] [-r] [-F] [-d] [-s] [-g term] [--version]
+                    [dest]
 
     Gitlabber - clones or pulls entire groups/projects tree from gitlab
-    
+
     positional arguments:
-      dest                  destination path for the cloned tree (created if doesn't exist)
-    
+    dest                  destination path for the cloned tree (created if doesn't exist)
+
     options:
-      -h, --help            show this help message and exit
-      -t token, --token token
+    -h, --help            show this help message and exit
+    -t token, --token token
                             gitlab personal access token https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html
-      -T, --hide-token      use an inline URL token (avoids storing the gitlab personal access token in the .git/config)
-      -u url, --url url     base gitlab url (e.g.: 'http://gitlab.mycompany.com')
-      --verbose             print more verbose output
-      -p, --print           print the tree without cloning
-      --print-format {json,yaml,tree}
+    -T, --hide-token      use an inline URL token (avoids storing the gitlab personal access token in the .git/config)
+    -u url, --url url     base gitlab url (e.g.: 'http://gitlab.mycompany.com')
+    --verbose             print more verbose output
+    -p, --print           print the tree without cloning
+    --print-format {json,yaml,tree}
                             print format (default: 'tree')
-      -n {name,path}, --naming {name,path}
+    -n {name,path}, --naming {name,path}
                             the folder naming strategy for projects from the gitlab API attributes (default: "name")
-      -m {ssh,http}, --method {ssh,http}
+    -m {ssh,http}, --method {ssh,http}
                             the git transport method to use for cloning (default: "ssh")
-      -a {include,exclude,only}, --archived {include,exclude,only}
+    -a {include,exclude,only}, --archived {include,exclude,only}
                             include archived projects and groups in the results (default: "include")
-      -i csv, --include csv
+    -i csv, --include csv
                             comma delimited list of glob patterns of paths to projects or groups to clone/pull
-      -x csv, --exclude csv
+    -x csv, --exclude csv
                             comma delimited list of glob patterns of paths to projects or groups to exclude from clone/pull
-      -r, --recursive       clone/pull git submodules recursively
-      -F, --use-fetch       clone/fetch git repository (mirrored repositories)
-      -d, --dont-checkout   don't checkout pulled git repository
-      -s, --include-shared  include shared projects in the results
-      --version             print the version
-    
+    -r, --recursive       clone/pull git submodules recursively
+    -F, --use-fetch       clone/fetch git repository (mirrored repositories)
+    -d, --dont-checkout   don't checkout pulled git repository
+    -s, --include-shared  include shared projects in the results
+    -g term, --group-search term
+                            only include groups matching the search term, filtering done at the API level (useful for large projects, see: https://docs.gitlab.com/ee/api/groups.html#search-for-group works with partial names of path or name)
+    --version             print the version
+
     examples:
-    
+
         clone an entire gitlab tree using a url and a token:
         gitlabber -t <personal access token> -u <gitlab url>
-    
+
         only print the gitlab tree:
         gitlabber -p .
-    
+
         clone only projects under subgroup 'MySubGroup' to location '~/GitlabRoot':
         gitlabber -i '/MyGroup/MySubGroup**' ~/GitlabRoot
-    
+
         clone only projects under group 'MyGroup' excluding any projects under subgroup 'MySubGroup':
         gitlabber -i '/MyGroup**' -x '/MyGroup/MySubGroup**' .
-    
+
         clone an entire gitlab tree except projects under groups named 'ArchiveGroup':
         gitlabber -x '/ArchiveGroup**' .
-    
+
         clone projects that start with a case insensitive 'w' using a regular expression:
         gitlabber -i '/{[w].*}' .
 
 
+
 Debugging
 ---------
-* You can use the `--verbose` flag to print Gitlabber debug messages
+* You can use the ``--verbose`` flag to print Gitlabber debug messages
 * For more verbose GitLab messages, you can get the `GitPython <https://gitpython.readthedocs.io/en/stable>`_ module to print more debug messages by setting the environment variable:
 
 .. code-block:: bash
@@ -185,10 +184,11 @@ Debugging
 
 Troubleshooting
 ---------------
-* `GitlabHttpError: 503`: make sure you provide the base URL to your GitLab installation (e.g., `https://gitlab.my.com` and not `https://gitlab.my.com/some/nested/path`)
-
+* ``GitlabHttpError: 503``: make sure you provide the base URL to your GitLab installation (e.g., `https://gitlab.my.com` and not `https://gitlab.my.com/some/nested/path`)
+* ``git.exc.GitCommandError: Cmd('git') failed due to: exit code(128)`` OR ``ERROR: The project you were looking for could not be found or you don't have permission to view it.``: if you are using Git's SSH method, follow the `SSH Guide <https://docs.gitlab.com/ee/user/ssh.html>`_ from Gitlab and ensure you have your SSH key in Gitlab for clone or use the HTTP method (``-m http`` flag)
+  
 Known Limitations
 -----------------
 * Renaming, moving and deleting projects: Gitlabber doesn't maintain local tree state (projects and groups). For that reason is does not rename move or delete local projects when they are modified on the server. When projects are moved or renamed, Gitlabber will clone them again under their new name or location. When deleted, Gitlabber will not delete the local project.
-* Folder naming strategy: Consecutively running Gitlabber with different values for the `-n` parameter will produce undesirable results. Use the same value as previous runs, or simply don't change it from the default (project name).
+* Folder naming strategy: Consecutively running Gitlabber with different values for the ``-n`` parameter will produce undesirable results. Use the same value as previous runs, or simply don't change it from the default (project name).
 * If you're going to clone a large number of projects, observe rate limits `for gitlab.com <https://docs.gitlab.com/ee/user/gitlab_com/index.html#gitlabcom-specific-rate-limits/>`_, and `for on-premise installations <https://docs.gitlab.com/ee/security/rate_limits.html>`_.

{gitlabber-1.2.2 → gitlabber-1.2.4}/README.rst

@@ -1,7 +1,7 @@
 .. image:: https://github.com/ezbz/gitlabber/actions/workflows/python-app.yml/badge.svg?branch=master
     :target: https://github.com/ezbz/gitlabber/actions/workflows/python-app.yml
 
-.. image:: https://codecov.io/gh/ezbz/gitlabber/branch/master/graph/badge.svg
+.. image:: https://codecov.io/gh/ezbz/gitlabber/branch/main/graph/badge.svg
   :target: https://codecov.io/gh/ezbz/gitlabber
   
 .. image:: https://badge.fury.io/py/gitlabber.svg
@@ -29,7 +29,6 @@ Purpose
 Gitlabber clones or pulls all projects under a subset of groups / subgroups by building a tree from the GitLab API and allowing you to specify which subset of the tree you want to clone using glob patterns and/or regex expressions.
 
 
-
 Installation
 ------------
 
@@ -40,12 +39,13 @@ Installation
     pip install gitlabber
 
 * You'll need to create an `access token <https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html>`_ from GitLab with API scopes `read_repository`
-  and `read_api` (or `api`, for GitLab versions <12.0)
+  and ``read_api`` (or ``api``, for GitLab versions <12.0)
 
 Usage
 -----
 
-* Arguments can be provided via the CLI arguments directly or via environment variables::
+* Arguments can be provided via the CLI arguments directly or via environment variables:
+
     +---------------+---------------+---------------------------+
     | Argument      | Flag          | Environment Variable      |
     +===============+===============+===========================+
@@ -62,7 +62,7 @@ Usage
     | exclude       | -x            | `GITLABBER_EXCLUDE`       |
     +---------------+---------------+---------------------------+
 
-* To view the tree run the command with your includes/excludes and the `-p` flag. It will print your tree like so:
+* To view the tree run the command with your includes/excludes and the ``-p`` flag. It will print your tree like so:
 
 .. code-block:: bash
 
@@ -78,72 +78,78 @@ Usage
 
 * To see how to use glob patterns and regex to filter tree nodes, see the `globre project page <https://pypi.org/project/globre/#details>`_.
 
+* Include/Exclude patterns do not work at the API level but work on the results returned from the API, for large Gitlab installations this can take a lot of time, if you need to reduce the amound of API calls for such projects use the ``--group-search`` parameter to search only for the top level groups the interest you using the `Gitlab Group Search API <https://docs.gitlab.com/ee/api/groups.html#search-for-group>` which allows you to do a partial like query for a Group's path or name
+
 * Cloning vs Pulling: when running Gitlabber consecutively with the same parameters, it will scan the local tree structure; if the project directory exists and is a valid git repository (has .git folder in it) Gitlabber will perform a git pull in the directory, otherwise the project directory will be created and the GitLab project will be cloned into it.
 
-* Cloning submodules: use the `-r` flag to recurse git submodules, uses the `--recursive` for cloning and utilizes `GitPython's smart update method <https://github.com/gitpython-developers/GitPython/blob/20f4a9d49b466a18f1af1fdfb480bc4520a4cdc2/git/objects/submodule/root.py#L67>`_ for updating cloned repositories
+* Cloning submodules: use the ``-r`` flag to recurse git submodules, uses the ``--recursive`` for cloning and utilizes `GitPython's smart update method <https://github.com/gitpython-developers/GitPython/blob/20f4a9d49b466a18f1af1fdfb480bc4520a4cdc2/git/objects/submodule/root.py#L67>`_ for updating cloned repositories
 
 * Printed Usage:
 
 .. code-block:: bash
 
- usage: gitlabber [-h] [-t token] [-T] [-u url] [--verbose] [-p] [--print-format {json,yaml,tree}] [-n {name,path}] [-m {ssh,http}] [-a {include,exclude,only}] [-i csv] [-x csv] [-r] [-F] [-d] [-s] [--version]
-                 [dest]
+    usage: gitlabber [-h] [-t token] [-T] [-u url] [--verbose] [-p] [--print-format {json,yaml,tree}] [-n {name,path}] [-m {ssh,http}]
+                    [-a {include,exclude,only}] [-i csv] [-x csv] [-r] [-F] [-d] [-s] [-g term] [--version]
+                    [dest]
 
     Gitlabber - clones or pulls entire groups/projects tree from gitlab
-    
+
     positional arguments:
-      dest                  destination path for the cloned tree (created if doesn't exist)
-    
+    dest                  destination path for the cloned tree (created if doesn't exist)
+
     options:
-      -h, --help            show this help message and exit
-      -t token, --token token
+    -h, --help            show this help message and exit
+    -t token, --token token
                             gitlab personal access token https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html
-      -T, --hide-token      use an inline URL token (avoids storing the gitlab personal access token in the .git/config)
-      -u url, --url url     base gitlab url (e.g.: 'http://gitlab.mycompany.com')
-      --verbose             print more verbose output
-      -p, --print           print the tree without cloning
-      --print-format {json,yaml,tree}
+    -T, --hide-token      use an inline URL token (avoids storing the gitlab personal access token in the .git/config)
+    -u url, --url url     base gitlab url (e.g.: 'http://gitlab.mycompany.com')
+    --verbose             print more verbose output
+    -p, --print           print the tree without cloning
+    --print-format {json,yaml,tree}
                             print format (default: 'tree')
-      -n {name,path}, --naming {name,path}
+    -n {name,path}, --naming {name,path}
                             the folder naming strategy for projects from the gitlab API attributes (default: "name")
-      -m {ssh,http}, --method {ssh,http}
+    -m {ssh,http}, --method {ssh,http}
                             the git transport method to use for cloning (default: "ssh")
-      -a {include,exclude,only}, --archived {include,exclude,only}
+    -a {include,exclude,only}, --archived {include,exclude,only}
                             include archived projects and groups in the results (default: "include")
-      -i csv, --include csv
+    -i csv, --include csv
                             comma delimited list of glob patterns of paths to projects or groups to clone/pull
-      -x csv, --exclude csv
+    -x csv, --exclude csv
                             comma delimited list of glob patterns of paths to projects or groups to exclude from clone/pull
-      -r, --recursive       clone/pull git submodules recursively
-      -F, --use-fetch       clone/fetch git repository (mirrored repositories)
-      -d, --dont-checkout   don't checkout pulled git repository
-      -s, --include-shared  include shared projects in the results
-      --version             print the version
-    
+    -r, --recursive       clone/pull git submodules recursively
+    -F, --use-fetch       clone/fetch git repository (mirrored repositories)
+    -d, --dont-checkout   don't checkout pulled git repository
+    -s, --include-shared  include shared projects in the results
+    -g term, --group-search term
+                            only include groups matching the search term, filtering done at the API level (useful for large projects, see: https://docs.gitlab.com/ee/api/groups.html#search-for-group works with partial names of path or name)
+    --version             print the version
+
     examples:
-    
+
         clone an entire gitlab tree using a url and a token:
         gitlabber -t <personal access token> -u <gitlab url>
-    
+
         only print the gitlab tree:
         gitlabber -p .
-    
+
         clone only projects under subgroup 'MySubGroup' to location '~/GitlabRoot':
         gitlabber -i '/MyGroup/MySubGroup**' ~/GitlabRoot
-    
+
         clone only projects under group 'MyGroup' excluding any projects under subgroup 'MySubGroup':
         gitlabber -i '/MyGroup**' -x '/MyGroup/MySubGroup**' .
-    
+
         clone an entire gitlab tree except projects under groups named 'ArchiveGroup':
         gitlabber -x '/ArchiveGroup**' .
-    
+
         clone projects that start with a case insensitive 'w' using a regular expression:
         gitlabber -i '/{[w].*}' .
 
 
+
 Debugging
 ---------
-* You can use the `--verbose` flag to print Gitlabber debug messages
+* You can use the ``--verbose`` flag to print Gitlabber debug messages
 * For more verbose GitLab messages, you can get the `GitPython <https://gitpython.readthedocs.io/en/stable>`_ module to print more debug messages by setting the environment variable:
 
 .. code-block:: bash
@@ -152,10 +158,11 @@ Debugging
 
 Troubleshooting
 ---------------
-* `GitlabHttpError: 503`: make sure you provide the base URL to your GitLab installation (e.g., `https://gitlab.my.com` and not `https://gitlab.my.com/some/nested/path`)
-
+* ``GitlabHttpError: 503``: make sure you provide the base URL to your GitLab installation (e.g., `https://gitlab.my.com` and not `https://gitlab.my.com/some/nested/path`)
+* ``git.exc.GitCommandError: Cmd('git') failed due to: exit code(128)`` OR ``ERROR: The project you were looking for could not be found or you don't have permission to view it.``: if you are using Git's SSH method, follow the `SSH Guide <https://docs.gitlab.com/ee/user/ssh.html>`_ from Gitlab and ensure you have your SSH key in Gitlab for clone or use the HTTP method (``-m http`` flag)
+  
 Known Limitations
 -----------------
 * Renaming, moving and deleting projects: Gitlabber doesn't maintain local tree state (projects and groups). For that reason is does not rename move or delete local projects when they are modified on the server. When projects are moved or renamed, Gitlabber will clone them again under their new name or location. When deleted, Gitlabber will not delete the local project.
-* Folder naming strategy: Consecutively running Gitlabber with different values for the `-n` parameter will produce undesirable results. Use the same value as previous runs, or simply don't change it from the default (project name).
+* Folder naming strategy: Consecutively running Gitlabber with different values for the ``-n`` parameter will produce undesirable results. Use the same value as previous runs, or simply don't change it from the default (project name).
 * If you're going to clone a large number of projects, observe rate limits `for gitlab.com <https://docs.gitlab.com/ee/user/gitlab_com/index.html#gitlabcom-specific-rate-limits/>`_, and `for on-premise installations <https://docs.gitlab.com/ee/security/rate_limits.html>`_.

gitlabber-1.2.4/gitlabber/__init__.py

@@ -0,0 +1 @@
+__version__ = '1.2.4'

{gitlabber-1.2.2 → gitlabber-1.2.4}/gitlabber/cli.py

@@ -37,9 +37,13 @@ def main():
     includes=split(args.include)
     excludes=split(args.exclude)
 
+    args_print = vars(args).copy()
+    args_print['token'] = '__hidden__'
+    log.debug("running with args [%s]", args_print)
+
     tree = GitlabTree(args.url, args.token, args.method, args.naming, args.archived.api_value, includes,
                       excludes, args.file, args.concurrency, args.recursive, args.verbose,
-                      args.use_fetch, args.hide_token)
+                      args.use_fetch, args.hide_token, group_search=args.group_search)
     log.debug("Reading projects tree from gitlab at [%s]", args.url)
     tree.load_tree()
 
@@ -202,16 +206,17 @@ def parse_args(argv=None):
         action='store_true',
         default=True,
         help='include shared projects in the results')
+    parser.add_argument(
+        '-g',
+        '--group-search',
+        metavar=('term'),
+        help='only include groups matching the search term, filtering done at the API level (useful for large projects, see: https://docs.gitlab.com/ee/api/groups.html#search-for-group works with partial names of path or name)')
     parser.add_argument(
         '--version',
         action='store_true',
         help='print the version')
 
-    args = parser.parse_args(argv)
-    args_print = vars(args).copy()
-    args_print['token'] = 'xxxxx'
-    log.debug("running with args [%s]", args_print)
-    return args
+    return parser.parse_args(argv)
 
 def validate_path(value):
     if value.endswith('/'):

{gitlabber-1.2.2 → gitlabber-1.2.4}/gitlabber/git.py

@@ -25,6 +25,7 @@ def sync_tree(root, dest, concurrency=1, disable_progress=False, recursive=False
     actions = get_git_actions(root, dest, recursive, use_fetch, hide_token)
     with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor:
         executor.map(clone_or_pull_project, actions)
+    
     elapsed = progress.finish_progress()
     log.debug("Syncing projects took [%s]", elapsed)
 
@@ -57,6 +58,7 @@ def clone_or_pull_project(action):
         '''
         log.debug("updating existing project %s", action.path)
         progress.show_progress(action.node.name, 'pull')
+        
         try:
             repo = git.Repo(action.path)
             if(not action.use_fetch):
@@ -74,6 +76,9 @@ def clone_or_pull_project(action):
         '''
         Clone new project
         '''
+        if(action.node.type != "project"):
+            log.debug("Skipping clone of node with type [%s] (empty subgroup/group)", action.node.type)
+            return
         log.debug("cloning new project %s", action.path)
         progress.show_progress(action.node.name, 'clone')
         multi_options = []
@@ -88,5 +93,5 @@ def clone_or_pull_project(action):
             log.fatal("User interrupted")
             sys.exit(0)
         except Exception as e:
-            log.debug("Error cloning project %s", action.path, exc_info=True)
+            log.error("Error cloning project %s", action.path, exc_info=True)
 

{gitlabber-1.2.2 → gitlabber-1.2.4}/gitlabber/gitlab_tree.py

@@ -19,11 +19,11 @@ log = logging.getLogger(__name__)
 class GitlabTree:
 
     def __init__(self, url, token, method, naming=None, archived=None, includes=[], excludes=[], in_file=None, concurrency=1, recursive=False, disable_progress=False,
-                include_shared=True, use_fetch=False, hide_token=False):
+                include_shared=True, use_fetch=False, hide_token=False, group_search=None):
         self.includes = includes
         self.excludes = excludes
         self.url = url
-        self.root = Node("", root_path="", url=url)
+        self.root = Node("", root_path="", url=url, type="root")
         self.gitlab = Gitlab(url, private_token=token,
                              ssl_verify=GitlabTree.get_ca_path())
         self.method = method
@@ -38,6 +38,7 @@ class GitlabTree:
         self.include_shared = include_shared
         self.use_fetch = use_fetch
         self.hide_token = hide_token
+        self.group_search = group_search
 
     @staticmethod
     def get_ca_path():
@@ -93,8 +94,8 @@ class GitlabTree:
     def root_path(self, node):
         return "/".join([str(n.name) for n in node.path])
 
-    def make_node(self, name, parent, url):
-        node = Node(name=name, parent=parent, url=url)
+    def make_node(self, type, name, parent, url):
+        node = Node(name=name, parent=parent, url=url, type=type)
         node.root_path = self.root_path(node)
         return node
 
@@ -108,7 +109,7 @@ class GitlabTree:
                   log.debug("Generated URL: %s", project_url)
               else:
                   log.debug("Hiding token from project url: %s", project_url)
-            node = self.make_node(project_id, parent,
+            node = self.make_node("project", project_id, parent,
                                   url=project_url)
             self.progress.show_progress(node.name, 'project')
 
@@ -118,16 +119,16 @@ class GitlabTree:
             self.progress.update_progress_length(len(projects))
             self.add_projects(parent, projects)
         except GitlabListError as error:
-            log.error(f"Error getting projects on {group.name} (ID: {group.id}): {error} )")
+            log.error(f"Error getting projects on {group.name} id: [{group.id}]  error message: [{error.error_message}]")
 
     def get_subgroups(self, group, parent):
-        subgroups = group.subgroups.list(as_list=False, archived=self.archived, get_all=True)
+        subgroups = group.subgroups.list(as_list=False, get_all=True)
         self.progress.update_progress_length(len(subgroups))
         for subgroup_def in subgroups:
             try:
                 subgroup = self.gitlab.groups.get(subgroup_def.id)
                 subgroup_id = subgroup.name if self.naming == FolderNaming.NAME else subgroup.path
-                node = self.make_node(subgroup_id, parent, url=subgroup.web_url)
+                node = self.make_node("subgroup", subgroup_id, parent, url=subgroup.web_url)
                 self.progress.show_progress(node.name, 'group')
                 self.get_subgroups(subgroup, node)
                 self.get_projects(subgroup, node)
@@ -139,12 +140,14 @@ class GitlabTree:
                     raise error
 
     def load_gitlab_tree(self):
-        groups = self.gitlab.groups.list(as_list=False, archived=self.archived, get_all=True)
+        log.debug(f"Starting group search with archived: {self.archived} search term: {self.group_search}")
+                    
+        groups = self.gitlab.groups.list(as_list=False, archived=self.archived, get_all=True, search=self.group_search)
         self.progress.init_progress(len(groups))
         for group in groups:
             if group.parent_id is None:
                 group_id = group.name if self.naming == FolderNaming.NAME else group.path
-                node = self.make_node(group_id, self.root, url=group.web_url)
+                node = self.make_node("group", group_id, self.root, url=group.web_url)
                 self.progress.show_progress(node.name, 'group')
                 self.get_subgroups(group, node)
                 self.get_projects(group, node)        

{gitlabber-1.2.2 → gitlabber-1.2.4}/gitlabber/progress.py

@@ -7,13 +7,12 @@ class ProgressBar:
         self.progress = None
         self.description = description
         self.disabled = disabled
-        self.start = None
+        self.start = time.time()
 
     def init_progress(self, total):
         if self.progress is None:
-            self.start = time.time()
             self.progress = tqdm(total=total, unit="projects",
-                                 bar_format="{desc}: {percentage:.1f}%|{bar:80}| {n_fmt}/{total_fmt}{postfix}", desc=self.description, leave=False, disable=self.disabled)
+                                 bar_format="{desc}: {percentage:.1f}%|{bar:80}| {n_fmt}/{total_fmt}{postfix}", desc=self.description, leave=False, disable=self.disabled)        
 
     def update_progress_length(self, added):
         if self.progress is not None:
@@ -29,7 +28,7 @@ class ProgressBar:
     def finish_progress(self):
         if self.progress is not None:
             self.progress.close()
-            end = time.time()
-            hours, rem = divmod(end-self.start, 3600)
-            minutes, seconds = divmod(rem, 60)
-            return "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds)
+        end = time.time()   
+        hours, rem = divmod(end-self.start, 3600)
+        minutes, seconds = divmod(rem, 60)
+        return "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds)