gitlabber 1.2.8__tar.gz → 2.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. {gitlabber-1.2.8/gitlabber.egg-info → gitlabber-2.0.0}/PKG-INFO +123 -37
  2. {gitlabber-1.2.8 → gitlabber-2.0.0}/README.rst +118 -32
  3. gitlabber-2.0.0/gitlabber/__init__.py +8 -0
  4. gitlabber-2.0.0/gitlabber/__main__.py +8 -0
  5. gitlabber-2.0.0/gitlabber/archive.py +40 -0
  6. gitlabber-2.0.0/gitlabber/auth.py +62 -0
  7. gitlabber-2.0.0/gitlabber/cli.py +492 -0
  8. gitlabber-2.0.0/gitlabber/config.py +98 -0
  9. gitlabber-2.0.0/gitlabber/exceptions.py +137 -0
  10. gitlabber-2.0.0/gitlabber/format.py +20 -0
  11. gitlabber-2.0.0/gitlabber/git.py +412 -0
  12. gitlabber-2.0.0/gitlabber/gitlab_tree.py +327 -0
  13. gitlabber-2.0.0/gitlabber/method.py +18 -0
  14. gitlabber-2.0.0/gitlabber/naming.py +18 -0
  15. gitlabber-2.0.0/gitlabber/progress.py +192 -0
  16. gitlabber-2.0.0/gitlabber/rate_limiter.py +93 -0
  17. gitlabber-2.0.0/gitlabber/tree_builder.py +565 -0
  18. gitlabber-2.0.0/gitlabber/url_builder.py +78 -0
  19. {gitlabber-1.2.8 → gitlabber-2.0.0/gitlabber.egg-info}/PKG-INFO +123 -37
  20. gitlabber-2.0.0/gitlabber.egg-info/SOURCES.txt +118 -0
  21. {gitlabber-1.2.8 → gitlabber-2.0.0}/gitlabber.egg-info/requires.txt +4 -2
  22. {gitlabber-1.2.8 → gitlabber-2.0.0}/pyproject.toml +27 -7
  23. gitlabber-2.0.0/tests/__pycache__/__init__.cpython-311.pyc +0 -0
  24. gitlabber-2.0.0/tests/__pycache__/__init__.cpython-312.pyc +0 -0
  25. gitlabber-2.0.0/tests/__pycache__/__init__.cpython-313.pyc +0 -0
  26. gitlabber-2.0.0/tests/__pycache__/conftest.cpython-313-pytest-9.0.1.pyc +0 -0
  27. gitlabber-2.0.0/tests/__pycache__/gitlab_test_utils.cpython-311.pyc +0 -0
  28. gitlabber-2.0.0/tests/__pycache__/gitlab_test_utils.cpython-312.pyc +0 -0
  29. gitlabber-2.0.0/tests/__pycache__/gitlab_test_utils.cpython-313.pyc +0 -0
  30. gitlabber-2.0.0/tests/__pycache__/io_test_util.cpython-311.pyc +0 -0
  31. gitlabber-2.0.0/tests/__pycache__/io_test_util.cpython-312.pyc +0 -0
  32. gitlabber-2.0.0/tests/__pycache__/io_test_util.cpython-313.pyc +0 -0
  33. gitlabber-2.0.0/tests/__pycache__/test_archive.cpython-311-pytest-8.2.2.pyc +0 -0
  34. gitlabber-2.0.0/tests/__pycache__/test_archive.cpython-312-pytest-8.3.3.pyc +0 -0
  35. gitlabber-2.0.0/tests/__pycache__/test_archive.cpython-312-pytest-8.3.4.pyc +0 -0
  36. gitlabber-2.0.0/tests/__pycache__/test_archive.cpython-312-pytest-8.3.5.pyc +0 -0
  37. gitlabber-2.0.0/tests/__pycache__/test_archive.cpython-313-pytest-9.0.1.pyc +0 -0
  38. gitlabber-2.0.0/tests/__pycache__/test_auth.cpython-313-pytest-9.0.1.pyc +0 -0
  39. gitlabber-2.0.0/tests/__pycache__/test_cli.cpython-311-pytest-8.2.2.pyc +0 -0
  40. gitlabber-2.0.0/tests/__pycache__/test_cli.cpython-312-pytest-8.3.3.pyc +0 -0
  41. gitlabber-2.0.0/tests/__pycache__/test_cli.cpython-312-pytest-8.3.4.pyc +0 -0
  42. gitlabber-2.0.0/tests/__pycache__/test_cli.cpython-312-pytest-8.3.5.pyc +0 -0
  43. gitlabber-2.0.0/tests/__pycache__/test_cli.cpython-313-pytest-9.0.1.pyc +0 -0
  44. gitlabber-2.0.0/tests/__pycache__/test_config.cpython-313-pytest-9.0.1.pyc +0 -0
  45. gitlabber-2.0.0/tests/__pycache__/test_e2e.cpython-311-pytest-8.2.2.pyc +0 -0
  46. gitlabber-2.0.0/tests/__pycache__/test_e2e.cpython-312-pytest-8.3.3.pyc +0 -0
  47. gitlabber-2.0.0/tests/__pycache__/test_e2e.cpython-312-pytest-8.3.4.pyc +0 -0
  48. gitlabber-2.0.0/tests/__pycache__/test_e2e.cpython-312-pytest-8.3.5.pyc +0 -0
  49. gitlabber-2.0.0/tests/__pycache__/test_e2e.cpython-313-pytest-9.0.1.pyc +0 -0
  50. gitlabber-2.0.0/tests/__pycache__/test_e2e.cpython-313.pyc +0 -0
  51. gitlabber-2.0.0/tests/__pycache__/test_format.cpython-311-pytest-8.2.2.pyc +0 -0
  52. gitlabber-2.0.0/tests/__pycache__/test_format.cpython-312-pytest-8.3.3.pyc +0 -0
  53. gitlabber-2.0.0/tests/__pycache__/test_format.cpython-312-pytest-8.3.4.pyc +0 -0
  54. gitlabber-2.0.0/tests/__pycache__/test_format.cpython-312-pytest-8.3.5.pyc +0 -0
  55. gitlabber-2.0.0/tests/__pycache__/test_format.cpython-313-pytest-9.0.1.pyc +0 -0
  56. gitlabber-2.0.0/tests/__pycache__/test_git.cpython-311-pytest-8.2.2.pyc +0 -0
  57. gitlabber-2.0.0/tests/__pycache__/test_git.cpython-312-pytest-8.3.3.pyc +0 -0
  58. gitlabber-2.0.0/tests/__pycache__/test_git.cpython-312-pytest-8.3.4.pyc +0 -0
  59. gitlabber-2.0.0/tests/__pycache__/test_git.cpython-312-pytest-8.3.5.pyc +0 -0
  60. gitlabber-2.0.0/tests/__pycache__/test_git.cpython-313-pytest-9.0.1.pyc +0 -0
  61. gitlabber-2.0.0/tests/__pycache__/test_gitlab_tree.cpython-311-pytest-8.2.2.pyc +0 -0
  62. gitlabber-2.0.0/tests/__pycache__/test_gitlab_tree.cpython-312-pytest-8.3.3.pyc +0 -0
  63. gitlabber-2.0.0/tests/__pycache__/test_gitlab_tree.cpython-312-pytest-8.3.4.pyc +0 -0
  64. gitlabber-2.0.0/tests/__pycache__/test_gitlab_tree.cpython-312-pytest-8.3.5.pyc +0 -0
  65. gitlabber-2.0.0/tests/__pycache__/test_gitlab_tree.cpython-313-pytest-9.0.1.pyc +0 -0
  66. gitlabber-2.0.0/tests/__pycache__/test_helpers.cpython-313-pytest-9.0.1.pyc +0 -0
  67. gitlabber-2.0.0/tests/__pycache__/test_integration.cpython-311-pytest-8.2.2.pyc +0 -0
  68. gitlabber-2.0.0/tests/__pycache__/test_integration.cpython-312-pytest-8.3.3.pyc +0 -0
  69. gitlabber-2.0.0/tests/__pycache__/test_integration.cpython-312-pytest-8.3.4.pyc +0 -0
  70. gitlabber-2.0.0/tests/__pycache__/test_integration.cpython-312-pytest-8.3.5.pyc +0 -0
  71. gitlabber-2.0.0/tests/__pycache__/test_integration.cpython-313-pytest-9.0.1.pyc +0 -0
  72. gitlabber-2.0.0/tests/__pycache__/test_main.cpython-313-pytest-9.0.1.pyc +0 -0
  73. gitlabber-2.0.0/tests/__pycache__/test_method.cpython-311-pytest-8.2.2.pyc +0 -0
  74. gitlabber-2.0.0/tests/__pycache__/test_method.cpython-312-pytest-8.3.3.pyc +0 -0
  75. gitlabber-2.0.0/tests/__pycache__/test_method.cpython-312-pytest-8.3.4.pyc +0 -0
  76. gitlabber-2.0.0/tests/__pycache__/test_method.cpython-312-pytest-8.3.5.pyc +0 -0
  77. gitlabber-2.0.0/tests/__pycache__/test_method.cpython-313-pytest-9.0.1.pyc +0 -0
  78. gitlabber-2.0.0/tests/__pycache__/test_naming.cpython-311-pytest-8.2.2.pyc +0 -0
  79. gitlabber-2.0.0/tests/__pycache__/test_naming.cpython-312-pytest-8.3.3.pyc +0 -0
  80. gitlabber-2.0.0/tests/__pycache__/test_naming.cpython-312-pytest-8.3.4.pyc +0 -0
  81. gitlabber-2.0.0/tests/__pycache__/test_naming.cpython-312-pytest-8.3.5.pyc +0 -0
  82. gitlabber-2.0.0/tests/__pycache__/test_naming.cpython-313-pytest-9.0.1.pyc +0 -0
  83. gitlabber-2.0.0/tests/__pycache__/test_performance.cpython-313-pytest-9.0.1.pyc +0 -0
  84. gitlabber-2.0.0/tests/__pycache__/test_performance.cpython-313.pyc +0 -0
  85. gitlabber-2.0.0/tests/__pycache__/test_progress.cpython-313-pytest-9.0.1.pyc +0 -0
  86. gitlabber-2.0.0/tests/__pycache__/test_rate_limiter.cpython-313-pytest-9.0.1.pyc +0 -0
  87. gitlabber-2.0.0/tests/__pycache__/test_url_builder.cpython-313-pytest-9.0.1.pyc +0 -0
  88. gitlabber-2.0.0/tests/conftest.py +92 -0
  89. {gitlabber-1.2.8 → gitlabber-2.0.0}/tests/gitlab_test_utils.py +65 -32
  90. gitlabber-2.0.0/tests/io_test_util.py +53 -0
  91. gitlabber-2.0.0/tests/test_archive.py +27 -0
  92. gitlabber-2.0.0/tests/test_auth.py +51 -0
  93. gitlabber-2.0.0/tests/test_cli.py +114 -0
  94. gitlabber-2.0.0/tests/test_config.py +93 -0
  95. {gitlabber-1.2.8 → gitlabber-2.0.0}/tests/test_e2e.py +65 -14
  96. gitlabber-2.0.0/tests/test_format.py +23 -0
  97. gitlabber-2.0.0/tests/test_git.py +219 -0
  98. {gitlabber-1.2.8 → gitlabber-2.0.0}/tests/test_gitlab_tree.py +61 -7
  99. gitlabber-2.0.0/tests/test_helpers.py +391 -0
  100. gitlabber-2.0.0/tests/test_integration.py +86 -0
  101. gitlabber-2.0.0/tests/test_main.py +16 -0
  102. gitlabber-2.0.0/tests/test_method.py +20 -0
  103. gitlabber-2.0.0/tests/test_naming.py +20 -0
  104. gitlabber-2.0.0/tests/test_performance.py +236 -0
  105. gitlabber-2.0.0/tests/test_progress.py +99 -0
  106. gitlabber-2.0.0/tests/test_rate_limiter.py +218 -0
  107. gitlabber-2.0.0/tests/test_url_builder.py +95 -0
  108. gitlabber-1.2.8/gitlabber/__init__.py +0 -2
  109. gitlabber-1.2.8/gitlabber/__main__.py +0 -2
  110. gitlabber-1.2.8/gitlabber/archive.py +0 -23
  111. gitlabber-1.2.8/gitlabber/cli.py +0 -236
  112. gitlabber-1.2.8/gitlabber/format.py +0 -19
  113. gitlabber-1.2.8/gitlabber/git.py +0 -100
  114. gitlabber-1.2.8/gitlabber/gitlab_tree.py +0 -236
  115. gitlabber-1.2.8/gitlabber/method.py +0 -19
  116. gitlabber-1.2.8/gitlabber/naming.py +0 -18
  117. gitlabber-1.2.8/gitlabber/progress.py +0 -34
  118. gitlabber-1.2.8/gitlabber.egg-info/SOURCES.txt +0 -38
  119. gitlabber-1.2.8/tests/io_test_util.py +0 -25
  120. gitlabber-1.2.8/tests/test_archive.py +0 -22
  121. gitlabber-1.2.8/tests/test_cli.py +0 -139
  122. gitlabber-1.2.8/tests/test_format.py +0 -17
  123. gitlabber-1.2.8/tests/test_git.py +0 -184
  124. gitlabber-1.2.8/tests/test_integration.py +0 -28
  125. gitlabber-1.2.8/tests/test_method.py +0 -18
  126. gitlabber-1.2.8/tests/test_naming.py +0 -17
  127. {gitlabber-1.2.8 → gitlabber-2.0.0}/LICENSE.txt +0 -0
  128. {gitlabber-1.2.8 → gitlabber-2.0.0}/MANIFEST.in +0 -0
  129. {gitlabber-1.2.8 → gitlabber-2.0.0}/gitlabber.egg-info/dependency_links.txt +0 -0
  130. {gitlabber-1.2.8 → gitlabber-2.0.0}/gitlabber.egg-info/entry_points.txt +0 -0
  131. {gitlabber-1.2.8 → gitlabber-2.0.0}/gitlabber.egg-info/top_level.txt +0 -0
  132. {gitlabber-1.2.8 → gitlabber-2.0.0}/setup.cfg +0 -0
  133. {gitlabber-1.2.8 → gitlabber-2.0.0}/setup.py +0 -0
  134. {gitlabber-1.2.8 → gitlabber-2.0.0}/tests/__init__.py +0 -0
  135. {gitlabber-1.2.8 → gitlabber-2.0.0}/tests/test-input.yaml +0 -0
  136. {gitlabber-1.2.8 → gitlabber-2.0.0}/tests/test-output.json +0 -0
  137. {gitlabber-1.2.8 → gitlabber-2.0.0}/tests/test-output.tree +0 -0
  138. {gitlabber-1.2.8 → gitlabber-2.0.0}/tests/test-output.yaml +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: gitlabber
3
- Version: 1.2.8
3
+ Version: 2.0.0
4
4
  Summary: A Gitlab clone/pull utility for backing up or cloning Gitlab groups
5
5
  Author-email: Erez Mazor <erezmazor@gmail.com>
6
6
  License: MIT
@@ -14,19 +14,19 @@ Classifier: License :: OSI Approved :: MIT License
14
14
  Classifier: Natural Language :: English
15
15
  Classifier: Operating System :: OS Independent
16
16
  Classifier: Programming Language :: Python :: 3
17
- Classifier: Programming Language :: Python :: 3.9
18
- Classifier: Programming Language :: Python :: 3.10
19
17
  Classifier: Programming Language :: Python :: 3.11
20
18
  Classifier: Programming Language :: Python :: 3.12
21
19
  Classifier: Programming Language :: Python :: 3.13
22
- Requires-Python: >=3
20
+ Requires-Python: >=3.11
23
21
  Description-Content-Type: text/x-rst
24
22
  License-File: LICENSE.txt
25
- Requires-Dist: docopt
26
23
  Requires-Dist: anytree
27
24
  Requires-Dist: globre
28
25
  Requires-Dist: pyyaml
29
- Requires-Dist: tqdm
26
+ Requires-Dist: pydantic>=2.7
27
+ Requires-Dist: pydantic-settings>=2.7
28
+ Requires-Dist: typer>=0.12
29
+ Requires-Dist: rich
30
30
  Requires-Dist: GitPython
31
31
  Requires-Dist: python-gitlab
32
32
  Provides-Extra: test
@@ -70,54 +70,84 @@ Gitlabber clones or pulls all projects under a subset of groups / subgroups by b
70
70
  Installation
71
71
  ------------
72
72
 
73
- * You can install Gitlabber from `PyPi <https://pypi.org/project/gitlabber>`_:
73
+ System Requirements
74
+ -------------------
75
+ * Python 3.11 or higher
76
+ * Git 2.0 or higher
77
+ * Network access to GitLab instance
74
78
 
75
- .. code-block:: bash
79
+ Installation Methods
80
+ ~~~~~~~~~~~~~~~~~~~~
81
+ * PyPI (recommended):
76
82
 
77
- pip install gitlabber
83
+ .. code-block:: bash
84
+
85
+ pip install gitlabber
86
+
87
+ * From source:
88
+
89
+ .. code-block:: bash
90
+
91
+ git clone https://github.com/ezbz/gitlabber.git
92
+ cd gitlabber
93
+ pip install -e .
78
94
 
79
95
  * You'll need to create an `access token <https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html>`_ from GitLab with API scopes `read_repository`
80
96
  and ``read_api`` (or ``api``, for GitLab versions <12.0)
81
97
 
98
+ Quick Start
99
+ -----------
100
+
101
+ .. code-block:: bash
102
+
103
+ # Install gitlabber
104
+ pip install gitlabber
105
+
106
+ # Clone all your GitLab projects
107
+ gitlabber -t <your-token> -u <gitlab-url> .
108
+
82
109
  Usage
83
110
  -----
84
111
 
85
112
  * Arguments can be provided via the CLI arguments directly or via environment variables:
86
113
 
87
- +---------------+---------------+---------------------------+
88
- | Argument | Flag | Environment Variable |
89
- +===============+===============+===========================+
90
- | token | -t | `GITLAB_TOKEN` |
91
- +---------------+---------------+---------------------------+
92
- | url | -u | `GITLAB_URL` |
93
- +---------------+---------------+---------------------------+
94
- | method | -m | `GITLABBER_CLONE_METHOD` |
95
- +---------------+---------------+---------------------------+
96
- | naming | -n | `GITLABBER_FOLDER_NAMING` |
97
- +---------------+---------------+---------------------------+
98
- | include | -i | `GITLABBER_INCLUDE` |
99
- +---------------+---------------+---------------------------+
100
- | exclude | -x | `GITLABBER_EXCLUDE` |
101
- +---------------+---------------+---------------------------+
114
+ +------------------+------------------+---------------------------+
115
+ | Argument | Flag | Environment Variable |
116
+ +==================+==================+===========================+
117
+ | token | -t | `GITLAB_TOKEN` |
118
+ +------------------+------------------+---------------------------+
119
+ | url | -u | `GITLAB_URL` |
120
+ +------------------+------------------+---------------------------+
121
+ | method | -m | `GITLABBER_CLONE_METHOD` |
122
+ +------------------+------------------+---------------------------+
123
+ | naming | -n | `GITLABBER_FOLDER_NAMING` |
124
+ +------------------+------------------+---------------------------+
125
+ | include | -i | `GITLABBER_INCLUDE` |
126
+ +------------------+------------------+---------------------------+
127
+ | exclude | -x | `GITLABBER_EXCLUDE` |
128
+ +------------------+------------------+---------------------------+
102
129
 
103
130
  * To view the tree run the command with your includes/excludes and the ``-p`` flag. It will print your tree like so:
104
131
 
105
- .. code-block:: bash
132
+ .. code-block:: bash
133
+
134
+ root [http://gitlab.my.com]
135
+ ├── group1 [/group1]
136
+ │ └── subgroup1 [/group1/subgroup1]
137
+ │ └── project1 [/group1/subgroup1/project1]
138
+ └── group2 [/group2]
139
+ ├── subgroup1 [/group2/subgroup1]
140
+ │ └── project2 [/group2/subgroup1/project2]
141
+ ├── subgroup2 [/group2/subgroup2]
142
+ └── subgroup3 [/group2/subgroup3]
106
143
 
107
- root [http://gitlab.my.com]
108
- ├── group1 [/group1]
109
- │ └── subgroup1 [/group1/subgroup1]
110
- │ └── project1 [/group1/subgroup1/project1]
111
- └── group2 [/group2]
112
- ├── subgroup1 [/group2/subgroup1]
113
- │ └── project2 [/group2/subgroup1/project2]
114
- ├── subgroup2 [/group2/subgroup2]
115
- └── subgroup3 [/group2/subgroup3]
116
144
 
117
145
  * To see how to use glob patterns and regex to filter tree nodes, see the `globre project page <https://pypi.org/project/globre/#details>`_.
118
146
 
119
147
  * Include/Exclude patterns do not work at the API level but work on the results returned from the API, for large Gitlab installations this can take a lot of time, if you need to reduce the amound of API calls for such projects use the ``--group-search`` parameter to search only for the top level groups the interest you using the `Gitlab Group Search API <https://docs.gitlab.com/ee/api/groups.html#search-for-group>`_ which allows you to do a partial like query for a Group's path or name
120
148
 
149
+ * **Performance optimization**: For large GitLab instances with many groups and projects, use the ``--api-concurrency`` option to dramatically speed up tree building. This enables parallel API calls (default: 5 concurrent requests) which can provide **4-6x speedup** in real-world scenarios. For example, building a tree with 21 subgroups and 21 projects can be reduced from ~96 seconds (sequential) to ~16-21 seconds (with ``--api-concurrency 5-10``). The ``-c/--concurrency`` option controls parallel git operations (cloning/pulling), while ``--api-concurrency`` controls parallel API calls (fetching groups/projects). Both can be tuned independently based on your needs.
150
+
121
151
  * Cloning vs Pulling: when running Gitlabber consecutively with the same parameters, it will scan the local tree structure; if the project directory exists and is a valid git repository (has .git folder in it) Gitlabber will perform a git pull in the directory, otherwise the project directory will be created and the GitLab project will be cloned into it.
122
152
 
123
153
  * Cloning submodules: use the ``-r`` flag to recurse git submodules, uses the ``--recursive`` for cloning and utilizes `GitPython's smart update method <https://github.com/gitpython-developers/GitPython/blob/20f4a9d49b466a18f1af1fdfb480bc4520a4cdc2/git/objects/submodule/root.py#L67>`_ for updating cloned repositories
@@ -127,7 +157,7 @@ Usage
127
157
  .. code-block:: bash
128
158
 
129
159
  usage: gitlabber [-h] [-t token] [-T] [-u url] [--verbose] [-p] [--print-format {json,yaml,tree}] [-n {name,path}] [-m {ssh,http}]
130
- [-a {include,exclude,only}] [-i csv] [-x csv] [-r] [-F] [-d] [-s] [-g term] [-U] [-o options] [--version]
160
+ [-a {include,exclude,only}] [-i csv] [-x csv] [-c N] [--api-concurrency N] [-r] [-F] [-d] [-s] [-g term] [-U] [-o options] [--version]
131
161
  [dest]
132
162
 
133
163
  Gitlabber - clones or pulls entire groups/projects tree from gitlab
@@ -149,12 +179,16 @@ Usage
149
179
  the folder naming strategy for projects from the gitlab API attributes (default: "name")
150
180
  -m {ssh,http}, --method {ssh,http}
151
181
  the git transport method to use for cloning (default: "ssh")
182
+ --fail-fast exit immediately when encountering discovery errors
152
183
  -a {include,exclude,only}, --archived {include,exclude,only}
153
184
  include archived projects and groups in the results (default: "include")
154
185
  -i csv, --include csv
155
186
  comma delimited list of glob patterns of paths to projects or groups to clone/pull
156
187
  -x csv, --exclude csv
157
188
  comma delimited list of glob patterns of paths to projects or groups to exclude from clone/pull
189
+ -c N, --concurrency N
190
+ number of concurrent git operations (default: 1)
191
+ --api-concurrency N number of concurrent API calls for tree building (default: 5)
158
192
  -r, --recursive clone/pull git submodules recursively
159
193
  -F, --use-fetch clone/fetch git repository (mirrored repositories)
160
194
  -s, --include-shared include shared projects in the results
@@ -191,16 +225,48 @@ Usage
191
225
  perform a shallow clone of the git repositories
192
226
  gitlabber -o "\-\-depth=1," .
193
227
 
228
+ speed up tree building for large GitLab instances with parallel API calls (4-6x faster)
229
+ # Real-world example: 96s → 16-21s for instances with many subgroups/projects
230
+ gitlabber --api-concurrency 10 -t <token> -u <url> .
231
+
232
+ use both API and git concurrency for maximum performance
233
+ # API concurrency speeds up tree discovery, git concurrency speeds up cloning
234
+ gitlabber --api-concurrency 5 -c 10 -t <token> -u <url> .
235
+
236
+ **Performance Results:**
237
+ * Sequential (``--api-concurrency 1``): ~96 seconds
238
+ * With ``--api-concurrency 5``: ~21 seconds (**4.6x speedup**)
239
+ * With ``--api-concurrency 10``: ~16 seconds (**6x speedup**)
194
240
 
241
+ *Note: Actual speedup depends on your GitLab instance structure (number of groups, subgroups, and projects). Instances with many nested subgroups benefit most from higher concurrency values.*
242
+
243
+ Common Use Cases
244
+ ----------------
245
+
246
+ Clone Specific Groups
247
+ ---------------------
248
+
249
+ .. code-block:: bash
250
+
251
+ # Clone only projects from a specific group
252
+ gitlabber -i '/MyGroup/**' .
253
+
254
+ Exclude Archived Projects
255
+ -------------------------
256
+
257
+ .. code-block:: bash
258
+
259
+ # Clone all non-archived projects
260
+ gitlabber -a exclude .
195
261
 
196
262
  Debugging
197
263
  ---------
198
264
  * You can use the ``--verbose`` flag to print Gitlabber debug messages
199
265
  * For more verbose GitLab messages, you can get the `GitPython <https://gitpython.readthedocs.io/en/stable>`_ module to print more debug messages by setting the environment variable:
200
266
 
201
- .. code-block:: bash
267
+ .. code-block:: bash
202
268
 
203
- export GIT_PYTHON_TRACE='full'
269
+ export GIT_PYTHON_TRACE='full'
204
270
 
205
271
  Troubleshooting
206
272
  ---------------
@@ -213,6 +279,26 @@ Known Limitations
213
279
  * Folder naming strategy: Consecutively running Gitlabber with different values for the ``-n`` parameter will produce undesirable results. Use the same value as previous runs, or simply don't change it from the default (project name).
214
280
  * If you're going to clone a large number of projects, observe rate limits `for gitlab.com <https://docs.gitlab.com/ee/user/gitlab_com/index.html#gitlabcom-specific-rate-limits/>`_, and `for on-premise installations <https://docs.gitlab.com/ee/security/rate_limits.html>`_.
215
281
 
282
+ Branch Synchronization Issues
283
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
284
+
285
+ Gitlabber's default pull behavior (``git pull -v -- origin``) may fail when you have a local branch checked out that no longer exists on the remote repository. This commonly occurs after:
286
+
287
+ - Merging a merge request where the remote branch is automatically deleted
288
+ - Manual deletion of remote branches
289
+
290
+ **Error Example:**
291
+
292
+ .. code-block:: text
293
+
294
+ git.exc.GitCommandError: Cmd('git') failed due to: exit code(1)
295
+ cmdline: git pull -v -- origin
296
+
297
+ **Workaround:** Use the ``--use-fetch`` flag instead of the default pull behavior:
298
+
299
+ .. code-block:: bash
300
+
301
+ gitlabber --use-fetch -t <your-token> -u <gitlab-url> .
216
302
 
217
303
  Links
218
304
  -----
@@ -32,54 +32,84 @@ Gitlabber clones or pulls all projects under a subset of groups / subgroups by b
32
32
  Installation
33
33
  ------------
34
34
 
35
- * You can install Gitlabber from `PyPi <https://pypi.org/project/gitlabber>`_:
35
+ System Requirements
36
+ -------------------
37
+ * Python 3.11 or higher
38
+ * Git 2.0 or higher
39
+ * Network access to GitLab instance
36
40
 
37
- .. code-block:: bash
41
+ Installation Methods
42
+ ~~~~~~~~~~~~~~~~~~~~
43
+ * PyPI (recommended):
38
44
 
39
- pip install gitlabber
45
+ .. code-block:: bash
46
+
47
+ pip install gitlabber
48
+
49
+ * From source:
50
+
51
+ .. code-block:: bash
52
+
53
+ git clone https://github.com/ezbz/gitlabber.git
54
+ cd gitlabber
55
+ pip install -e .
40
56
 
41
57
  * You'll need to create an `access token <https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html>`_ from GitLab with API scopes `read_repository`
42
58
  and ``read_api`` (or ``api``, for GitLab versions <12.0)
43
59
 
60
+ Quick Start
61
+ -----------
62
+
63
+ .. code-block:: bash
64
+
65
+ # Install gitlabber
66
+ pip install gitlabber
67
+
68
+ # Clone all your GitLab projects
69
+ gitlabber -t <your-token> -u <gitlab-url> .
70
+
44
71
  Usage
45
72
  -----
46
73
 
47
74
  * Arguments can be provided via the CLI arguments directly or via environment variables:
48
75
 
49
- +---------------+---------------+---------------------------+
50
- | Argument | Flag | Environment Variable |
51
- +===============+===============+===========================+
52
- | token | -t | `GITLAB_TOKEN` |
53
- +---------------+---------------+---------------------------+
54
- | url | -u | `GITLAB_URL` |
55
- +---------------+---------------+---------------------------+
56
- | method | -m | `GITLABBER_CLONE_METHOD` |
57
- +---------------+---------------+---------------------------+
58
- | naming | -n | `GITLABBER_FOLDER_NAMING` |
59
- +---------------+---------------+---------------------------+
60
- | include | -i | `GITLABBER_INCLUDE` |
61
- +---------------+---------------+---------------------------+
62
- | exclude | -x | `GITLABBER_EXCLUDE` |
63
- +---------------+---------------+---------------------------+
76
+ +------------------+------------------+---------------------------+
77
+ | Argument | Flag | Environment Variable |
78
+ +==================+==================+===========================+
79
+ | token | -t | `GITLAB_TOKEN` |
80
+ +------------------+------------------+---------------------------+
81
+ | url | -u | `GITLAB_URL` |
82
+ +------------------+------------------+---------------------------+
83
+ | method | -m | `GITLABBER_CLONE_METHOD` |
84
+ +------------------+------------------+---------------------------+
85
+ | naming | -n | `GITLABBER_FOLDER_NAMING` |
86
+ +------------------+------------------+---------------------------+
87
+ | include | -i | `GITLABBER_INCLUDE` |
88
+ +------------------+------------------+---------------------------+
89
+ | exclude | -x | `GITLABBER_EXCLUDE` |
90
+ +------------------+------------------+---------------------------+
64
91
 
65
92
  * To view the tree run the command with your includes/excludes and the ``-p`` flag. It will print your tree like so:
66
93
 
67
- .. code-block:: bash
94
+ .. code-block:: bash
95
+
96
+ root [http://gitlab.my.com]
97
+ ├── group1 [/group1]
98
+ │ └── subgroup1 [/group1/subgroup1]
99
+ │ └── project1 [/group1/subgroup1/project1]
100
+ └── group2 [/group2]
101
+ ├── subgroup1 [/group2/subgroup1]
102
+ │ └── project2 [/group2/subgroup1/project2]
103
+ ├── subgroup2 [/group2/subgroup2]
104
+ └── subgroup3 [/group2/subgroup3]
68
105
 
69
- root [http://gitlab.my.com]
70
- ├── group1 [/group1]
71
- │ └── subgroup1 [/group1/subgroup1]
72
- │ └── project1 [/group1/subgroup1/project1]
73
- └── group2 [/group2]
74
- ├── subgroup1 [/group2/subgroup1]
75
- │ └── project2 [/group2/subgroup1/project2]
76
- ├── subgroup2 [/group2/subgroup2]
77
- └── subgroup3 [/group2/subgroup3]
78
106
 
79
107
  * To see how to use glob patterns and regex to filter tree nodes, see the `globre project page <https://pypi.org/project/globre/#details>`_.
80
108
 
81
109
  * Include/Exclude patterns do not work at the API level but work on the results returned from the API, for large Gitlab installations this can take a lot of time, if you need to reduce the amound of API calls for such projects use the ``--group-search`` parameter to search only for the top level groups the interest you using the `Gitlab Group Search API <https://docs.gitlab.com/ee/api/groups.html#search-for-group>`_ which allows you to do a partial like query for a Group's path or name
82
110
 
111
+ * **Performance optimization**: For large GitLab instances with many groups and projects, use the ``--api-concurrency`` option to dramatically speed up tree building. This enables parallel API calls (default: 5 concurrent requests) which can provide **4-6x speedup** in real-world scenarios. For example, building a tree with 21 subgroups and 21 projects can be reduced from ~96 seconds (sequential) to ~16-21 seconds (with ``--api-concurrency 5-10``). The ``-c/--concurrency`` option controls parallel git operations (cloning/pulling), while ``--api-concurrency`` controls parallel API calls (fetching groups/projects). Both can be tuned independently based on your needs.
112
+
83
113
  * Cloning vs Pulling: when running Gitlabber consecutively with the same parameters, it will scan the local tree structure; if the project directory exists and is a valid git repository (has .git folder in it) Gitlabber will perform a git pull in the directory, otherwise the project directory will be created and the GitLab project will be cloned into it.
84
114
 
85
115
  * Cloning submodules: use the ``-r`` flag to recurse git submodules, uses the ``--recursive`` for cloning and utilizes `GitPython's smart update method <https://github.com/gitpython-developers/GitPython/blob/20f4a9d49b466a18f1af1fdfb480bc4520a4cdc2/git/objects/submodule/root.py#L67>`_ for updating cloned repositories
@@ -89,7 +119,7 @@ Usage
89
119
  .. code-block:: bash
90
120
 
91
121
  usage: gitlabber [-h] [-t token] [-T] [-u url] [--verbose] [-p] [--print-format {json,yaml,tree}] [-n {name,path}] [-m {ssh,http}]
92
- [-a {include,exclude,only}] [-i csv] [-x csv] [-r] [-F] [-d] [-s] [-g term] [-U] [-o options] [--version]
122
+ [-a {include,exclude,only}] [-i csv] [-x csv] [-c N] [--api-concurrency N] [-r] [-F] [-d] [-s] [-g term] [-U] [-o options] [--version]
93
123
  [dest]
94
124
 
95
125
  Gitlabber - clones or pulls entire groups/projects tree from gitlab
@@ -111,12 +141,16 @@ Usage
111
141
  the folder naming strategy for projects from the gitlab API attributes (default: "name")
112
142
  -m {ssh,http}, --method {ssh,http}
113
143
  the git transport method to use for cloning (default: "ssh")
144
+ --fail-fast exit immediately when encountering discovery errors
114
145
  -a {include,exclude,only}, --archived {include,exclude,only}
115
146
  include archived projects and groups in the results (default: "include")
116
147
  -i csv, --include csv
117
148
  comma delimited list of glob patterns of paths to projects or groups to clone/pull
118
149
  -x csv, --exclude csv
119
150
  comma delimited list of glob patterns of paths to projects or groups to exclude from clone/pull
151
+ -c N, --concurrency N
152
+ number of concurrent git operations (default: 1)
153
+ --api-concurrency N number of concurrent API calls for tree building (default: 5)
120
154
  -r, --recursive clone/pull git submodules recursively
121
155
  -F, --use-fetch clone/fetch git repository (mirrored repositories)
122
156
  -s, --include-shared include shared projects in the results
@@ -153,16 +187,48 @@ Usage
153
187
  perform a shallow clone of the git repositories
154
188
  gitlabber -o "\-\-depth=1," .
155
189
 
190
+ speed up tree building for large GitLab instances with parallel API calls (4-6x faster)
191
+ # Real-world example: 96s → 16-21s for instances with many subgroups/projects
192
+ gitlabber --api-concurrency 10 -t <token> -u <url> .
193
+
194
+ use both API and git concurrency for maximum performance
195
+ # API concurrency speeds up tree discovery, git concurrency speeds up cloning
196
+ gitlabber --api-concurrency 5 -c 10 -t <token> -u <url> .
197
+
198
+ **Performance Results:**
199
+ * Sequential (``--api-concurrency 1``): ~96 seconds
200
+ * With ``--api-concurrency 5``: ~21 seconds (**4.6x speedup**)
201
+ * With ``--api-concurrency 10``: ~16 seconds (**6x speedup**)
156
202
 
203
+ *Note: Actual speedup depends on your GitLab instance structure (number of groups, subgroups, and projects). Instances with many nested subgroups benefit most from higher concurrency values.*
204
+
205
+ Common Use Cases
206
+ ----------------
207
+
208
+ Clone Specific Groups
209
+ ---------------------
210
+
211
+ .. code-block:: bash
212
+
213
+ # Clone only projects from a specific group
214
+ gitlabber -i '/MyGroup/**' .
215
+
216
+ Exclude Archived Projects
217
+ -------------------------
218
+
219
+ .. code-block:: bash
220
+
221
+ # Clone all non-archived projects
222
+ gitlabber -a exclude .
157
223
 
158
224
  Debugging
159
225
  ---------
160
226
  * You can use the ``--verbose`` flag to print Gitlabber debug messages
161
227
  * For more verbose GitLab messages, you can get the `GitPython <https://gitpython.readthedocs.io/en/stable>`_ module to print more debug messages by setting the environment variable:
162
228
 
163
- .. code-block:: bash
229
+ .. code-block:: bash
164
230
 
165
- export GIT_PYTHON_TRACE='full'
231
+ export GIT_PYTHON_TRACE='full'
166
232
 
167
233
  Troubleshooting
168
234
  ---------------
@@ -175,7 +241,27 @@ Known Limitations
175
241
  * Folder naming strategy: Consecutively running Gitlabber with different values for the ``-n`` parameter will produce undesirable results. Use the same value as previous runs, or simply don't change it from the default (project name).
176
242
  * If you're going to clone a large number of projects, observe rate limits `for gitlab.com <https://docs.gitlab.com/ee/user/gitlab_com/index.html#gitlabcom-specific-rate-limits/>`_, and `for on-premise installations <https://docs.gitlab.com/ee/security/rate_limits.html>`_.
177
243
 
244
+ Branch Synchronization Issues
245
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
246
+
247
+ Gitlabber's default pull behavior (``git pull -v -- origin``) may fail when you have a local branch checked out that no longer exists on the remote repository. This commonly occurs after:
248
+
249
+ - Merging a merge request where the remote branch is automatically deleted
250
+ - Manual deletion of remote branches
251
+
252
+ **Error Example:**
253
+
254
+ .. code-block:: text
255
+
256
+ git.exc.GitCommandError: Cmd('git') failed due to: exit code(1)
257
+ cmdline: git pull -v -- origin
258
+
259
+ **Workaround:** Use the ``--use-fetch`` flag instead of the default pull behavior:
260
+
261
+ .. code-block:: bash
262
+
263
+ gitlabber --use-fetch -t <your-token> -u <gitlab-url> .
178
264
 
179
265
  Links
180
266
  -----
181
- * Gitlabber `tutorial <https://medium.com/@natskvi/clone-all-your-gitlab-groups-repos-before-starting-to-code-dd559ec5c8d6>`_ by nsk
267
+ * Gitlabber `tutorial <https://medium.com/@natskvi/clone-all-your-gitlab-groups-repos-before-starting-to-code-dd559ec5c8d6>`_ by nsk
@@ -0,0 +1,8 @@
1
+ """Gitlabber - A tool for cloning GitLab project hierarchies.
2
+
3
+ Gitlabber allows you to clone entire GitLab group/subgroup hierarchies
4
+ while maintaining the directory structure. It supports filtering, progress
5
+ tracking, and various configuration options.
6
+ """
7
+
8
+ __version__ = '2.0.0'
@@ -0,0 +1,8 @@
1
+ """Entry point for running gitlabber as a module.
2
+
3
+ This module allows gitlabber to be executed as:
4
+ python -m gitlabber
5
+ """
6
+
7
+ from .cli import main
8
+ main()
@@ -0,0 +1,40 @@
1
+ """Enumeration for handling archived GitLab projects and groups.
2
+
3
+ This module provides the ArchivedResults enum which controls how archived
4
+ projects and groups are handled during tree building and filtering.
5
+ """
6
+
7
+ from typing import Optional
8
+ import enum
9
+
10
+
11
+ class ArchivedResults(enum.Enum):
12
+ """Enumeration for handling archived results in GitLab projects.
13
+
14
+ Attributes:
15
+ INCLUDE: Include both archived and non-archived projects
16
+ EXCLUDE: Exclude archived projects
17
+ ONLY: Only include archived projects
18
+ """
19
+ INCLUDE = (1, None)
20
+ EXCLUDE = (2, False)
21
+ ONLY = (3, True)
22
+
23
+ def __init__(self, int_value: int, api_value: Optional[bool]) -> None:
24
+ """Initialize the enum value.
25
+
26
+ Args:
27
+ int_value: Integer value for internal use
28
+ api_value: Boolean value for GitLab API, None for INCLUDE
29
+ """
30
+ self.int_value = int_value
31
+ self.api_value = api_value
32
+
33
+ def __str__(self) -> str:
34
+ """Return the lowercase name of the enum value."""
35
+ return self.name.lower()
36
+
37
+ def __repr__(self) -> str:
38
+ """Return the string representation of the enum value."""
39
+ return str(self)
40
+
@@ -0,0 +1,62 @@
1
+ """Authentication providers for GitLab API access.
2
+
3
+ This module defines the authentication interface and implementations
4
+ for authenticating with GitLab instances. It supports token-based
5
+ authentication and provides a no-op provider for testing.
6
+ """
7
+
8
+ from abc import ABC, abstractmethod
9
+ from typing import Optional
10
+ from gitlab import Gitlab
11
+ from gitlab.exceptions import GitlabAuthenticationError
12
+
13
+ class AuthProvider(ABC):
14
+ """Interface for GitLab authentication providers."""
15
+
16
+ @abstractmethod
17
+ def authenticate(self, gitlab_client: Gitlab) -> None:
18
+ """Authenticate the GitLab client.
19
+
20
+ Args:
21
+ gitlab_client: The GitLab client to authenticate
22
+
23
+ Raises:
24
+ GitlabAuthenticationError: If authentication fails
25
+ """
26
+ pass
27
+
28
+ class TokenAuthProvider(AuthProvider):
29
+ """Authentication provider using a personal access token."""
30
+
31
+ def __init__(self, token: str) -> None:
32
+ """Initialize with a personal access token.
33
+
34
+ Args:
35
+ token: GitLab personal access token
36
+ """
37
+ self.token = token
38
+
39
+ def authenticate(self, gitlab_client: Gitlab) -> None:
40
+ """Authenticate using the personal access token.
41
+
42
+ Args:
43
+ gitlab_client: The GitLab client to authenticate
44
+
45
+ Raises:
46
+ GitlabAuthenticationError: If authentication fails
47
+ """
48
+ gitlab_client.auth()
49
+
50
+ class NoAuthProvider(AuthProvider):
51
+ """Authentication provider that performs no authentication.
52
+
53
+ This is useful for testing or when authentication is handled externally.
54
+ """
55
+
56
+ def authenticate(self, gitlab_client: Gitlab) -> None:
57
+ """Perform no authentication.
58
+
59
+ Args:
60
+ gitlab_client: The GitLab client (not used)
61
+ """
62
+ pass