kospex 0.0.14__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kospex-0.0.14/LICENSE +21 -0
- kospex-0.0.14/PKG-INFO +61 -0
- kospex-0.0.14/README.md +143 -0
- kospex-0.0.14/README.short.md +17 -0
- kospex-0.0.14/pyproject.toml +43 -0
- kospex-0.0.14/setup.cfg +4 -0
- kospex-0.0.14/src/kgit.py +329 -0
- kospex-0.0.14/src/kospex.egg-info/PKG-INFO +61 -0
- kospex-0.0.14/src/kospex.egg-info/SOURCES.txt +59 -0
- kospex-0.0.14/src/kospex.egg-info/dependency_links.txt +1 -0
- kospex-0.0.14/src/kospex.egg-info/entry_points.txt +7 -0
- kospex-0.0.14/src/kospex.egg-info/requires.txt +9 -0
- kospex-0.0.14/src/kospex.egg-info/top_level.txt +18 -0
- kospex-0.0.14/src/kospex.py +949 -0
- kospex-0.0.14/src/kospex_bitbucket.py +97 -0
- kospex-0.0.14/src/kospex_core.py +1125 -0
- kospex-0.0.14/src/kospex_dependencies.py +887 -0
- kospex-0.0.14/src/kospex_git.py +346 -0
- kospex-0.0.14/src/kospex_github.py +208 -0
- kospex-0.0.14/src/kospex_mergestat.py +154 -0
- kospex-0.0.14/src/kospex_query.py +1517 -0
- kospex-0.0.14/src/kospex_schema.py +360 -0
- kospex-0.0.14/src/kospex_utils.py +1270 -0
- kospex-0.0.14/src/kospex_web.py +60 -0
- kospex-0.0.14/src/kreaper.py +80 -0
- kospex-0.0.14/src/krunner.py +344 -0
- kospex-0.0.14/src/krunner_utils.py +257 -0
- kospex-0.0.14/src/kwatch.py +29 -0
- kospex-0.0.14/src/kweb.py +734 -0
- kospex-0.0.14/src/templates/404.html +30 -0
- kospex-0.0.14/src/templates/_bootstrap_html_head.html +9 -0
- kospex-0.0.14/src/templates/_datatable_scripts.html +2 -0
- kospex-0.0.14/src/templates/_footer_scripts.html +5 -0
- kospex-0.0.14/src/templates/_id_header_macro.html +6 -0
- kospex-0.0.14/src/templates/_radar_macro.html +46 -0
- kospex-0.0.14/src/templates/_single_radar.html +41 -0
- kospex-0.0.14/src/templates/bubble.html +615 -0
- kospex-0.0.14/src/templates/commits.html +46 -0
- kospex-0.0.14/src/templates/developer_view.html +98 -0
- kospex-0.0.14/src/templates/developers.html +93 -0
- kospex-0.0.14/src/templates/graph.html +651 -0
- kospex-0.0.14/src/templates/graph2.html +696 -0
- kospex-0.0.14/src/templates/header.html +31 -0
- kospex-0.0.14/src/templates/hotspots.html +58 -0
- kospex-0.0.14/src/templates/landscape.html +88 -0
- kospex-0.0.14/src/templates/meta-author-domains.html +44 -0
- kospex-0.0.14/src/templates/metadata.html +107 -0
- kospex-0.0.14/src/templates/observations.html +47 -0
- kospex-0.0.14/src/templates/observations_repo.html +48 -0
- kospex-0.0.14/src/templates/observations_repo_key.html +49 -0
- kospex-0.0.14/src/templates/orgs.html +67 -0
- kospex-0.0.14/src/templates/osi.html +19 -0
- kospex-0.0.14/src/templates/repo_files.html +53 -0
- kospex-0.0.14/src/templates/repo_view.html +154 -0
- kospex-0.0.14/src/templates/repos.html +152 -0
- kospex-0.0.14/src/templates/servers.html +55 -0
- kospex-0.0.14/src/templates/summary.html +191 -0
- kospex-0.0.14/src/templates/tech-change.html +78 -0
- kospex-0.0.14/src/templates/treemap.html +641 -0
- kospex-0.0.14/tests/test_kgit.py +37 -0
- kospex-0.0.14/tests/test_kospex.py +90 -0
kospex-0.0.14/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023 Peter Freiberg
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
kospex-0.0.14/PKG-INFO
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
|
+
Name: kospex
|
|
3
|
+
Version: 0.0.14
|
|
4
|
+
Summary: The kospex CLI tool.
|
|
5
|
+
Author-email: Peter Freiberg <peter.freiberg@gmail.com>
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2023 Peter Freiberg
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
|
|
28
|
+
Project-URL: Homepage, https://kospex.io
|
|
29
|
+
Project-URL: Issues, https://github.com/kospex/kospex/issues
|
|
30
|
+
Project-URL: Repository, https://github.com/kospex/kospex
|
|
31
|
+
Keywords: analytics,git,code analytics,predictive maintenance
|
|
32
|
+
Requires-Python: >=3.11
|
|
33
|
+
Description-Content-Type: text/markdown
|
|
34
|
+
License-File: LICENSE
|
|
35
|
+
Requires-Dist: Click
|
|
36
|
+
Requires-Dist: flask
|
|
37
|
+
Requires-Dist: prettytable
|
|
38
|
+
Requires-Dist: dateutils
|
|
39
|
+
Requires-Dist: panopticas
|
|
40
|
+
Requires-Dist: python-dotenv
|
|
41
|
+
Requires-Dist: sqlite_utils
|
|
42
|
+
Requires-Dist: requests
|
|
43
|
+
Requires-Dist: PyYAML
|
|
44
|
+
|
|
45
|
+
# kospex
|
|
46
|
+
|
|
47
|
+
Kospex is a CLI which aims to _"look at the guts of your code"_ to help gain insights into your developers and technology landscape.
|
|
48
|
+
It uses database structure from the excellent [Mergestat lite](https://github.com/mergestat/mergestat-lite) to model data from git repositories.
|
|
49
|
+
|
|
50
|
+
For details on changes, see the [changelog](https://github.com/kospex/kospex/blob/main/CHANGELOG.md)
|
|
51
|
+
|
|
52
|
+
## Installation, setup and usage
|
|
53
|
+
|
|
54
|
+
See the official [installation documentation](https://kospex.io/getting-started)
|
|
55
|
+
|
|
56
|
+
## What is a kospex?
|
|
57
|
+
|
|
58
|
+
We're aiming to [k]now your c[o]de by in[spe]cting the haruspe[x].
|
|
59
|
+
From Wikipedia, _The Latin terms haruspex and haruspicina are from an archaic word, hīra = "entrails, intestines"_
|
|
60
|
+
|
|
61
|
+
So we're going to help look at the "guts of your code" to gain an understanding of the applications, technology landscape (sprawl?) and developers.
|
kospex-0.0.14/README.md
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
# kospex
|
|
2
|
+
|
|
3
|
+
Kospex is a CLI which aims to _"look at the guts of your code"_ to help gain insights into your developers and technology landscape.
|
|
4
|
+
It uses database structure from the excellent [Mergestat lite](https://github.com/mergestat/mergestat-lite) to model data from git repositories.
|
|
5
|
+
|
|
6
|
+
## Step 1: Installation, setup and usage
|
|
7
|
+
|
|
8
|
+
kospex is currently a python module with commands. It works by analysing cloned repositories on the filesystem.
|
|
9
|
+
|
|
10
|
+
**Optional but strongely recommended** - use a python virtual env.
|
|
11
|
+
|
|
12
|
+
Installing using pip:
|
|
13
|
+
|
|
14
|
+
> pip install kospex
|
|
15
|
+
|
|
16
|
+
For complexity and file type analysis, kospex uses the scc binary.
|
|
17
|
+
It is optional, but enables much better file type guessing and provide complexity metrics.
|
|
18
|
+
Follow the instructions for installing [scc](https://github.com/boyter/scc)
|
|
19
|
+
|
|
20
|
+
### Step 2: Initial kospex setup
|
|
21
|
+
|
|
22
|
+
kospex uses a git repositoriy layout for cloning repos to disk.
|
|
23
|
+
|
|
24
|
+
The following structure is used \
|
|
25
|
+
/BASE/GIT_SERVER/ORG/REPO
|
|
26
|
+
|
|
27
|
+
If you are ok to use the ~/code directory for cloned repos, then run:
|
|
28
|
+
> kospex init --default
|
|
29
|
+
|
|
30
|
+
See section "Git code layout for running analysis" below for more details.
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
### Step 3: sync some data and play with some commands
|
|
34
|
+
|
|
35
|
+
For an existing repo on disk:
|
|
36
|
+
> kospex sync [GIT_REPO]
|
|
37
|
+
|
|
38
|
+
You can also use the _kgit_ command to clone and sync a repo you have access to
|
|
39
|
+
|
|
40
|
+
> kgit clone https://github.com/mergestat/mergestat-lite
|
|
41
|
+
|
|
42
|
+
The above command will clone into the KOSPEX_CODE/GIT_SERVER/ORG/REPO structure
|
|
43
|
+
|
|
44
|
+
Some commands to try:
|
|
45
|
+
|
|
46
|
+
> kospex developers -repo [GIT_REPO]
|
|
47
|
+
>
|
|
48
|
+
> kospex tech-landscape -metadata
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
## Git code layout for running analysis
|
|
52
|
+
|
|
53
|
+
One option, if you're inspecting code on your own laptop (or virtual machine), is to use use your home directory.
|
|
54
|
+
|
|
55
|
+
~/kospex/ \
|
|
56
|
+
We'll place config files and the kospex DB (Sqlite3) in here for sync'ed data \
|
|
57
|
+
~/code/ \
|
|
58
|
+
This should be your GIT_DATA_DIRECTORY with a structure like \
|
|
59
|
+
GIT_SERVER/ORG/REPO \
|
|
60
|
+
\
|
|
61
|
+
For example, in your ~/code it might look like: \
|
|
62
|
+
github.com/kospex/kospex \
|
|
63
|
+
github.com/mergestat/mergestat-lite
|
|
64
|
+
|
|
65
|
+
This way we have a nice deterministic way of separating different orgs, potentially different instances (e.g. you have an on premise bitbucket and use GitHub.com) as well.
|
|
66
|
+
|
|
67
|
+
## Key Use Cases and features
|
|
68
|
+
|
|
69
|
+
- Identify technology landscape
|
|
70
|
+
- Identify active developers (e.g. who's had their code committer in the last 90 days)
|
|
71
|
+
- Identify key person or offboarding risk
|
|
72
|
+
- Identify potential complexity challenges (or conceptual integrity concerns)
|
|
73
|
+
- Aggregate repo metadata into a single database for easier and faster querying
|
|
74
|
+
|
|
75
|
+
## Queries to try
|
|
76
|
+
|
|
77
|
+
### sync a repo
|
|
78
|
+
|
|
79
|
+
> kospex sync PATH/TO/GIT_REPO
|
|
80
|
+
|
|
81
|
+
_Most functions require the data to be synced._
|
|
82
|
+
|
|
83
|
+
### Show recent developers who've committed in X days
|
|
84
|
+
|
|
85
|
+
List the active developers (90 days) in the given repo (sync required)
|
|
86
|
+
> kospex developers -repo PATH/TO/REPO
|
|
87
|
+
|
|
88
|
+
List the developers in the given repo_id (sync'ed data in the kospex DB)
|
|
89
|
+
> kospex developers -repo_id=github.com˜ORG˜REPO
|
|
90
|
+
|
|
91
|
+
use -days NUM for seen in the last # of days (e.g. 90 or 365)
|
|
92
|
+
|
|
93
|
+
### Identify technology landscape
|
|
94
|
+
|
|
95
|
+
List the overall tech stack, based on file extension, for all sync'ed repos
|
|
96
|
+
> kospex tech-landscape
|
|
97
|
+
|
|
98
|
+
List the overall tech stack for all sync'ed repos (using scc)
|
|
99
|
+
> kospex tech-landscape -metadata
|
|
100
|
+
|
|
101
|
+
List the overall tech stack for a repo (using scc)
|
|
102
|
+
> kospex tech-landscape -repo PATH/TO/REPO
|
|
103
|
+
|
|
104
|
+
List the tech stack in the given repo_id (sync'ed data in the kospex DB)
|
|
105
|
+
> kospex tech-landscape -repo_id=github.com˜ORG˜REPO
|
|
106
|
+
|
|
107
|
+
## Design principles
|
|
108
|
+
|
|
109
|
+
- Precompute data where possible and useful
|
|
110
|
+
- Flatten tables, data warehouse style, to enabled easier querying and slicing by git server, owner and repo
|
|
111
|
+
- Be as agnostic to the git provider (i.e. GitHub, BitBucket, GitLab) for base use cases
|
|
112
|
+
- Be mindful that "there is no perfect", only indicators
|
|
113
|
+
- Separate cloning and pull updates from the analysis
|
|
114
|
+
|
|
115
|
+
## Roadmap
|
|
116
|
+
|
|
117
|
+
- Build out automated functional and regressions testing (Currently manual)
|
|
118
|
+
- Build the ability to identify key person or offboarding risk
|
|
119
|
+
- Improve use case documentation
|
|
120
|
+
- Provide a mechanism to better map author_emails to users
|
|
121
|
+
|
|
122
|
+
## Data extractions and assumptions
|
|
123
|
+
|
|
124
|
+
Most tables have a column, _repo_id, in the format of GIT_SERVER˜OWNER˜REPO
|
|
125
|
+
So for the repository https://github.com/kospex/kospex the _repo_id would be _github.com˜kospex˜kospex_
|
|
126
|
+
|
|
127
|
+
Most queries use author_email from git to mean a "developer".
|
|
128
|
+
|
|
129
|
+
## How do I develop and improve kospex?
|
|
130
|
+
|
|
131
|
+
The easiest way to do development on the kospex repo is to clone kospex and run in "dev mode" using the following commands:
|
|
132
|
+
> git clone https://github.com/kospex/kospex
|
|
133
|
+
> cd kospex
|
|
134
|
+
> pip install -e .
|
|
135
|
+
|
|
136
|
+
## What is a kospex?
|
|
137
|
+
|
|
138
|
+
We're aiming to [k]now your c[o]de by in[spe]cting the haruspe[x].
|
|
139
|
+
From Wikipedia, _The Latin terms haruspex and haruspicina are from an archaic word, hīra = "entrails, intestines"_
|
|
140
|
+
|
|
141
|
+
So we're going to help look at the "guts of your code" to gain an understanding of the applications, technology landscape (sprawl?) and developers.
|
|
142
|
+
|
|
143
|
+
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# kospex
|
|
2
|
+
|
|
3
|
+
Kospex is a CLI which aims to _"look at the guts of your code"_ to help gain insights into your developers and technology landscape.
|
|
4
|
+
It uses database structure from the excellent [Mergestat lite](https://github.com/mergestat/mergestat-lite) to model data from git repositories.
|
|
5
|
+
|
|
6
|
+
For details on changes, see the [changelog](https://github.com/kospex/kospex/blob/main/CHANGELOG.md)
|
|
7
|
+
|
|
8
|
+
## Installation, setup and usage
|
|
9
|
+
|
|
10
|
+
See the official [installation documentation](https://kospex.io/getting-started)
|
|
11
|
+
|
|
12
|
+
## What is a kospex?
|
|
13
|
+
|
|
14
|
+
We're aiming to [k]now your c[o]de by in[spe]cting the haruspe[x].
|
|
15
|
+
From Wikipedia, _The Latin terms haruspex and haruspicina are from an archaic word, hīra = "entrails, intestines"_
|
|
16
|
+
|
|
17
|
+
So we're going to help look at the "guts of your code" to gain an understanding of the applications, technology landscape (sprawl?) and developers.
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=71.1.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "kospex"
|
|
7
|
+
version = "0.0.14"
|
|
8
|
+
description = "The kospex CLI tool."
|
|
9
|
+
authors = [{ name = "Peter Freiberg", email = "peter.freiberg@gmail.com" }]
|
|
10
|
+
|
|
11
|
+
dependencies = [
|
|
12
|
+
"Click",
|
|
13
|
+
"flask",
|
|
14
|
+
"prettytable",
|
|
15
|
+
"dateutils",
|
|
16
|
+
"panopticas",
|
|
17
|
+
"python-dotenv",
|
|
18
|
+
"sqlite_utils",
|
|
19
|
+
"requests",
|
|
20
|
+
"PyYAML",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
requires-python = ">=3.11"
|
|
24
|
+
license = { file = "LICENSE" }
|
|
25
|
+
readme = "README.short.md"
|
|
26
|
+
keywords = ["analytics", "git", "code analytics", "predictive maintenance"]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
[tool.setuptools.package-data]
|
|
30
|
+
templates = ["*.html"]
|
|
31
|
+
|
|
32
|
+
[project.scripts]
|
|
33
|
+
kgit = "kgit:cli"
|
|
34
|
+
kospex = "kospex:cli"
|
|
35
|
+
krunner = "krunner:cli"
|
|
36
|
+
kreaper = "kreaper:cli"
|
|
37
|
+
kweb = "kweb:kweb"
|
|
38
|
+
kwatch = "kwatch:cli"
|
|
39
|
+
|
|
40
|
+
[project.urls]
|
|
41
|
+
Homepage = "https://kospex.io"
|
|
42
|
+
Issues = "https://github.com/kospex/kospex/issues"
|
|
43
|
+
Repository = "https://github.com/kospex/kospex"
|
kospex-0.0.14/setup.cfg
ADDED
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""This is the kospex git CLI helper tool."""
|
|
3
|
+
import time
|
|
4
|
+
import os
|
|
5
|
+
import json
|
|
6
|
+
import subprocess
|
|
7
|
+
import click
|
|
8
|
+
from prettytable import PrettyTable
|
|
9
|
+
from kospex_core import GitRepo, Kospex
|
|
10
|
+
import kospex_utils as KospexUtils
|
|
11
|
+
from kospex_git import KospexGit
|
|
12
|
+
from kospex_github import KospexGithub
|
|
13
|
+
from kospex_bitbucket import KospexBitbucket
|
|
14
|
+
|
|
15
|
+
KospexUtils.init()
|
|
16
|
+
kgit = KospexGit()
|
|
17
|
+
kospex = Kospex()
|
|
18
|
+
|
|
19
|
+
@click.group()
|
|
20
|
+
def cli():
|
|
21
|
+
"""kgit (Kospex Git) is a utility for doing git things with kospex use cases.
|
|
22
|
+
|
|
23
|
+
For documentation on how commands run `kgit COMMAND --help`.
|
|
24
|
+
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
@cli.command("status")
|
|
28
|
+
@click.argument('repo', type=GitRepo())
|
|
29
|
+
# pylint: disable=unused-argument
|
|
30
|
+
def status(repo):
|
|
31
|
+
"""Date and commit metadata for the given repo."""
|
|
32
|
+
print(f"\nRepo status for path: {repo}")
|
|
33
|
+
st = time.time()
|
|
34
|
+
|
|
35
|
+
stats = KospexUtils.get_git_stats(repo, 90)
|
|
36
|
+
table = PrettyTable()
|
|
37
|
+
table.field_names = ["Subject", "Value"]
|
|
38
|
+
table.align["Subject"] = "l"
|
|
39
|
+
table.align["Value"] = "r"
|
|
40
|
+
for subject, details in stats.items():
|
|
41
|
+
table.add_row([subject, details])
|
|
42
|
+
print(table)
|
|
43
|
+
print("Notes:")
|
|
44
|
+
print("\tdirectory sizes are in KB.")
|
|
45
|
+
print("\tunique authors are the number of unique authors in the last 90 days.")
|
|
46
|
+
et = time.time()
|
|
47
|
+
elapsed_time = et - st
|
|
48
|
+
print('\nExecution time:', elapsed_time, 'seconds', "\n")
|
|
49
|
+
|
|
50
|
+
#@cli.command("mailmap")
|
|
51
|
+
@click.option('-sync', is_flag=True, default=False, help="Sync .mailmap to the database (Default)")
|
|
52
|
+
@click.argument('filename', required=False, type=click.Path(exists=True))
|
|
53
|
+
def mailmap(sync, filename):
|
|
54
|
+
"""
|
|
55
|
+
Parse a .mailmap file and disply
|
|
56
|
+
If the -sync is passed, sync the mailmap file to the kospex database.
|
|
57
|
+
"""
|
|
58
|
+
mmap = KospexUtils.parse_mailmap(filename)
|
|
59
|
+
for entry in mmap:
|
|
60
|
+
print(entry)
|
|
61
|
+
|
|
62
|
+
#@cli.command("branches")
|
|
63
|
+
@click.option('-sync', is_flag=True, default=False, help="Sync branches to the database")
|
|
64
|
+
@click.argument('repo', type=GitRepo())
|
|
65
|
+
def branches(sync, repo):
|
|
66
|
+
"""
|
|
67
|
+
Show the branches for a given repo
|
|
68
|
+
If the -sync is passed, sync the branches to the kospex database.
|
|
69
|
+
"""
|
|
70
|
+
kgit.set_repo(repo)
|
|
71
|
+
os.chdir(repo)
|
|
72
|
+
cmd = ['git', 'branch', '--all']
|
|
73
|
+
result = subprocess.run(cmd, capture_output=True, text=True).stdout.split('\n')
|
|
74
|
+
bob = []
|
|
75
|
+
for i in result:
|
|
76
|
+
if i.lstrip():
|
|
77
|
+
bob.append(i.lstrip()) # remove leading spaces
|
|
78
|
+
print(bob)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@cli.command("clone")
|
|
82
|
+
@click.option('-sync', is_flag=True, default=True, help="Sync the repo to the database (Default)")
|
|
83
|
+
@click.option('-filename', type=click.Path(exists=True), help="File with HTTP git clone URLs")
|
|
84
|
+
@click.argument('repo',type=click.STRING, required=False)
|
|
85
|
+
def clone(sync, filename,repo):
|
|
86
|
+
"""
|
|
87
|
+
Clone the given repo into our KOSPEX_CODE directory.
|
|
88
|
+
Example:
|
|
89
|
+
kgit clone https://github.com/ORG/REPO
|
|
90
|
+
"""
|
|
91
|
+
# We're going to shell out to git to do the clone
|
|
92
|
+
kospex = Kospex()
|
|
93
|
+
|
|
94
|
+
if repo and filename:
|
|
95
|
+
exit("You can't specify both a repo and a filename. Please choose one.")
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
if repo:
|
|
99
|
+
repo_path = kgit.clone_repo(repo)
|
|
100
|
+
if sync and repo_path:
|
|
101
|
+
print("Syncing repo: " + repo_path)
|
|
102
|
+
kospex.sync_repo(repo_path)
|
|
103
|
+
|
|
104
|
+
elif filename:
|
|
105
|
+
with open(filename, "r", encoding='utf-8') as file:
|
|
106
|
+
for line in file:
|
|
107
|
+
repo = line.strip()
|
|
108
|
+
if repo.startswith("#"):
|
|
109
|
+
print(f"\n\nSkipping commented line: {repo}\n\n")
|
|
110
|
+
else:
|
|
111
|
+
repo_path = kgit.clone_repo(repo)
|
|
112
|
+
if not repo_path:
|
|
113
|
+
print(f"\n\nERROR with {repo}\n\n")
|
|
114
|
+
|
|
115
|
+
if sync and repo_path:
|
|
116
|
+
print("Syncing: " + repo)
|
|
117
|
+
kospex = Kospex()
|
|
118
|
+
kospex.sync_repo(repo_path)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@cli.command("pull")
|
|
122
|
+
@click.option('-sync', is_flag=True, default=True, help="Sync the repo to the database (Default)")
|
|
123
|
+
def pull(sync):
|
|
124
|
+
"""
|
|
125
|
+
Check if we're in a git repo, do a git pull,
|
|
126
|
+
and sync to the kospex DB.
|
|
127
|
+
"""
|
|
128
|
+
current = os.getcwd()
|
|
129
|
+
git_base = KospexUtils.find_git_base(current)
|
|
130
|
+
if git_base:
|
|
131
|
+
os.chdir(git_base)
|
|
132
|
+
os.system("git pull")
|
|
133
|
+
if sync:
|
|
134
|
+
print("Syncing to kospex DB ...")
|
|
135
|
+
kospex.sync_repo(git_base)
|
|
136
|
+
os.chdir(current)
|
|
137
|
+
else:
|
|
138
|
+
print(f"{current} does not appear to be in a git repo")
|
|
139
|
+
|
|
140
|
+
@cli.command("github")
|
|
141
|
+
@click.option('-org', type=click.STRING)
|
|
142
|
+
@click.option('-user', type=click.STRING)
|
|
143
|
+
@click.option('-no-auth', is_flag=True, help="Access the Github API unauthenticated.")
|
|
144
|
+
@click.option('-list-repos', is_flag=True, type=click.STRING)
|
|
145
|
+
@click.option('-sync', is_flag=True)
|
|
146
|
+
@click.option('-test-auth', is_flag=True, default=False, help="Test GITHUB_AUTH_TOKEN can authenticate.")
|
|
147
|
+
@click.option('-out-repo-list', type=click.Path(), help="File to write clone URLs to.")
|
|
148
|
+
@click.option('-ssh-clone-url',is_flag=True, help="Write SSH clone urls to file instead of HTTPS")
|
|
149
|
+
def github(org, user, no_auth, list_repos, sync, test_auth, out_repo_list,ssh_clone_url):
|
|
150
|
+
"""
|
|
151
|
+
Interact with the GitHub API.
|
|
152
|
+
|
|
153
|
+
For authenticated access, you must set the GITHUB_AUTH_TOKEN environment variable.
|
|
154
|
+
This is a Personal Access Token (PAT) with the necessary permissions.
|
|
155
|
+
|
|
156
|
+
"""
|
|
157
|
+
|
|
158
|
+
gh = KospexGithub()
|
|
159
|
+
repos = []
|
|
160
|
+
|
|
161
|
+
if test_auth:
|
|
162
|
+
|
|
163
|
+
found = gh.get_env_credentials()
|
|
164
|
+
if found:
|
|
165
|
+
print("Found Github GITHUB_AUTH_TOKEN in the environment.")
|
|
166
|
+
else:
|
|
167
|
+
print("Could not find Github GITHUB_AUTH_TOKEN in the environment.")
|
|
168
|
+
print("Please set GITHUB_AUTH_TOKEN.")
|
|
169
|
+
exit(1)
|
|
170
|
+
|
|
171
|
+
if gh.test_auth():
|
|
172
|
+
print("Authentication successful.")
|
|
173
|
+
else:
|
|
174
|
+
print("Authentication failed. Check your GITHUB_AUTH_TOKEN")
|
|
175
|
+
exit(0)
|
|
176
|
+
|
|
177
|
+
if not org and not user:
|
|
178
|
+
print("You must specify either an organization or a user.")
|
|
179
|
+
exit(1)
|
|
180
|
+
|
|
181
|
+
gh.get_env_credentials()
|
|
182
|
+
|
|
183
|
+
auth = False
|
|
184
|
+
if not no_auth:
|
|
185
|
+
gh.get_env_credentials()
|
|
186
|
+
#gh.set_access_token(os.environ.get("GITHUB_PAT"))
|
|
187
|
+
auth = True
|
|
188
|
+
|
|
189
|
+
owner = org or user
|
|
190
|
+
account_type = gh.get_account_type(owner)
|
|
191
|
+
kospex = Kospex()
|
|
192
|
+
|
|
193
|
+
if not account_type:
|
|
194
|
+
print(f"Could not find account type for {owner}.")
|
|
195
|
+
print(f"Most likely {owner} does not exist or has a typo.")
|
|
196
|
+
exit(1)
|
|
197
|
+
|
|
198
|
+
if org:
|
|
199
|
+
repos = gh.get_org_repos(org)
|
|
200
|
+
elif user:
|
|
201
|
+
repos = gh.get_user_repos(user)
|
|
202
|
+
else:
|
|
203
|
+
print("You must specify either an organization or a user.")
|
|
204
|
+
|
|
205
|
+
details = []
|
|
206
|
+
|
|
207
|
+
print(f"\nFinding repos for: {owner} ({account_type})\n")
|
|
208
|
+
|
|
209
|
+
if repos:
|
|
210
|
+
for repo in repos:
|
|
211
|
+
record = {}
|
|
212
|
+
record['name'] = repo.get('name')
|
|
213
|
+
record['fork'] = repo.get('fork')
|
|
214
|
+
record['updated_at'] = repo.get('updated_at')
|
|
215
|
+
record['pushed_at'] = repo.get('pushed_at')
|
|
216
|
+
#if record.get("fork"):
|
|
217
|
+
# full_repo = gh.get_repo(owner=")
|
|
218
|
+
# record['parent'] = repo.get('parent').get('full_name')
|
|
219
|
+
|
|
220
|
+
record['private'] = repo.get('private')
|
|
221
|
+
|
|
222
|
+
if ssh_clone_url:
|
|
223
|
+
# If the boolean is set set the ssh_url instead
|
|
224
|
+
record['clone_url'] = repo.get('ssh_url')
|
|
225
|
+
else:
|
|
226
|
+
record['clone_url'] = repo.get('clone_url')
|
|
227
|
+
print(f"Found repo: {repo['name']}")
|
|
228
|
+
details.append(record)
|
|
229
|
+
|
|
230
|
+
if sync:
|
|
231
|
+
clone_url = repo.get('clone_url')
|
|
232
|
+
repo_path = kgit.clone_repo(clone_url)
|
|
233
|
+
print(f"Syncing repo: {clone_url} in directorty {repo_path}")
|
|
234
|
+
kospex.sync_repo(repo_path)
|
|
235
|
+
|
|
236
|
+
table = PrettyTable()
|
|
237
|
+
table.field_names = ["Name", "fork", "private", "owner", "clone_url", "pushed_at", "status"]
|
|
238
|
+
table.align["Name"] = "l"
|
|
239
|
+
table.align["clone_url"] = "l"
|
|
240
|
+
table.align["status"] = "l"
|
|
241
|
+
|
|
242
|
+
for detail in details:
|
|
243
|
+
#print(detail)
|
|
244
|
+
days_ago = KospexUtils.days_ago(detail.get("pushed_at"))
|
|
245
|
+
status = "Unknown"
|
|
246
|
+
if days_ago:
|
|
247
|
+
status = KospexUtils.development_status(days_ago)
|
|
248
|
+
table.add_row([detail.get("name"), detail.get("fork"),
|
|
249
|
+
detail.get("private"), owner, detail.get("clone_url"),
|
|
250
|
+
detail.get("pushed_at"), status])
|
|
251
|
+
|
|
252
|
+
print(table)
|
|
253
|
+
|
|
254
|
+
# Write out the repo list to a file
|
|
255
|
+
if out_repo_list:
|
|
256
|
+
with open(out_repo_list, "w", encoding='utf-8') as file:
|
|
257
|
+
for detail in details:
|
|
258
|
+
file.write(detail['clone_url'] + "\n")
|
|
259
|
+
|
|
260
|
+
@cli.command("bitbucket")
|
|
261
|
+
@click.option('-workspace', type=click.STRING, help="Workspace to query (Mandatory)")
|
|
262
|
+
#@click.option('-no-auth', is_flag=True, help="Access the Github API unauthenticated.")
|
|
263
|
+
#@click.option('-list-repos', is_flag=True, type=click.STRING)
|
|
264
|
+
#@click.option('-sync', is_flag=True)
|
|
265
|
+
@click.option('-out-repo-list', type=click.Path(), help="File to write clone URLs to.")
|
|
266
|
+
@click.option('-test-auth', is_flag=True, default=False, help="Test BitBucket credentials can authenticate.")
|
|
267
|
+
@click.option('-out-raw', type=click.Path(), help="Output raw JSON results to the specified filename")
|
|
268
|
+
def bitbucket(workspace, out_repo_list, test_auth, out_raw):
|
|
269
|
+
"""
|
|
270
|
+
Interact with the BitBucket API to query repos in a workspace.
|
|
271
|
+
|
|
272
|
+
This command requires the following environment variables to be set:
|
|
273
|
+
BITBUCKET_USERNAME and
|
|
274
|
+
BITBUCKET_APP_PASSWORD
|
|
275
|
+
|
|
276
|
+
The bitbucket username is in the "account settings" section of your bitbucket account.
|
|
277
|
+
This is NOT your email address.
|
|
278
|
+
|
|
279
|
+
"""
|
|
280
|
+
|
|
281
|
+
bb = KospexBitbucket()
|
|
282
|
+
click.echo()
|
|
283
|
+
if bb.get_env_credentials():
|
|
284
|
+
print("Found bitbucket credentials in the environment.")
|
|
285
|
+
else:
|
|
286
|
+
print("Could not find bitbucket credentials in the environment.")
|
|
287
|
+
print("Please set BITBUCKET_USERNAME and BITBUCKET_APP_PASSWORD.\n")
|
|
288
|
+
exit(1)
|
|
289
|
+
|
|
290
|
+
if test_auth:
|
|
291
|
+
if bb.test_auth():
|
|
292
|
+
print("Authentication successful.\n")
|
|
293
|
+
else:
|
|
294
|
+
print("\nAuthentication FAILED!.",
|
|
295
|
+
"\nCheck your BITBUCKET_USERNAME and BITBUCKET_APP_PASSWORD\n")
|
|
296
|
+
exit(0)
|
|
297
|
+
|
|
298
|
+
if not workspace:
|
|
299
|
+
print("\nERROR: You MUST specify a workspace.\n")
|
|
300
|
+
exit(1)
|
|
301
|
+
|
|
302
|
+
table = PrettyTable()
|
|
303
|
+
table.field_names = ["Name", "clone_url", "is_private"]
|
|
304
|
+
table.align["Name"] = "l"
|
|
305
|
+
table.align["clone_url"] = "l"
|
|
306
|
+
table.align["is_private"] = "c"
|
|
307
|
+
|
|
308
|
+
repos = bb.get_repos(workspace)
|
|
309
|
+
|
|
310
|
+
# TODO - provide an option to write this table to a CSV
|
|
311
|
+
# TODO - add extra metadata like created, last updated and repo status
|
|
312
|
+
for r in repos:
|
|
313
|
+
#print(r.get("full_name"), bb.get_https_clone_url(r))
|
|
314
|
+
table.add_row([r.get("slug"), bb.get_https_clone_url(r), r.get("is_private")])
|
|
315
|
+
|
|
316
|
+
print(table)
|
|
317
|
+
|
|
318
|
+
if out_repo_list:
|
|
319
|
+
with open(out_repo_list, "w", encoding='utf-8') as file:
|
|
320
|
+
for r in repos:
|
|
321
|
+
file.write(bb.get_https_clone_url(r) + "\n")
|
|
322
|
+
|
|
323
|
+
if out_raw:
|
|
324
|
+
with open(out_raw, "w", encoding='utf-8') as raw_file:
|
|
325
|
+
raw_file.write(json.dumps(repos))
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
if __name__ == '__main__':
|
|
329
|
+
cli()
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
|
+
Name: kospex
|
|
3
|
+
Version: 0.0.14
|
|
4
|
+
Summary: The kospex CLI tool.
|
|
5
|
+
Author-email: Peter Freiberg <peter.freiberg@gmail.com>
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2023 Peter Freiberg
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
|
|
28
|
+
Project-URL: Homepage, https://kospex.io
|
|
29
|
+
Project-URL: Issues, https://github.com/kospex/kospex/issues
|
|
30
|
+
Project-URL: Repository, https://github.com/kospex/kospex
|
|
31
|
+
Keywords: analytics,git,code analytics,predictive maintenance
|
|
32
|
+
Requires-Python: >=3.11
|
|
33
|
+
Description-Content-Type: text/markdown
|
|
34
|
+
License-File: LICENSE
|
|
35
|
+
Requires-Dist: Click
|
|
36
|
+
Requires-Dist: flask
|
|
37
|
+
Requires-Dist: prettytable
|
|
38
|
+
Requires-Dist: dateutils
|
|
39
|
+
Requires-Dist: panopticas
|
|
40
|
+
Requires-Dist: python-dotenv
|
|
41
|
+
Requires-Dist: sqlite_utils
|
|
42
|
+
Requires-Dist: requests
|
|
43
|
+
Requires-Dist: PyYAML
|
|
44
|
+
|
|
45
|
+
# kospex
|
|
46
|
+
|
|
47
|
+
Kospex is a CLI which aims to _"look at the guts of your code"_ to help gain insights into your developers and technology landscape.
|
|
48
|
+
It uses database structure from the excellent [Mergestat lite](https://github.com/mergestat/mergestat-lite) to model data from git repositories.
|
|
49
|
+
|
|
50
|
+
For details on changes, see the [changelog](https://github.com/kospex/kospex/blob/main/CHANGELOG.md)
|
|
51
|
+
|
|
52
|
+
## Installation, setup and usage
|
|
53
|
+
|
|
54
|
+
See the official [installation documentation](https://kospex.io/getting-started)
|
|
55
|
+
|
|
56
|
+
## What is a kospex?
|
|
57
|
+
|
|
58
|
+
We're aiming to [k]now your c[o]de by in[spe]cting the haruspe[x].
|
|
59
|
+
From Wikipedia, _The Latin terms haruspex and haruspicina are from an archaic word, hīra = "entrails, intestines"_
|
|
60
|
+
|
|
61
|
+
So we're going to help look at the "guts of your code" to gain an understanding of the applications, technology landscape (sprawl?) and developers.
|