codedd-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codedd_cli/__init__.py +3 -0
- codedd_cli/__main__.py +19 -0
- codedd_cli/api/__init__.py +4 -0
- codedd_cli/api/client.py +118 -0
- codedd_cli/api/endpoints.py +44 -0
- codedd_cli/api/exceptions.py +25 -0
- codedd_cli/auditor/__init__.py +6 -0
- codedd_cli/auditor/architecture_analyzer.py +1241 -0
- codedd_cli/auditor/architecture_prompts.py +171 -0
- codedd_cli/auditor/complexity_analyzer.py +942 -0
- codedd_cli/auditor/dependency_scanner.py +2478 -0
- codedd_cli/auditor/file_auditor.py +572 -0
- codedd_cli/auditor/git_stats_collector.py +332 -0
- codedd_cli/auditor/response_parser.py +487 -0
- codedd_cli/auditor/vulnerability_validator.py +324 -0
- codedd_cli/auth/__init__.py +4 -0
- codedd_cli/auth/session.py +41 -0
- codedd_cli/auth/token_manager.py +87 -0
- codedd_cli/cli.py +69 -0
- codedd_cli/commands/__init__.py +1 -0
- codedd_cli/commands/audit_cmd.py +1877 -0
- codedd_cli/commands/audits_cmd.py +273 -0
- codedd_cli/commands/auth_cmd.py +230 -0
- codedd_cli/commands/config_cmd.py +454 -0
- codedd_cli/commands/scope_cmd.py +967 -0
- codedd_cli/config/__init__.py +4 -0
- codedd_cli/config/constants.py +22 -0
- codedd_cli/config/settings.py +369 -0
- codedd_cli/llm/__init__.py +1 -0
- codedd_cli/llm/key_manager.py +271 -0
- codedd_cli/models/__init__.py +5 -0
- codedd_cli/models/account.py +13 -0
- codedd_cli/models/audit.py +32 -0
- codedd_cli/models/local_directory.py +26 -0
- codedd_cli/scanner/__init__.py +18 -0
- codedd_cli/scanner/file_classifier.py +247 -0
- codedd_cli/scanner/file_walker.py +207 -0
- codedd_cli/scanner/line_counter.py +75 -0
- codedd_cli/utils/__init__.py +1 -0
- codedd_cli/utils/directory_validator.py +179 -0
- codedd_cli/utils/display.py +493 -0
- codedd_cli/utils/payload_inspector.py +180 -0
- codedd_cli/utils/security.py +14 -0
- codedd_cli/utils/validators.py +37 -0
- codedd_cli-0.1.0.dist-info/METADATA +276 -0
- codedd_cli-0.1.0.dist-info/RECORD +49 -0
- codedd_cli-0.1.0.dist-info/WHEEL +4 -0
- codedd_cli-0.1.0.dist-info/entry_points.txt +3 -0
- codedd_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Data models for audit entities returned by the CLI API."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from typing import Optional
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass(frozen=True)
|
|
8
|
+
class Audit:
|
|
9
|
+
"""A single-repository audit."""
|
|
10
|
+
|
|
11
|
+
audit_uuid: str
|
|
12
|
+
audit_name: str
|
|
13
|
+
audit_status: str
|
|
14
|
+
audit_type: str = "single"
|
|
15
|
+
ai_synthesis: Optional[str] = None
|
|
16
|
+
repo_url: str = ""
|
|
17
|
+
number_files: int = 0
|
|
18
|
+
lines_of_code: int = 0
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class GroupAudit:
|
|
23
|
+
"""A group audit (portfolio / multi-repo)."""
|
|
24
|
+
|
|
25
|
+
audit_uuid: str
|
|
26
|
+
audit_name: str
|
|
27
|
+
audit_status: str
|
|
28
|
+
audit_type: str = "group"
|
|
29
|
+
ai_synthesis: Optional[str] = None
|
|
30
|
+
number_of_sub_audits: int = 0
|
|
31
|
+
number_files: int = 0
|
|
32
|
+
lines_of_code: int = 0
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Data model for a validated local directory added to an audit scope."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Optional
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass
|
|
8
|
+
class LocalDirectory:
|
|
9
|
+
"""
|
|
10
|
+
Represents a validated local Git repository directory.
|
|
11
|
+
|
|
12
|
+
Attributes:
|
|
13
|
+
path: Absolute path to the directory.
|
|
14
|
+
repo_name: Short name derived from the directory basename.
|
|
15
|
+
branch: Currently checked-out branch (or HEAD ref).
|
|
16
|
+
commit_hash: Short hash of the current HEAD commit.
|
|
17
|
+
is_valid: Whether all validation checks passed.
|
|
18
|
+
error: Validation error message (empty when valid).
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
path: str
|
|
22
|
+
repo_name: str = ""
|
|
23
|
+
branch: str = ""
|
|
24
|
+
commit_hash: str = ""
|
|
25
|
+
is_valid: bool = False
|
|
26
|
+
error: str = ""
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Local repository scanner — walks directories, classifies files, and counts LoC.
|
|
3
|
+
|
|
4
|
+
This package runs **entirely on the user's machine**. No source code or file
|
|
5
|
+
contents are ever sent to CodeDD; only the resulting metadata (file paths,
|
|
6
|
+
types, and line counts) is transmitted via the API.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from codedd_cli.scanner.file_classifier import get_file_type, should_exclude_file
|
|
10
|
+
from codedd_cli.scanner.line_counter import count_lines
|
|
11
|
+
from codedd_cli.scanner.file_walker import scan_repository
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"get_file_type",
|
|
15
|
+
"should_exclude_file",
|
|
16
|
+
"count_lines",
|
|
17
|
+
"scan_repository",
|
|
18
|
+
]
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"""
|
|
2
|
+
File type classification for the local scanner.
|
|
3
|
+
|
|
4
|
+
Mirrors the server-side ``file_extensions.py`` logic so that the metadata
|
|
5
|
+
produced locally matches exactly what the CodeDD platform expects.
|
|
6
|
+
|
|
7
|
+
Categories:
|
|
8
|
+
Source Code, Configuration, Documentation, Data File,
|
|
9
|
+
Binary, Media, Archive, System, Security, Other
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import os
|
|
13
|
+
|
|
14
|
+
# ---------------------------------------------------------------------------
|
|
15
|
+
# Extension sets — kept in sync with the server-side definitions
|
|
16
|
+
# ---------------------------------------------------------------------------
|
|
17
|
+
|
|
18
|
+
CONFIGURATION_FILES = {
|
|
19
|
+
'.yaml', '.yml', '.ini', '__init__.py', '.env', '.conf', '.config', '.cfg',
|
|
20
|
+
'.properties', '.prefs', '.props', '.toml', '.cnf', '.json',
|
|
21
|
+
'.dist', '.opt', '.plist', '.reg', '.settings', '.editorconfig',
|
|
22
|
+
'.htaccess', '.htpasswd', '.npmrc', '.nvmrc', '.bowerrc',
|
|
23
|
+
'.eslintrc', '.prettierrc', '.stylelintrc', '.babelrc',
|
|
24
|
+
'.eslintignore', '.prettierignore', '.dockerignore', '.gitignore',
|
|
25
|
+
'.env.local', '.env.development', '.env.production', '.env.test',
|
|
26
|
+
'Dockerfile', 'docker-compose.yml', 'Makefile', 'CMakeLists.txt',
|
|
27
|
+
'webpack.config.js', 'rollup.config.js', 'vite.config.js', 'snowpack.config.js',
|
|
28
|
+
'tsconfig.json', 'package.json', 'composer.json', 'project.json',
|
|
29
|
+
'angular.json', 'nx.json', 'lerna.json', 'rush.json',
|
|
30
|
+
'.travis.yml', '.gitlab-ci.yml', '.github/workflows', 'Jenkinsfile',
|
|
31
|
+
'azure-pipelines.yml', '.circleci/config.yml', 'bitbucket-pipelines.yml',
|
|
32
|
+
'buildkite.yml', 'appveyor.yml', 'cloudbuild.yaml', 'codeship-services.yml',
|
|
33
|
+
'setup.cfg', 'pyproject.toml', 'requirements.txt', 'Pipfile',
|
|
34
|
+
'tox.ini', 'pytest.ini', '.coveragerc', '.flake8', 'poetry.lock',
|
|
35
|
+
'conda.yaml', 'environment.yml', 'setup.py', 'MANIFEST.in',
|
|
36
|
+
'docker-compose.override.yml', 'docker-compose.prod.yml', 'docker-compose.dev.yml',
|
|
37
|
+
'kubernetes.yaml', 'helm.yaml', 'values.yaml', 'Chart.yaml', 'kustomization.yaml',
|
|
38
|
+
'terraform.tfvars', 'terragrunt.hcl', 'serverless.yml', 'cloudformation.yaml',
|
|
39
|
+
'pulumi.yaml', 'ansible.cfg', 'inventory.yml', 'playbook.yml',
|
|
40
|
+
'.browserslistrc', '.postcssrc', '.cssnanorc', '.stylelintignore',
|
|
41
|
+
'tailwind.config.js', 'next.config.js', 'nuxt.config.js', 'svelte.config.js',
|
|
42
|
+
'remix.config.js', 'astro.config.mjs', '.parcelrc', 'gatsby-config.js',
|
|
43
|
+
'gunicorn.conf.py', 'uwsgi.ini', 'nginx.conf', 'apache2.conf',
|
|
44
|
+
'php.ini', 'my.cnf', 'postgresql.conf', 'redis.conf', 'supervisord.conf',
|
|
45
|
+
'pm2.config.js', 'nodemon.json', 'nest-cli.json', 'vercel.json',
|
|
46
|
+
'.nycrc', '.mocharc', '.jestrc', 'karma.conf.js', 'cypress.json',
|
|
47
|
+
'playwright.config.js', 'sonar-project.properties', '.yamllint',
|
|
48
|
+
'vitest.config.js', 'ava.config.js', 'wallaby.js', 'jasmine.json',
|
|
49
|
+
'.vscode/settings.json', '.vscode/launch.json', '.idea/workspace.xml',
|
|
50
|
+
'.eclipse/org.eclipse.jdt.core.prefs', '.netbeans/project.properties',
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
SOURCE_CODE_FILES = {
|
|
54
|
+
'.js', '.jsx', '.ts', '.tsx', '.vue', '.svelte', '.php', '.html', '.htm',
|
|
55
|
+
'.css', '.scss', '.sass', '.less', '.styl', '.wasm', '.mjs', '.cjs',
|
|
56
|
+
'.astro', '.mdx', '.razor', '.cshtml', '.jsp', '.asp', '.aspx',
|
|
57
|
+
'.php4', '.php5', '.phtml', '.ctp', '.module', '.inc.php',
|
|
58
|
+
'.d.ts', '.js.map', '.mts', '.cts', '.jsx.map', '.tsx.map',
|
|
59
|
+
'.es6', '.es', '.iife.js', '.umd.js', '.amd.js', '.esm.js',
|
|
60
|
+
'.py', '.java', '.cpp', '.cc', '.cxx', '.hpp', '.c', '.h', '.cs', '.fs',
|
|
61
|
+
'.go', '.rs', '.rb', '.rake', '.swift', '.kt', '.kts', '.scala', '.sc',
|
|
62
|
+
'.clj', '.cljs', '.cljc', '.erl', '.ex', '.exs', '.hs', '.lhs', '.lua',
|
|
63
|
+
'.pl', '.pm', '.t', '.r', '.rmd', '.jl', '.dart', '.groovy', '.tcl',
|
|
64
|
+
'.nim', '.cr', '.ml', '.re', '.res', '.elm', '.zig', '.v', '.gleam',
|
|
65
|
+
'.hx', '.ceylon', '.idr', '.purs', '.dhall', '.bal', '.rkt', '.io',
|
|
66
|
+
'.sh', '.bash', '.zsh', '.fish', '.bat', '.cmd', '.ps1', '.psm1',
|
|
67
|
+
'.vbs', '.vba', '.awk', '.sed', '.ksh', '.csh', '.tcsh', '.nu',
|
|
68
|
+
'.erb', '.haml', '.slim', '.pug', '.jade', '.jinja', '.jinja2',
|
|
69
|
+
'.mustache', '.handlebars', '.hbs', '.twig', '.liquid', '.njk',
|
|
70
|
+
'.blade.php', '.volt', '.latte', '.smarty', '.plates', '.tpl',
|
|
71
|
+
'.sql', '.hql', '.cypher', '.graphql', '.gql', '.proto', '.thrift',
|
|
72
|
+
'.cmake', '.gradle', '.m4', '.am', '.in', '.ac', '.f90', '.f95', '.f03',
|
|
73
|
+
'.m', '.mm', '.xib', '.storyboard',
|
|
74
|
+
'.gd', '.unity', '.unityproj', '.prefab', '.mat', '.shader',
|
|
75
|
+
'.hlsl', '.cg', '.fx', '.fxh', '.usf', '.ush',
|
|
76
|
+
'.ipynb', '.sage', '.sce', '.sci', '.stan', '.do',
|
|
77
|
+
'.sps', '.sas', '.mlx', '.mplstyle',
|
|
78
|
+
'.py3', '.pyx', '.pxd', '.pxi', '.rpy', '.rpym', '.rviz',
|
|
79
|
+
'.asm', '.s', '.nasm', '.gas', '.lst', '.mac',
|
|
80
|
+
'.vh', '.vhd', '.vhdl', '.sv', '.svh', '.svi',
|
|
81
|
+
'.mli', '.fsi', '.fsx', '.fsscript',
|
|
82
|
+
'.scm', '.ss', '.rktl', '.odin', '.d', '.di',
|
|
83
|
+
'.mod.c', '.dts', '.dtsi',
|
|
84
|
+
'.prisma', '.sdl', '.openapi', '.swagger', '.raml', '.wsdl',
|
|
85
|
+
'.wsf',
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
DOCUMENTATION_FILES = {
|
|
89
|
+
'.md', '.markdown', '.txt', '.rtf', '.rst', '.asciidoc', '.adoc',
|
|
90
|
+
'.tex', '.latex', '.wiki', '.org', '.pod', '.rdoc', '.textile',
|
|
91
|
+
'.creole', '.mediawiki', '.dokuwiki', '.asc', '.man', '.mdwn',
|
|
92
|
+
'.doc', '.docx', '.odt', '.pages', '.wpd', '.wps',
|
|
93
|
+
'.pdf', '.xps', '.oxps', '.epub', '.mobi',
|
|
94
|
+
'.ppt', '.pptx', '.key', '.odp', '.pps', '.ppsx',
|
|
95
|
+
'.xls', '.xlsx', '.numbers', '.ods', '.csv', '.tsv',
|
|
96
|
+
'.drawio', '.vsdx', '.vsd', '.dgml', '.dgm',
|
|
97
|
+
'CHANGELOG', 'CONTRIBUTING', 'AUTHORS', 'MAINTAINERS',
|
|
98
|
+
'SECURITY', 'SUPPORT', 'THANKS', 'UPGRADING', 'VERSION',
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
DATA_FILES = {
|
|
102
|
+
'.json', '.xml', '.yaml', '.yml', '.csv', '.tsv', '.xlsx', '.xls',
|
|
103
|
+
'.parquet', '.orc', '.avro', '.arrow', '.feather', '.jsonl', '.ndjson',
|
|
104
|
+
'.db', '.sqlite', '.sqlite3', '.mdb', '.accdb', '.dbf', '.fdb',
|
|
105
|
+
'.rdb', '.pdb', '.odb', '.mdf', '.ldf', '.frm', '.ibd',
|
|
106
|
+
'.hdf', '.h5', '.fits', '.nc', '.cdf', '.grib',
|
|
107
|
+
'.shp', '.shx', '.kml', '.kmz', '.gpx', '.osm', '.geojson',
|
|
108
|
+
'.pb', '.pbtxt', '.ckpt', '.tflite', '.onnx',
|
|
109
|
+
'.pt', '.pth', '.safetensors', '.npy', '.npz',
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
BINARY_FILES = {
|
|
113
|
+
'.exe', '.msi', '.app', '.dmg', '.pkg', '.deb', '.rpm',
|
|
114
|
+
'.apk', '.ipa', '.dll', '.so', '.dylib', '.a', '.lib',
|
|
115
|
+
'.jar', '.war', '.ear', '.aar', '.gem',
|
|
116
|
+
'.whl', '.egg', '.pyd', '.pyo', '.pyc',
|
|
117
|
+
'.obj', '.o', '.ko', '.class', '.dex',
|
|
118
|
+
'.ilk', '.exp', '.pch',
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
MEDIA_FILES = {
|
|
122
|
+
'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.tif',
|
|
123
|
+
'.webp', '.heic', '.heif', '.raw', '.psd', '.ico', '.icns', '.avif',
|
|
124
|
+
'.svg', '.eps', '.ai',
|
|
125
|
+
'.mp3', '.wav', '.flac', '.m4a', '.aac', '.ogg', '.wma', '.opus',
|
|
126
|
+
'.mp4', '.mkv', '.avi', '.mov', '.wmv', '.flv', '.webm', '.m4v',
|
|
127
|
+
'.fbx', '.3ds', '.blend', '.stl', '.dae', '.dwg', '.dxf',
|
|
128
|
+
'.glb', '.gltf', '.usdz',
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
ARCHIVE_FILES = {
|
|
132
|
+
'.zip', '.rar', '.7z', '.tar', '.gz', '.bz2', '.xz',
|
|
133
|
+
'.tgz', '.tbz2', '.txz', '.cab', '.iso',
|
|
134
|
+
'.bak', '.backup', '.bkp',
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
SYSTEM_FILES = {
|
|
138
|
+
'.tmp', '.temp', '.swp', '.swo', '.swn',
|
|
139
|
+
'.cache', '.part', '.crdownload',
|
|
140
|
+
'.log', '.logs', '.out', '.err',
|
|
141
|
+
'.dump', '.dmp', '.crash', '.core',
|
|
142
|
+
'.lnk', '.url', '.pid', '.lock', '.sock',
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
SECURITY_FILES = {
|
|
146
|
+
'.key', '.pem', '.crt', '.cer', '.der', '.p12', '.pfx',
|
|
147
|
+
'.p7b', '.p7c', '.keystore', '.jks', '.truststore',
|
|
148
|
+
'.csr', '.pub', '.gpg', '.asc',
|
|
149
|
+
'.htpasswd', '.netrc', '.pgpass',
|
|
150
|
+
'.secrets', '.credentials', '.kdbx', '.vault',
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
# Directories always excluded from scanning
|
|
154
|
+
EXCLUDED_DIRECTORIES = {
|
|
155
|
+
'.git', '__pycache__', 'node_modules', '.venv', 'venv', 'env',
|
|
156
|
+
'.tox', '.mypy_cache', '.pytest_cache', '.ruff_cache',
|
|
157
|
+
'.next', '.nuxt', 'dist', 'build', '.build',
|
|
158
|
+
'target', 'out', 'bin', '.gradle', '.mvn',
|
|
159
|
+
'.idea', '.vscode', '.vs',
|
|
160
|
+
'vendor', 'bower_components',
|
|
161
|
+
'.terraform', '.serverless',
|
|
162
|
+
'coverage', '.nyc_output', 'htmlcov',
|
|
163
|
+
'.eggs', '*.egg-info',
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def get_file_extension(file_path: str) -> str:
|
|
168
|
+
"""Extract the file extension from a path (lowercase)."""
|
|
169
|
+
return os.path.splitext(file_path)[1].lower()
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def get_file_type(file_path: str) -> str:
|
|
173
|
+
"""
|
|
174
|
+
Determine the type of file based on its extension or filename.
|
|
175
|
+
|
|
176
|
+
Returns one of: 'Source Code', 'Configuration', 'Documentation',
|
|
177
|
+
'Data File', 'Binary', 'Media', 'Archive', 'System', 'Security', 'Other'.
|
|
178
|
+
"""
|
|
179
|
+
filename = os.path.basename(file_path)
|
|
180
|
+
|
|
181
|
+
# Check complete filename first (e.g. "Dockerfile", "Makefile")
|
|
182
|
+
if filename in CONFIGURATION_FILES:
|
|
183
|
+
return 'Configuration'
|
|
184
|
+
if filename in DOCUMENTATION_FILES:
|
|
185
|
+
return 'Documentation'
|
|
186
|
+
|
|
187
|
+
# Try the last extension (handles compound like .d.ts)
|
|
188
|
+
parts = filename.split('.')
|
|
189
|
+
if len(parts) > 1:
|
|
190
|
+
last_ext = f'.{parts[-1]}'
|
|
191
|
+
for ext_set, label in _EXTENSION_ORDER:
|
|
192
|
+
if last_ext in ext_set:
|
|
193
|
+
return label
|
|
194
|
+
|
|
195
|
+
# Fall back to os.path.splitext
|
|
196
|
+
extension = get_file_extension(file_path)
|
|
197
|
+
for ext_set, label in _EXTENSION_ORDER:
|
|
198
|
+
if extension in ext_set:
|
|
199
|
+
return label
|
|
200
|
+
|
|
201
|
+
return 'Other'
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
# Lookup order (source code first to match server behaviour)
|
|
205
|
+
_EXTENSION_ORDER = [
|
|
206
|
+
(SOURCE_CODE_FILES, 'Source Code'),
|
|
207
|
+
(CONFIGURATION_FILES, 'Configuration'),
|
|
208
|
+
(DOCUMENTATION_FILES, 'Documentation'),
|
|
209
|
+
(DATA_FILES, 'Data File'),
|
|
210
|
+
(BINARY_FILES, 'Binary'),
|
|
211
|
+
(MEDIA_FILES, 'Media'),
|
|
212
|
+
(ARCHIVE_FILES, 'Archive'),
|
|
213
|
+
(SYSTEM_FILES, 'System'),
|
|
214
|
+
(SECURITY_FILES, 'Security'),
|
|
215
|
+
]
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def should_exclude_file(file_path: str) -> bool:
|
|
219
|
+
"""
|
|
220
|
+
Return True if a file should be excluded from LoC counting.
|
|
221
|
+
|
|
222
|
+
Excluded: files inside .git directories, configuration files, and
|
|
223
|
+
non-source categories (docs, data, binary, media, archives, system, security).
|
|
224
|
+
"""
|
|
225
|
+
# Anything inside a .git directory
|
|
226
|
+
parts = file_path.replace('\\', '/').split('/')
|
|
227
|
+
if '.git' in parts:
|
|
228
|
+
return True
|
|
229
|
+
|
|
230
|
+
filename = os.path.basename(file_path)
|
|
231
|
+
if filename in CONFIGURATION_FILES:
|
|
232
|
+
return True
|
|
233
|
+
|
|
234
|
+
extension = get_file_extension(file_path)
|
|
235
|
+
return extension in _EXCLUDED_EXTENSIONS
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def should_exclude_directory(dir_name: str) -> bool:
|
|
239
|
+
"""Return True if a directory should be skipped entirely during scanning."""
|
|
240
|
+
return dir_name in EXCLUDED_DIRECTORIES
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
# Pre-computed set of extensions excluded from LoC counting
|
|
244
|
+
_EXCLUDED_EXTENSIONS = (
|
|
245
|
+
DOCUMENTATION_FILES | DATA_FILES | BINARY_FILES |
|
|
246
|
+
MEDIA_FILES | ARCHIVE_FILES | SYSTEM_FILES | SECURITY_FILES
|
|
247
|
+
)
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Local repository scanner.
|
|
3
|
+
|
|
4
|
+
Walks a Git repository directory, classifies each file, counts lines of code,
|
|
5
|
+
and produces a structured metadata payload ready for submission to the CodeDD API.
|
|
6
|
+
|
|
7
|
+
**No file contents are included** — only paths, types, and line counts.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
import os
|
|
12
|
+
import subprocess
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Dict, List, Optional, Tuple
|
|
16
|
+
|
|
17
|
+
from codedd_cli.scanner.file_classifier import (
|
|
18
|
+
get_file_type,
|
|
19
|
+
should_exclude_directory,
|
|
20
|
+
should_exclude_file,
|
|
21
|
+
)
|
|
22
|
+
from codedd_cli.scanner.line_counter import count_lines
|
|
23
|
+
|
|
24
|
+
logger = logging.getLogger(__name__)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass
|
|
28
|
+
class FileMetadata:
|
|
29
|
+
"""Metadata for a single scanned file (no content)."""
|
|
30
|
+
relative_path: str
|
|
31
|
+
file_type: str
|
|
32
|
+
lines_of_code: int
|
|
33
|
+
lines_of_doc: int
|
|
34
|
+
selected_for_audit: bool = True
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass
|
|
38
|
+
class FolderMetadata:
|
|
39
|
+
"""Aggregated metadata for a folder."""
|
|
40
|
+
relative_path: str
|
|
41
|
+
lines_of_code: int = 0
|
|
42
|
+
file_count: int = 0
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass
|
|
46
|
+
class ScanResult:
|
|
47
|
+
"""Complete scan result for a single repository directory."""
|
|
48
|
+
root_path: str
|
|
49
|
+
repo_name: str
|
|
50
|
+
branch: str
|
|
51
|
+
commit_hash: str
|
|
52
|
+
files: List[FileMetadata] = field(default_factory=list)
|
|
53
|
+
folders: List[FolderMetadata] = field(default_factory=list)
|
|
54
|
+
total_files: int = 0
|
|
55
|
+
total_lines_of_code: int = 0
|
|
56
|
+
total_lines_of_doc: int = 0
|
|
57
|
+
errors: List[str] = field(default_factory=list)
|
|
58
|
+
|
|
59
|
+
def to_dict(self) -> dict:
|
|
60
|
+
"""Serialise to a plain dict suitable for JSON encoding."""
|
|
61
|
+
return {
|
|
62
|
+
"root_path": self.root_path,
|
|
63
|
+
"repo_name": self.repo_name,
|
|
64
|
+
"branch": self.branch,
|
|
65
|
+
"commit_hash": self.commit_hash,
|
|
66
|
+
"total_files": self.total_files,
|
|
67
|
+
"total_lines_of_code": self.total_lines_of_code,
|
|
68
|
+
"total_lines_of_doc": self.total_lines_of_doc,
|
|
69
|
+
"files": [
|
|
70
|
+
{
|
|
71
|
+
"relative_path": f.relative_path,
|
|
72
|
+
"file_type": f.file_type,
|
|
73
|
+
"lines_of_code": f.lines_of_code,
|
|
74
|
+
"lines_of_doc": f.lines_of_doc,
|
|
75
|
+
"selected_for_audit": f.selected_for_audit,
|
|
76
|
+
}
|
|
77
|
+
for f in self.files
|
|
78
|
+
],
|
|
79
|
+
"folders": [
|
|
80
|
+
{
|
|
81
|
+
"relative_path": fd.relative_path,
|
|
82
|
+
"lines_of_code": fd.lines_of_code,
|
|
83
|
+
"file_count": fd.file_count,
|
|
84
|
+
}
|
|
85
|
+
for fd in self.folders
|
|
86
|
+
],
|
|
87
|
+
"errors": self.errors,
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _git_info(repo_path: str) -> Tuple[str, str]:
|
|
92
|
+
"""Return (branch, short_commit_hash) for a repo, or empty strings on error."""
|
|
93
|
+
try:
|
|
94
|
+
branch = subprocess.run(
|
|
95
|
+
["git", "rev-parse", "--abbrev-ref", "HEAD"],
|
|
96
|
+
cwd=repo_path, capture_output=True, text=True, timeout=10,
|
|
97
|
+
).stdout.strip()
|
|
98
|
+
|
|
99
|
+
commit = subprocess.run(
|
|
100
|
+
["git", "rev-parse", "--short", "HEAD"],
|
|
101
|
+
cwd=repo_path, capture_output=True, text=True, timeout=10,
|
|
102
|
+
).stdout.strip()
|
|
103
|
+
|
|
104
|
+
return branch, commit
|
|
105
|
+
except Exception:
|
|
106
|
+
return "", ""
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def scan_repository(
|
|
110
|
+
root_path: str,
|
|
111
|
+
progress_callback: Optional[callable] = None,
|
|
112
|
+
) -> ScanResult:
|
|
113
|
+
"""
|
|
114
|
+
Walk *root_path*, classify every file, and count lines of code.
|
|
115
|
+
|
|
116
|
+
Args:
|
|
117
|
+
root_path: Absolute path to the repository root.
|
|
118
|
+
progress_callback: Optional callable(current, total, file_path) invoked
|
|
119
|
+
for each file processed. Useful for Rich progress bars.
|
|
120
|
+
|
|
121
|
+
Returns:
|
|
122
|
+
A ``ScanResult`` containing all file and folder metadata.
|
|
123
|
+
"""
|
|
124
|
+
root = Path(root_path).resolve()
|
|
125
|
+
repo_name = root.name
|
|
126
|
+
branch, commit = _git_info(str(root))
|
|
127
|
+
|
|
128
|
+
result = ScanResult(
|
|
129
|
+
root_path=str(root),
|
|
130
|
+
repo_name=repo_name,
|
|
131
|
+
branch=branch,
|
|
132
|
+
commit_hash=commit,
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
# Phase 1: collect all eligible file paths
|
|
136
|
+
file_paths: List[str] = []
|
|
137
|
+
for dirpath, dirnames, filenames in os.walk(str(root)):
|
|
138
|
+
# Filter out excluded directories in-place so os.walk skips them
|
|
139
|
+
dirnames[:] = [
|
|
140
|
+
d for d in dirnames
|
|
141
|
+
if not should_exclude_directory(d)
|
|
142
|
+
]
|
|
143
|
+
for fname in filenames:
|
|
144
|
+
full_path = os.path.join(dirpath, fname)
|
|
145
|
+
# Skip symlinks
|
|
146
|
+
if os.path.islink(full_path):
|
|
147
|
+
continue
|
|
148
|
+
file_paths.append(full_path)
|
|
149
|
+
|
|
150
|
+
total_files = len(file_paths)
|
|
151
|
+
folder_aggregates: Dict[str, FolderMetadata] = {}
|
|
152
|
+
|
|
153
|
+
# Phase 2: classify and count
|
|
154
|
+
for idx, full_path in enumerate(file_paths):
|
|
155
|
+
try:
|
|
156
|
+
rel_path = os.path.relpath(full_path, start=str(root)).replace("\\", "/")
|
|
157
|
+
file_type = get_file_type(full_path)
|
|
158
|
+
|
|
159
|
+
# Count LoC (returns (0, 0) for excluded files)
|
|
160
|
+
loc, doc = count_lines(full_path)
|
|
161
|
+
|
|
162
|
+
fm = FileMetadata(
|
|
163
|
+
relative_path=rel_path,
|
|
164
|
+
file_type=file_type,
|
|
165
|
+
lines_of_code=loc,
|
|
166
|
+
lines_of_doc=doc,
|
|
167
|
+
selected_for_audit=(not should_exclude_file(full_path) and loc > 0),
|
|
168
|
+
)
|
|
169
|
+
result.files.append(fm)
|
|
170
|
+
|
|
171
|
+
if fm.selected_for_audit:
|
|
172
|
+
result.total_files += 1
|
|
173
|
+
result.total_lines_of_code += loc
|
|
174
|
+
result.total_lines_of_doc += doc
|
|
175
|
+
|
|
176
|
+
# Aggregate into parent folder and ensure all ancestor folders exist.
|
|
177
|
+
# The server-side import (import_file_list.py) discovers ALL directories
|
|
178
|
+
# via os.scandir. The CLI must produce the same set so the folder
|
|
179
|
+
# hierarchy can be reconstructed on the server when building TypeDB
|
|
180
|
+
# directory_content relations (folder → parent_folder → root).
|
|
181
|
+
parent_rel = os.path.dirname(rel_path).replace("\\", "/")
|
|
182
|
+
if parent_rel and parent_rel != ".":
|
|
183
|
+
# Direct parent gets LoC / file_count
|
|
184
|
+
if parent_rel not in folder_aggregates:
|
|
185
|
+
folder_aggregates[parent_rel] = FolderMetadata(relative_path=parent_rel)
|
|
186
|
+
folder_aggregates[parent_rel].lines_of_code += loc
|
|
187
|
+
folder_aggregates[parent_rel].file_count += 1
|
|
188
|
+
|
|
189
|
+
# Ensure every ancestor folder exists (with 0 direct metrics)
|
|
190
|
+
ancestor = os.path.dirname(parent_rel).replace("\\", "/")
|
|
191
|
+
while ancestor and ancestor != ".":
|
|
192
|
+
if ancestor not in folder_aggregates:
|
|
193
|
+
folder_aggregates[ancestor] = FolderMetadata(relative_path=ancestor)
|
|
194
|
+
ancestor = os.path.dirname(ancestor).replace("\\", "/")
|
|
195
|
+
|
|
196
|
+
if progress_callback:
|
|
197
|
+
progress_callback(idx + 1, total_files, rel_path)
|
|
198
|
+
|
|
199
|
+
except Exception as exc:
|
|
200
|
+
error_msg = f"Error scanning {full_path}: {exc}"
|
|
201
|
+
logger.warning(error_msg)
|
|
202
|
+
result.errors.append(error_msg)
|
|
203
|
+
|
|
204
|
+
# Build sorted folder list
|
|
205
|
+
result.folders = sorted(folder_aggregates.values(), key=lambda f: f.relative_path)
|
|
206
|
+
|
|
207
|
+
return result
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Local line-of-code counter.
|
|
3
|
+
|
|
4
|
+
Counts non-empty lines of code and comment/doc lines for a single file.
|
|
5
|
+
This is a pure-Python implementation that mirrors the server-side
|
|
6
|
+
``get_code_and_doc_lines`` fallback logic — no external ``linecounter``
|
|
7
|
+
binary is required.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
from typing import Tuple
|
|
12
|
+
|
|
13
|
+
from codedd_cli.scanner.file_classifier import (
|
|
14
|
+
get_file_type,
|
|
15
|
+
should_exclude_file,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
# File types eligible for LoC counting
|
|
19
|
+
_COUNTABLE_TYPES = frozenset({
|
|
20
|
+
'Source Code', 'Configuration', 'Documentation',
|
|
21
|
+
'System', 'Security', 'Other',
|
|
22
|
+
})
|
|
23
|
+
|
|
24
|
+
# Single-line comment prefixes by convention
|
|
25
|
+
_COMMENT_PREFIXES = ('#', '//', '/*', '*', '*/', '--', ';', '%', 'REM ')
|
|
26
|
+
|
|
27
|
+
# Maximum file size (in bytes) to attempt reading — skip very large binaries
|
|
28
|
+
_MAX_FILE_SIZE = 50 * 1024 * 1024 # 50 MB
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def count_lines(file_path: str) -> Tuple[int, int]:
|
|
32
|
+
"""
|
|
33
|
+
Count lines of code and lines of documentation/comments in *file_path*.
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
file_path: Absolute path to the file.
|
|
37
|
+
|
|
38
|
+
Returns:
|
|
39
|
+
Tuple of (lines_of_code, lines_of_doc).
|
|
40
|
+
Returns ``(0, 0)`` for excluded or unreadable files.
|
|
41
|
+
"""
|
|
42
|
+
if should_exclude_file(file_path):
|
|
43
|
+
return 0, 0
|
|
44
|
+
|
|
45
|
+
file_type = get_file_type(file_path)
|
|
46
|
+
if file_type not in _COUNTABLE_TYPES:
|
|
47
|
+
return 0, 0
|
|
48
|
+
|
|
49
|
+
# Safety: skip extremely large files
|
|
50
|
+
try:
|
|
51
|
+
size = os.path.getsize(file_path)
|
|
52
|
+
if size > _MAX_FILE_SIZE:
|
|
53
|
+
return 0, 0
|
|
54
|
+
except OSError:
|
|
55
|
+
return 0, 0
|
|
56
|
+
|
|
57
|
+
try:
|
|
58
|
+
with open(file_path, 'r', encoding='utf-8', errors='replace') as fh:
|
|
59
|
+
lines = fh.readlines()
|
|
60
|
+
except (OSError, PermissionError):
|
|
61
|
+
return 1, 0 # Minimum fallback
|
|
62
|
+
|
|
63
|
+
non_empty = 0
|
|
64
|
+
comment_lines = 0
|
|
65
|
+
|
|
66
|
+
for line in lines:
|
|
67
|
+
stripped = line.strip()
|
|
68
|
+
if not stripped:
|
|
69
|
+
continue
|
|
70
|
+
non_empty += 1
|
|
71
|
+
if stripped.startswith(_COMMENT_PREFIXES):
|
|
72
|
+
comment_lines += 1
|
|
73
|
+
|
|
74
|
+
code_lines = max(non_empty - comment_lines, 1) if non_empty > 0 else 0
|
|
75
|
+
return code_lines, comment_lines
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|