sphinx-codelinks 1.1.0__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sphinx_codelinks-1.1.0 → sphinx_codelinks-1.3.0}/PKG-INFO +7 -4
- {sphinx_codelinks-1.1.0 → sphinx_codelinks-1.3.0}/pyproject.toml +28 -59
- {sphinx_codelinks-1.1.0 → sphinx_codelinks-1.3.0}/src/sphinx_codelinks/analyse/analyse.py +41 -28
- {sphinx_codelinks-1.1.0 → sphinx_codelinks-1.3.0}/src/sphinx_codelinks/analyse/oneline_parser.py +2 -10
- {sphinx_codelinks-1.1.0 → sphinx_codelinks-1.3.0}/src/sphinx_codelinks/analyse/projects.py +4 -10
- {sphinx_codelinks-1.1.0 → sphinx_codelinks-1.3.0}/src/sphinx_codelinks/analyse/utils.py +141 -21
- {sphinx_codelinks-1.1.0 → sphinx_codelinks-1.3.0}/src/sphinx_codelinks/cmd.py +31 -5
- {sphinx_codelinks-1.1.0 → sphinx_codelinks-1.3.0}/src/sphinx_codelinks/config.py +53 -3
- sphinx_codelinks-1.3.0/src/sphinx_codelinks/logger.py +261 -0
- {sphinx_codelinks-1.1.0 → sphinx_codelinks-1.3.0}/src/sphinx_codelinks/source_discover/config.py +14 -0
- sphinx_codelinks-1.3.0/src/sphinx_codelinks/source_discover/source_discover.py +112 -0
- {sphinx_codelinks-1.1.0 → sphinx_codelinks-1.3.0}/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py +26 -14
- {sphinx_codelinks-1.1.0 → sphinx_codelinks-1.3.0}/src/sphinx_codelinks/sphinx_extension/source_tracing.py +42 -5
- sphinx_codelinks-1.1.0/src/sphinx_codelinks/logger.py +0 -94
- sphinx_codelinks-1.1.0/src/sphinx_codelinks/source_discover/source_discover.py +0 -66
- {sphinx_codelinks-1.1.0 → sphinx_codelinks-1.3.0}/LICENSE +0 -0
- {sphinx_codelinks-1.1.0 → sphinx_codelinks-1.3.0}/README.md +0 -0
- {sphinx_codelinks-1.1.0 → sphinx_codelinks-1.3.0}/src/sphinx_codelinks/__init__.py +0 -0
- {sphinx_codelinks-1.1.0 → sphinx_codelinks-1.3.0}/src/sphinx_codelinks/analyse/models.py +0 -0
- {sphinx_codelinks-1.1.0 → sphinx_codelinks-1.3.0}/src/sphinx_codelinks/needextend_write.py +0 -0
- {sphinx_codelinks-1.1.0 → sphinx_codelinks-1.3.0}/src/sphinx_codelinks/sphinx_extension/debug.py +0 -0
- {sphinx_codelinks-1.1.0 → sphinx_codelinks-1.3.0}/src/sphinx_codelinks/sphinx_extension/html_wrapper.py +0 -0
- {sphinx_codelinks-1.1.0 → sphinx_codelinks-1.3.0}/src/sphinx_codelinks/sphinx_extension/ub_sct.css +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: sphinx-codelinks
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: Fast Source Code Traceability for Sphinx-Needs
|
|
5
5
|
Author-email: team useblocks <info@useblocks.com>
|
|
6
6
|
Maintainer-email: Marco Heinemann <marco.heinemann@useblocks.com>, Jui-Wen Chen <jui-wen.chen@useblocks.com>
|
|
@@ -8,12 +8,12 @@ Requires-Python: >= 3.12
|
|
|
8
8
|
Description-Content-Type: text/markdown
|
|
9
9
|
License-File: LICENSE
|
|
10
10
|
Requires-Dist: comment-parser>=1.2.4
|
|
11
|
-
Requires-Dist:
|
|
11
|
+
Requires-Dist: ignore-python>=0.3.3
|
|
12
12
|
Requires-Dist: typer>=0.16.0
|
|
13
13
|
Requires-Dist: click < 8.2
|
|
14
14
|
Requires-Dist: jsonschema
|
|
15
|
-
Requires-Dist: sphinx>=7.4,<
|
|
16
|
-
Requires-Dist: sphinx-needs>=
|
|
15
|
+
Requires-Dist: sphinx>=7.4,<10
|
|
16
|
+
Requires-Dist: sphinx-needs>=5,<9
|
|
17
17
|
Requires-Dist: jinja2
|
|
18
18
|
Requires-Dist: pygments
|
|
19
19
|
Requires-Dist: docutils
|
|
@@ -23,6 +23,9 @@ Requires-Dist: tree-sitter-python>=0.23.6
|
|
|
23
23
|
Requires-Dist: tree-sitter~=0.25.1
|
|
24
24
|
Requires-Dist: tree-sitter-c-sharp>=0.23.1
|
|
25
25
|
Requires-Dist: tree-sitter-yaml>=0.7.1
|
|
26
|
+
Requires-Dist: tree-sitter-rust>=0.23.0
|
|
27
|
+
Requires-Dist: tree-sitter-go>=0.23.0
|
|
28
|
+
Requires-Dist: tree-sitter-json>=0.24.8
|
|
26
29
|
|
|
27
30
|
# Sphinx CodeLinks
|
|
28
31
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sphinx-codelinks"
|
|
3
|
-
version = "1.
|
|
3
|
+
version = "1.3.0"
|
|
4
4
|
description = "Fast Source Code Traceability for Sphinx-Needs"
|
|
5
5
|
authors = [{ name = "team useblocks", email = "info@useblocks.com" }]
|
|
6
6
|
maintainers = [
|
|
@@ -12,12 +12,12 @@ readme = "README.md"
|
|
|
12
12
|
requires-python = ">= 3.12"
|
|
13
13
|
dependencies = [
|
|
14
14
|
"comment-parser>=1.2.4",
|
|
15
|
-
"
|
|
15
|
+
"ignore-python>=0.3.3",
|
|
16
16
|
"typer>=0.16.0",
|
|
17
17
|
"click < 8.2", # click 8.2.* produces empty errors if no args are given
|
|
18
18
|
"jsonschema",
|
|
19
|
-
"sphinx>=7.4,<
|
|
20
|
-
"sphinx-needs>=
|
|
19
|
+
"sphinx>=7.4,<10",
|
|
20
|
+
"sphinx-needs>=5,<9",
|
|
21
21
|
"jinja2",
|
|
22
22
|
"pygments",
|
|
23
23
|
"docutils", # constrained by user or Sphinx
|
|
@@ -29,70 +29,48 @@ dependencies = [
|
|
|
29
29
|
"tree-sitter~=0.25.1",
|
|
30
30
|
"tree-sitter-c-sharp>=0.23.1",
|
|
31
31
|
"tree-sitter-yaml>=0.7.1",
|
|
32
|
+
"tree-sitter-rust>=0.23.0",
|
|
33
|
+
"tree-sitter-go>=0.23.0",
|
|
34
|
+
"tree-sitter-json>=0.24.8",
|
|
32
35
|
]
|
|
33
36
|
|
|
34
37
|
[build-system]
|
|
35
38
|
requires = ["flit_core >=3.4,<4"]
|
|
36
39
|
build-backend = "flit_core.buildapi"
|
|
37
40
|
|
|
38
|
-
[
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
"
|
|
42
|
-
"
|
|
41
|
+
[dependency-groups]
|
|
42
|
+
testing = [
|
|
43
|
+
"pytest>=8.2.2",
|
|
44
|
+
"pytest-cov>=5.0.0",
|
|
45
|
+
"pytest-docker>=3.1.2",
|
|
43
46
|
"syrupy>=4.9.1",
|
|
44
|
-
"
|
|
47
|
+
"psutil>=7.0.0",
|
|
45
48
|
"moto ~= 5.0",
|
|
46
|
-
"
|
|
49
|
+
"toml>=0.10.2",
|
|
50
|
+
"furo>=2024.5.6",
|
|
51
|
+
]
|
|
52
|
+
docs = [
|
|
53
|
+
"furo>=2024.5.6",
|
|
47
54
|
"myst-parser>=4.0.0",
|
|
48
|
-
"pydantic ~= 2.9",
|
|
49
|
-
"pip-licenses>=5.0.0",
|
|
50
|
-
"psutil>=7.0.0",
|
|
51
|
-
"pytest-cov>=5.0.0",
|
|
52
|
-
"pytest>=8.2.2",
|
|
53
|
-
"simple-build>=0.0.2",
|
|
54
55
|
"sphinx-design>=0.6.1",
|
|
55
|
-
"types-psutil>=7.0.0.20250218",
|
|
56
|
-
"uv>=0.5.5",
|
|
57
|
-
"pytest-docker>=3.1.2",
|
|
58
|
-
"shiv>=1.0.8",
|
|
59
|
-
"insta-science>=0.2.1",
|
|
60
|
-
"types-jsonschema>=4.23.0.20241208",
|
|
61
|
-
"toml>=0.10.2",
|
|
62
56
|
"sphinx-code-tabs>=0.5.5",
|
|
63
57
|
"sphinxcontrib-typer>=0.5.1",
|
|
64
58
|
"sphinxcontrib-video>=0.4.1",
|
|
65
59
|
]
|
|
60
|
+
mypy = [
|
|
61
|
+
"mypy>=1.12.1",
|
|
62
|
+
"pydantic ~= 2.9",
|
|
63
|
+
"types-docutils",
|
|
64
|
+
"types-Pygments",
|
|
65
|
+
"types-psutil>=7.0.0.20250218",
|
|
66
|
+
"types-jsonschema>=4.23.0.20241208",
|
|
67
|
+
]
|
|
68
|
+
ruff = ["ruff>=0.8.0"]
|
|
69
|
+
build = ["simple-build>=0.0.2", "shiv>=1.0.8"]
|
|
66
70
|
|
|
67
71
|
[project.scripts]
|
|
68
72
|
codelinks = "sphinx_codelinks.cmd:app"
|
|
69
73
|
|
|
70
|
-
[tool.rye.scripts]
|
|
71
|
-
# linting and formatting
|
|
72
|
-
"mypy:all" = "mypy ."
|
|
73
|
-
"rye:lint" = "rye lint"
|
|
74
|
-
"rye:format" = "rye format"
|
|
75
|
-
"check" = { chain = ["rye:format", "rye:lint", "mypy:all"] }
|
|
76
|
-
# docs html
|
|
77
|
-
"docs:rm" = "rm -rf docs/_build/html"
|
|
78
|
-
"docs" = "sphinx-build -nW --keep-going -b html -T -c docs docs/source docs/_build/html"
|
|
79
|
-
"docs:clean" = { chain = ["docs:rm", "docs"] }
|
|
80
|
-
# needextend demo
|
|
81
|
-
"analyse" = "codelinks analyse tests/data/configs/minimum_config.toml"
|
|
82
|
-
"analyse:rm" = "rm -rf output/marked_content.json"
|
|
83
|
-
"write" = "codelinks write rst output/marked_content.json --outpath tests/data/needextend_demo/needextend.rst"
|
|
84
|
-
"write:rm" = "rm -rf tests/data/needextend_demo/needextend.rst"
|
|
85
|
-
"demo:rm" = "rm -rf tests/data/needextend_demo/_build"
|
|
86
|
-
"demo:build" = "sphinx-build -nW --keep-going -b html -T -c tests/data/needextend_demo tests/data/needextend_demo tests/data/needextend_demo/_build/html"
|
|
87
|
-
"demo:clean" = { chain = [
|
|
88
|
-
"demo:rm",
|
|
89
|
-
"analyse:rm",
|
|
90
|
-
"write:rm",
|
|
91
|
-
"analyse",
|
|
92
|
-
"write",
|
|
93
|
-
"demo:build",
|
|
94
|
-
] }
|
|
95
|
-
|
|
96
74
|
[tool.ruff.lint]
|
|
97
75
|
extend-select = [
|
|
98
76
|
# "ANN",
|
|
@@ -131,15 +109,6 @@ force-sort-within-sections = true
|
|
|
131
109
|
"PLR2004", # magic-value-comparison - valueable for tests
|
|
132
110
|
"S101", # assert - needed for tests
|
|
133
111
|
]
|
|
134
|
-
"**/build_hooks_*/**" = [
|
|
135
|
-
"S607", # start-process-with-partial-path - pyarmor call in rye context
|
|
136
|
-
"S603", # subprocess-without-shell-equals-true - pyarmor call
|
|
137
|
-
]
|
|
138
|
-
"scripts/*.py" = [
|
|
139
|
-
"T201", # print - used for output
|
|
140
|
-
"S607", # start-process-with-partial-path - pyarmor call in rye context
|
|
141
|
-
"S603", # subprocess-without-shell-equals-true - build scripts
|
|
142
|
-
]
|
|
143
112
|
"src/sphinx_codelinks/sphinx_extension/debug.py" = [
|
|
144
113
|
"T201", # print - used for output
|
|
145
114
|
"UP047", # on-pep695-generic-function - it's generic
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
from collections.abc import Generator
|
|
2
2
|
from dataclasses import dataclass
|
|
3
3
|
import json
|
|
4
|
-
import logging
|
|
5
4
|
from pathlib import Path
|
|
6
5
|
from typing import Any, TypedDict
|
|
7
6
|
|
|
@@ -26,14 +25,14 @@ from sphinx_codelinks.config import (
|
|
|
26
25
|
OneLineCommentStyle,
|
|
27
26
|
SourceAnalyseConfig,
|
|
28
27
|
)
|
|
28
|
+
from sphinx_codelinks.logger import get_logger
|
|
29
29
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
logger.addHandler(console)
|
|
30
|
+
logger = get_logger(__name__)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _count(n: int, noun: str) -> str:
|
|
34
|
+
"""Format ``n noun`` with a naive (append-s) plural for progress summaries."""
|
|
35
|
+
return f"{n} {noun}" if n == 1 else f"{n} {noun}s"
|
|
37
36
|
|
|
38
37
|
|
|
39
38
|
class AnalyseWarningType(TypedDict):
|
|
@@ -57,7 +56,10 @@ class SourceAnalyse:
|
|
|
57
56
|
def __init__(
|
|
58
57
|
self,
|
|
59
58
|
analyse_config: SourceAnalyseConfig,
|
|
59
|
+
*,
|
|
60
|
+
name: str = "",
|
|
60
61
|
) -> None:
|
|
62
|
+
self.name = name
|
|
61
63
|
self.analyse_config = analyse_config
|
|
62
64
|
self.src_files: list[SourceFile] = []
|
|
63
65
|
self.src_comments: list[SourceComment] = []
|
|
@@ -65,7 +67,11 @@ class SourceAnalyse:
|
|
|
65
67
|
self.oneline_needs: list[OneLineNeed] = []
|
|
66
68
|
self.marked_rst: list[MarkedRst] = []
|
|
67
69
|
self.all_marked_content: list[NeedIdRefs | OneLineNeed | MarkedRst] = []
|
|
68
|
-
|
|
70
|
+
# Use explicitly configured git_root if provided, otherwise auto-detect
|
|
71
|
+
if self.analyse_config.git_root is not None:
|
|
72
|
+
self.git_root: Path | None = self.analyse_config.git_root.resolve()
|
|
73
|
+
else:
|
|
74
|
+
self.git_root = utils.locate_git_root(self.analyse_config.src_dir)
|
|
69
75
|
self.git_remote_url: str | None = (
|
|
70
76
|
utils.get_remote_url(self.git_root) if self.git_root else None
|
|
71
77
|
)
|
|
@@ -106,9 +112,6 @@ class SourceAnalyse:
|
|
|
106
112
|
self.src_files.append(src_file)
|
|
107
113
|
self.src_comments.extend(src_comments)
|
|
108
114
|
|
|
109
|
-
logger.info(f"Source files loaded: {len(self.src_files)}")
|
|
110
|
-
logger.info(f"Source comments extracted: {len(self.src_comments)}")
|
|
111
|
-
|
|
112
115
|
def extract_marker(
|
|
113
116
|
self,
|
|
114
117
|
text: str,
|
|
@@ -127,6 +130,7 @@ class SourceAnalyse:
|
|
|
127
130
|
yield marker, need_ids, row_offset, start_column, end_column
|
|
128
131
|
row_offset += 1
|
|
129
132
|
|
|
133
|
+
# @Extract need ID references from code comments, IMPL_LNK_1, impl, [FE_LNK]
|
|
130
134
|
def extract_anchors(
|
|
131
135
|
self,
|
|
132
136
|
text: str,
|
|
@@ -197,20 +201,21 @@ class SourceAnalyse:
|
|
|
197
201
|
if not src_comment.source_file:
|
|
198
202
|
row_offset += 1
|
|
199
203
|
continue
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
)
|
|
204
|
+
lineno = src_comment.node.start_point.row + row_offset + 1
|
|
205
|
+
warning = AnalyseWarning(
|
|
206
|
+
str(src_comment.source_file.filepath),
|
|
207
|
+
lineno,
|
|
208
|
+
resolved.msg,
|
|
209
|
+
MarkedContentType.need,
|
|
210
|
+
resolved.sub_type.value,
|
|
208
211
|
)
|
|
212
|
+
self.oneline_warnings.append(warning)
|
|
209
213
|
row_offset += 1
|
|
210
214
|
continue
|
|
211
215
|
yield resolved, row_offset
|
|
212
216
|
row_offset += 1
|
|
213
217
|
|
|
218
|
+
# @Extract one-line traceability needs from comments, IMPL_ONE_1, impl, [FE_DEF, FE_CMT]
|
|
214
219
|
def extract_oneline_needs(
|
|
215
220
|
self,
|
|
216
221
|
text: str,
|
|
@@ -258,6 +263,7 @@ class SourceAnalyse:
|
|
|
258
263
|
)
|
|
259
264
|
return oneline_needs
|
|
260
265
|
|
|
266
|
+
# @Extract marked reStructuredText blocks from comments, IMPL_MRST_1, impl, [FE_RST_EXTRACTION]
|
|
261
267
|
def extract_marked_rst(
|
|
262
268
|
self,
|
|
263
269
|
text: str,
|
|
@@ -346,13 +352,6 @@ class SourceAnalyse:
|
|
|
346
352
|
if marked_rst:
|
|
347
353
|
self.marked_rst.append(marked_rst)
|
|
348
354
|
|
|
349
|
-
if self.analyse_config.get_need_id_refs:
|
|
350
|
-
logger.info(f"Need-id-refs extracted: {len(self.need_id_refs)}")
|
|
351
|
-
if self.analyse_config.get_oneline_needs:
|
|
352
|
-
logger.info(f"Oneline needs extracted: {len(self.oneline_needs)}")
|
|
353
|
-
if self.analyse_config.get_rst:
|
|
354
|
-
logger.info(f"Marked rst extracted: {len(self.marked_rst)}")
|
|
355
|
-
|
|
356
355
|
def merge_marked_content(self) -> None:
|
|
357
356
|
self.all_marked_content.extend(self.need_id_refs)
|
|
358
357
|
self.oneline_needs.sort(key=lambda x: x.source_map["start"]["row"])
|
|
@@ -371,9 +370,23 @@ class SourceAnalyse:
|
|
|
371
370
|
]
|
|
372
371
|
with output_path.open("w") as f:
|
|
373
372
|
json.dump(to_dump, f)
|
|
374
|
-
logger.info(f"Marked content dumped to {output_path}")
|
|
375
373
|
|
|
376
374
|
def run(self) -> None:
|
|
377
375
|
self.create_src_objects()
|
|
378
376
|
self.extract_marked_content()
|
|
379
377
|
self.merge_marked_content()
|
|
378
|
+
self._log_summary()
|
|
379
|
+
|
|
380
|
+
def _log_summary(self) -> None:
|
|
381
|
+
"""Emit a per-project marker (default-visible) plus a -v breakdown."""
|
|
382
|
+
label = f"codelinks [{self.name}]" if self.name else "codelinks"
|
|
383
|
+
logger.info(
|
|
384
|
+
f"{label}: {_count(len(self.src_files), 'file')}, "
|
|
385
|
+
f"{_count(len(self.all_marked_content), 'marker')}"
|
|
386
|
+
)
|
|
387
|
+
logger.debug(
|
|
388
|
+
f"{label}: {_count(len(self.src_comments), 'comment')}, "
|
|
389
|
+
f"{_count(len(self.oneline_needs), 'oneline need')}, "
|
|
390
|
+
f"{_count(len(self.need_id_refs), 'id-ref')}, "
|
|
391
|
+
f"{_count(len(self.marked_rst), 'marked-rst block')}"
|
|
392
|
+
)
|
{sphinx_codelinks-1.1.0 → sphinx_codelinks-1.3.0}/src/sphinx_codelinks/analyse/oneline_parser.py
RENAMED
|
@@ -1,17 +1,8 @@
|
|
|
1
1
|
from dataclasses import dataclass
|
|
2
2
|
from enum import Enum
|
|
3
|
-
import logging
|
|
4
3
|
|
|
5
4
|
from sphinx_codelinks.config import ESCAPE, UNIX_NEWLINE, OneLineCommentStyle
|
|
6
5
|
|
|
7
|
-
# initialize logger
|
|
8
|
-
logger = logging.getLogger(__name__)
|
|
9
|
-
logger.setLevel(logging.INFO)
|
|
10
|
-
# log to the console
|
|
11
|
-
console = logging.StreamHandler()
|
|
12
|
-
console.setLevel(logging.INFO)
|
|
13
|
-
logger.addHandler(console)
|
|
14
|
-
|
|
15
6
|
|
|
16
7
|
class WarningSubTypeEnum(str, Enum):
|
|
17
8
|
"""Enum for warning sub types."""
|
|
@@ -31,6 +22,7 @@ class OnelineParserInvalidWarning:
|
|
|
31
22
|
msg: str
|
|
32
23
|
|
|
33
24
|
|
|
25
|
+
# @One-line comment parser for traceability markers, IMPL_OLP_1, impl, [FE_DEF, FE_CMT]
|
|
34
26
|
def oneline_parser( # noqa: PLR0912, PLR0911 # handel warnings
|
|
35
27
|
oneline: str, oneline_config: OneLineCommentStyle
|
|
36
28
|
) -> dict[str, str | list[str] | int] | OnelineParserInvalidWarning | None:
|
|
@@ -53,7 +45,7 @@ def oneline_parser( # noqa: PLR0912, PLR0911 # handel warnings
|
|
|
53
45
|
|
|
54
46
|
# extract the string wrapped by start and end
|
|
55
47
|
start_idx = start_idx + len(oneline_config.start_sequence)
|
|
56
|
-
string = oneline[start_idx:end_idx]
|
|
48
|
+
string = oneline[start_idx:end_idx].strip()
|
|
57
49
|
|
|
58
50
|
# numbers of needs_fields which are required
|
|
59
51
|
cnt_required_fields = oneline_config.get_cnt_required_fields()
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import json
|
|
2
|
-
import logging
|
|
3
2
|
from pathlib import Path
|
|
4
3
|
from typing import cast
|
|
5
4
|
|
|
@@ -9,14 +8,9 @@ from sphinx_codelinks.analyse.analyse import (
|
|
|
9
8
|
SourceAnalyse,
|
|
10
9
|
)
|
|
11
10
|
from sphinx_codelinks.config import CodeLinksConfig, CodeLinksProjectConfigType
|
|
11
|
+
from sphinx_codelinks.logger import get_logger
|
|
12
12
|
|
|
13
|
-
|
|
14
|
-
logger = logging.getLogger(__name__)
|
|
15
|
-
logger.setLevel(logging.INFO)
|
|
16
|
-
# log to the console
|
|
17
|
-
console = logging.StreamHandler()
|
|
18
|
-
console.setLevel(logging.INFO)
|
|
19
|
-
logger.addHandler(console)
|
|
13
|
+
logger = get_logger(__name__)
|
|
20
14
|
|
|
21
15
|
|
|
22
16
|
class AnalyseProjects:
|
|
@@ -32,7 +26,7 @@ class AnalyseProjects:
|
|
|
32
26
|
|
|
33
27
|
def run(self) -> None:
|
|
34
28
|
for project, config in self.projects_configs.items():
|
|
35
|
-
src_analyse = SourceAnalyse(config["analyse_config"])
|
|
29
|
+
src_analyse = SourceAnalyse(config["analyse_config"], name=project)
|
|
36
30
|
src_analyse.run()
|
|
37
31
|
self.projects_analyse[project] = src_analyse
|
|
38
32
|
|
|
@@ -46,7 +40,7 @@ class AnalyseProjects:
|
|
|
46
40
|
}
|
|
47
41
|
with output_path.open("w") as f:
|
|
48
42
|
json.dump(to_dump, f)
|
|
49
|
-
logger.
|
|
43
|
+
logger.debug(f"codelinks: marked content dumped to {output_path}")
|
|
50
44
|
|
|
51
45
|
@classmethod
|
|
52
46
|
def load_warnings(cls, warnings_dir: Path) -> list[AnalyseWarning] | None:
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
from collections.abc import ByteString, Callable
|
|
2
2
|
import configparser
|
|
3
|
-
import logging
|
|
4
3
|
from pathlib import Path
|
|
5
4
|
from typing import TypedDict
|
|
6
5
|
from urllib.request import pathname2url
|
|
@@ -10,23 +9,44 @@ from tree_sitter import Language, Parser, Point, Query, QueryCursor
|
|
|
10
9
|
from tree_sitter import Node as TreeSitterNode
|
|
11
10
|
|
|
12
11
|
from sphinx_codelinks.config import UNIX_NEWLINE, CommentCategory
|
|
12
|
+
from sphinx_codelinks.logger import get_logger
|
|
13
13
|
from sphinx_codelinks.source_discover.config import CommentType
|
|
14
14
|
|
|
15
|
-
# Language-specific node types for scope detection
|
|
15
|
+
# Language-specific node types for scope detection.
|
|
16
|
+
#
|
|
17
|
+
# YAML and JSONC are intentionally absent. They are data formats, not code, so a
|
|
18
|
+
# comment associates with the surrounding data structure (key/value pair, list
|
|
19
|
+
# item, or scalar) rather than with an enclosing or following declaration. That
|
|
20
|
+
# needs a different algorithm (inline same-row association first, scalar targets,
|
|
21
|
+
# grammar-specific traversal), implemented in find_yaml_associated_structure and
|
|
22
|
+
# find_jsonc_associated_structure and dispatched from find_associated_scope.
|
|
23
|
+
# Those bespoke finders never read this table (only find_next_scope and
|
|
24
|
+
# find_enclosing_scope do), so an entry here would be dead.
|
|
16
25
|
SCOPE_NODE_TYPES = {
|
|
26
|
+
# @Python Scope Node Types, IMPL_PY_2, impl, [FE_PY]
|
|
17
27
|
CommentType.python: {"function_definition", "class_definition"},
|
|
28
|
+
# @C and C++ Scope Node Types, IMPL_C_2, impl, [FE_C_SUPPORT, FE_CPP]
|
|
18
29
|
CommentType.cpp: {"function_definition", "class_definition"},
|
|
19
30
|
CommentType.cs: {"method_declaration", "class_declaration", "property_declaration"},
|
|
20
|
-
|
|
31
|
+
# @Rust Scope Node Types, IMPL_RUST_2, impl, [FE_RUST];
|
|
32
|
+
CommentType.rust: {
|
|
33
|
+
"function_item",
|
|
34
|
+
"struct_item",
|
|
35
|
+
"enum_item",
|
|
36
|
+
"impl_item",
|
|
37
|
+
"trait_item",
|
|
38
|
+
"mod_item",
|
|
39
|
+
},
|
|
40
|
+
# @Go Scope Node Types, IMPL_GO_4, impl, [FE_GO]
|
|
41
|
+
CommentType.go: {
|
|
42
|
+
"function_declaration",
|
|
43
|
+
"method_declaration",
|
|
44
|
+
"type_declaration",
|
|
45
|
+
"type_spec",
|
|
46
|
+
},
|
|
21
47
|
}
|
|
22
48
|
|
|
23
|
-
|
|
24
|
-
logger = logging.getLogger(__name__)
|
|
25
|
-
logger.setLevel(logging.INFO)
|
|
26
|
-
# log to the console
|
|
27
|
-
console = logging.StreamHandler()
|
|
28
|
-
console.setLevel(logging.INFO)
|
|
29
|
-
logger.addHandler(console)
|
|
49
|
+
logger = get_logger(__name__)
|
|
30
50
|
|
|
31
51
|
GIT_HOST_URL_TEMPLATE = {
|
|
32
52
|
"github": "https://github.com/{owner}/{repo}/blob/{rev}/{path}#L{lineno}",
|
|
@@ -45,6 +65,27 @@ PYTHON_QUERY = """
|
|
|
45
65
|
CPP_QUERY = """(comment) @comment"""
|
|
46
66
|
C_SHARP_QUERY = """(comment) @comment"""
|
|
47
67
|
YAML_QUERY = """(comment) @comment"""
|
|
68
|
+
RUST_QUERY = """
|
|
69
|
+
(line_comment) @comment
|
|
70
|
+
(block_comment) @comment
|
|
71
|
+
"""
|
|
72
|
+
# @Go comment query for tree-sitter, IMPL_GO_3, impl, [FE_GO]
|
|
73
|
+
GO_QUERY = """
|
|
74
|
+
(comment) @comment
|
|
75
|
+
"""
|
|
76
|
+
JSONC_QUERY = """(comment) @comment"""
|
|
77
|
+
|
|
78
|
+
# JSON value node types that can be associated with a comment.
|
|
79
|
+
JSON_STRUCTURE_TYPES = {
|
|
80
|
+
"pair",
|
|
81
|
+
"object",
|
|
82
|
+
"array",
|
|
83
|
+
"string",
|
|
84
|
+
"number",
|
|
85
|
+
"true",
|
|
86
|
+
"false",
|
|
87
|
+
"null",
|
|
88
|
+
}
|
|
48
89
|
|
|
49
90
|
|
|
50
91
|
def is_text_file(filepath: Path, sample_size: int = 2048) -> bool:
|
|
@@ -62,6 +103,7 @@ def is_text_file(filepath: Path, sample_size: int = 2048) -> bool:
|
|
|
62
103
|
return False
|
|
63
104
|
|
|
64
105
|
|
|
106
|
+
# @Tree-sitter parser initialization for multiple languages, IMPL_LANG_1, impl, [FE_C_SUPPORT, FE_CPP, FE_PY, FE_YAML, FE_RUST, FE_GO, FE_JSONC]
|
|
65
107
|
def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]:
|
|
66
108
|
if comment_type == CommentType.cpp:
|
|
67
109
|
import tree_sitter_cpp # noqa: PLC0415
|
|
@@ -83,6 +125,21 @@ def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]:
|
|
|
83
125
|
|
|
84
126
|
parsed_language = Language(tree_sitter_yaml.language())
|
|
85
127
|
query = Query(parsed_language, YAML_QUERY)
|
|
128
|
+
elif comment_type == CommentType.rust:
|
|
129
|
+
import tree_sitter_rust # noqa: PLC0415
|
|
130
|
+
|
|
131
|
+
parsed_language = Language(tree_sitter_rust.language())
|
|
132
|
+
query = Query(parsed_language, RUST_QUERY)
|
|
133
|
+
elif comment_type == CommentType.go:
|
|
134
|
+
import tree_sitter_go # noqa: PLC0415
|
|
135
|
+
|
|
136
|
+
parsed_language = Language(tree_sitter_go.language())
|
|
137
|
+
query = Query(parsed_language, GO_QUERY)
|
|
138
|
+
elif comment_type == CommentType.jsonc:
|
|
139
|
+
import tree_sitter_json # noqa: PLC0415
|
|
140
|
+
|
|
141
|
+
parsed_language = Language(tree_sitter_json.language())
|
|
142
|
+
query = Query(parsed_language, JSONC_QUERY)
|
|
86
143
|
else:
|
|
87
144
|
raise ValueError(f"Unsupported comment style: {comment_type}")
|
|
88
145
|
parser = Parser(parsed_language)
|
|
@@ -98,6 +155,7 @@ def wrap_read_callable_point(
|
|
|
98
155
|
return read_callable_byte_offset
|
|
99
156
|
|
|
100
157
|
|
|
158
|
+
# @Comment extraction from source code using tree-sitter, IMPL_EXTR_1, impl, [FE_DEF]
|
|
101
159
|
def extract_comments(
|
|
102
160
|
src_string: ByteString, parser: Parser, query: Query
|
|
103
161
|
) -> list[TreeSitterNode] | None:
|
|
@@ -181,8 +239,11 @@ def find_yaml_next_structure(node: TreeSitterNode) -> TreeSitterNode | None:
|
|
|
181
239
|
return None
|
|
182
240
|
|
|
183
241
|
|
|
184
|
-
def
|
|
185
|
-
"""Find a previous named sibling that is on the same row as the comment.
|
|
242
|
+
def find_prev_sibling_on_same_row(node: TreeSitterNode) -> TreeSitterNode | None:
|
|
243
|
+
"""Find a previous named sibling that is on the same row as the comment.
|
|
244
|
+
|
|
245
|
+
Grammar-agnostic: used to detect inline comments in both YAML and JSONC.
|
|
246
|
+
"""
|
|
186
247
|
comment_row = node.start_point.row
|
|
187
248
|
current = node.prev_named_sibling
|
|
188
249
|
|
|
@@ -203,7 +264,7 @@ def find_yaml_prev_sibling_on_same_row(node: TreeSitterNode) -> TreeSitterNode |
|
|
|
203
264
|
def find_yaml_associated_structure(node: TreeSitterNode) -> TreeSitterNode | None:
|
|
204
265
|
"""Find the YAML structure (key-value pair, list item, etc.) associated with a comment."""
|
|
205
266
|
# First, check if this is an inline comment by looking for a previous sibling on the same row
|
|
206
|
-
prev_sibling_same_row =
|
|
267
|
+
prev_sibling_same_row = find_prev_sibling_on_same_row(node)
|
|
207
268
|
if prev_sibling_same_row:
|
|
208
269
|
return prev_sibling_same_row
|
|
209
270
|
|
|
@@ -222,6 +283,36 @@ def find_yaml_associated_structure(node: TreeSitterNode) -> TreeSitterNode | Non
|
|
|
222
283
|
return None
|
|
223
284
|
|
|
224
285
|
|
|
286
|
+
# @JSONC comment-to-structure association, IMPL_JSONC_2, impl, [FE_JSONC]
|
|
287
|
+
def find_jsonc_associated_structure(node: TreeSitterNode) -> TreeSitterNode | None:
|
|
288
|
+
"""Find the JSON structure (key/value pair, value, list item) for a comment.
|
|
289
|
+
|
|
290
|
+
JSON is data rather than code, so association follows the same intent as YAML:
|
|
291
|
+
an inline comment belongs to the value on its row, a leading comment belongs to
|
|
292
|
+
the following structure, otherwise it belongs to the enclosing structure.
|
|
293
|
+
"""
|
|
294
|
+
# Inline comment: a value/pair on the same row, before the comment
|
|
295
|
+
prev_sibling_same_row = find_prev_sibling_on_same_row(node)
|
|
296
|
+
if prev_sibling_same_row:
|
|
297
|
+
return prev_sibling_same_row
|
|
298
|
+
|
|
299
|
+
# Leading comment: the next structure following the comment
|
|
300
|
+
current = node.next_named_sibling
|
|
301
|
+
while current:
|
|
302
|
+
if current.type in JSON_STRUCTURE_TYPES:
|
|
303
|
+
return current
|
|
304
|
+
current = current.next_named_sibling
|
|
305
|
+
|
|
306
|
+
# Otherwise: the enclosing structure
|
|
307
|
+
parent = node.parent
|
|
308
|
+
while parent:
|
|
309
|
+
if parent.type in {"pair", "object", "array"}:
|
|
310
|
+
return parent
|
|
311
|
+
parent = parent.parent
|
|
312
|
+
|
|
313
|
+
return None
|
|
314
|
+
|
|
315
|
+
|
|
225
316
|
def find_associated_scope(
|
|
226
317
|
node: TreeSitterNode, comment_type: CommentType = CommentType.cpp
|
|
227
318
|
) -> TreeSitterNode | None:
|
|
@@ -230,6 +321,10 @@ def find_associated_scope(
|
|
|
230
321
|
# YAML uses different structure association logic
|
|
231
322
|
return find_yaml_associated_structure(node)
|
|
232
323
|
|
|
324
|
+
if comment_type == CommentType.jsonc:
|
|
325
|
+
# JSONC uses data-aware structure association logic
|
|
326
|
+
return find_jsonc_associated_structure(node)
|
|
327
|
+
|
|
233
328
|
if node.type == CommentCategory.docstring:
|
|
234
329
|
# Only for python's docstring
|
|
235
330
|
return find_enclosing_scope(node, comment_type)
|
|
@@ -248,7 +343,11 @@ def locate_git_root(src_dir: Path) -> Path | None:
|
|
|
248
343
|
for parent in parents:
|
|
249
344
|
if (parent / ".git").exists() and (parent / ".git").is_dir():
|
|
250
345
|
return parent
|
|
251
|
-
logger.warning(
|
|
346
|
+
logger.warning(
|
|
347
|
+
f"git root is not found in the parent of {src_dir}",
|
|
348
|
+
subtype="git_root",
|
|
349
|
+
location=str(src_dir),
|
|
350
|
+
)
|
|
252
351
|
return None
|
|
253
352
|
|
|
254
353
|
|
|
@@ -256,7 +355,11 @@ def get_remote_url(git_root: Path, remote_name: str = "origin") -> str | None:
|
|
|
256
355
|
"""Get remote url from .git/config."""
|
|
257
356
|
config_path = git_root / ".git" / "config"
|
|
258
357
|
if not config_path.exists():
|
|
259
|
-
|
|
358
|
+
logger.warning(
|
|
359
|
+
f"{config_path} does not exist",
|
|
360
|
+
subtype="git_config",
|
|
361
|
+
location=str(config_path),
|
|
362
|
+
)
|
|
260
363
|
return None
|
|
261
364
|
|
|
262
365
|
config = configparser.ConfigParser(allow_no_value=True, strict=False)
|
|
@@ -265,7 +368,11 @@ def get_remote_url(git_root: Path, remote_name: str = "origin") -> str | None:
|
|
|
265
368
|
if section in config and "url" in config[section]:
|
|
266
369
|
url: str = config[section]["url"]
|
|
267
370
|
return url
|
|
268
|
-
logger.warning(
|
|
371
|
+
logger.warning(
|
|
372
|
+
f"remote-url is not found in {config_path}",
|
|
373
|
+
subtype="git_remote",
|
|
374
|
+
location=str(config_path),
|
|
375
|
+
)
|
|
269
376
|
return None
|
|
270
377
|
|
|
271
378
|
|
|
@@ -273,16 +380,25 @@ def get_current_rev(git_root: Path) -> str | None:
|
|
|
273
380
|
"""Get current commit rev from .git/HEAD."""
|
|
274
381
|
head_path = git_root / ".git" / "HEAD"
|
|
275
382
|
if not head_path.exists():
|
|
276
|
-
|
|
383
|
+
logger.warning(
|
|
384
|
+
f"{head_path} does not exist",
|
|
385
|
+
subtype="git_head",
|
|
386
|
+
location=str(head_path),
|
|
387
|
+
)
|
|
277
388
|
return None
|
|
278
389
|
head_content = head_path.read_text().strip()
|
|
279
390
|
if not head_content.startswith("ref: "):
|
|
280
|
-
|
|
281
|
-
|
|
391
|
+
# Detached HEAD (e.g. CI checkouts): .git/HEAD holds the commit SHA
|
|
392
|
+
# directly, which is exactly the rev we want.
|
|
393
|
+
return head_content
|
|
282
394
|
|
|
283
395
|
ref_path = git_root / ".git" / head_content.split(":", 1)[1].strip()
|
|
284
396
|
if not ref_path.exists():
|
|
285
|
-
|
|
397
|
+
logger.warning(
|
|
398
|
+
f"{ref_path} does not exist",
|
|
399
|
+
subtype="git_ref",
|
|
400
|
+
location=str(ref_path),
|
|
401
|
+
)
|
|
286
402
|
return None
|
|
287
403
|
return ref_path.read_text().strip()
|
|
288
404
|
|
|
@@ -293,7 +409,10 @@ def form_https_url(
|
|
|
293
409
|
parsed_url = parse(git_url)
|
|
294
410
|
template = GIT_HOST_URL_TEMPLATE.get(parsed_url.platform)
|
|
295
411
|
if not template:
|
|
296
|
-
|
|
412
|
+
logger.warning(
|
|
413
|
+
f"Unsupported Git host: {parsed_url.platform}",
|
|
414
|
+
subtype="git_host",
|
|
415
|
+
)
|
|
297
416
|
return git_url
|
|
298
417
|
https_url = template.format(
|
|
299
418
|
owner=parsed_url.owner,
|
|
@@ -333,6 +452,7 @@ class ExtractedRstType(TypedDict):
|
|
|
333
452
|
end_idx: int
|
|
334
453
|
|
|
335
454
|
|
|
455
|
+
# @Extract reStructuredText blocks embedded in comments, IMPL_RST_1, impl, [FE_RST_EXTRACTION]
|
|
336
456
|
def extract_rst(
|
|
337
457
|
text: str, start_marker: str, end_marker: str
|
|
338
458
|
) -> ExtractedRstType | None:
|