etki 0.1.0a2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etki/__init__.py +3 -0
- etki/adapters/__init__.py +0 -0
- etki/adapters/ast_code_index.py +131 -0
- etki/adapters/azure_devops_work_item.py +101 -0
- etki/adapters/code_index.py +246 -0
- etki/adapters/composite_document.py +44 -0
- etki/adapters/confluence_document.py +113 -0
- etki/adapters/embedding_openai.py +48 -0
- etki/adapters/fakes/__init__.py +0 -0
- etki/adapters/fakes/code_repo.py +42 -0
- etki/adapters/fakes/document.py +30 -0
- etki/adapters/fakes/seed.py +156 -0
- etki/adapters/fakes/work_item.py +40 -0
- etki/adapters/file_work_item.py +65 -0
- etki/adapters/filesystem_document.py +50 -0
- etki/adapters/filesystem_wiki.py +438 -0
- etki/adapters/git_churn.py +60 -0
- etki/adapters/git_clone.py +80 -0
- etki/adapters/gitlab_work_item.py +103 -0
- etki/adapters/glpi_work_item.py +129 -0
- etki/adapters/graphify_code_repo.py +221 -0
- etki/adapters/jira_work_item.py +82 -0
- etki/adapters/joern_code_repo.py +65 -0
- etki/adapters/linear_work_item.py +108 -0
- etki/adapters/llm_anthropic.py +61 -0
- etki/adapters/llm_openai.py +49 -0
- etki/adapters/manifests.py +237 -0
- etki/adapters/package_download.py +399 -0
- etki/adapters/package_registries.py +251 -0
- etki/adapters/redmine_work_item.py +89 -0
- etki/adapters/registry.py +269 -0
- etki/adapters/rerank_tei.py +41 -0
- etki/adapters/sharepoint_document.py +125 -0
- etki/agent.py +260 -0
- etki/api/__init__.py +0 -0
- etki/api/app.py +296 -0
- etki/api/context.py +356 -0
- etki/api/schemas.py +19 -0
- etki/api/security.py +117 -0
- etki/api/static/app.css +342 -0
- etki/api/static/htmx.min.js +1 -0
- etki/api/static/pmo.css +286 -0
- etki/api/templates/analyze_results.html +34 -0
- etki/api/templates/ara.html +29 -0
- etki/api/templates/ask_llm_result.html +10 -0
- etki/api/templates/ask_turn.html +55 -0
- etki/api/templates/ayarlar.html +152 -0
- etki/api/templates/baseline_timeline.html +52 -0
- etki/api/templates/cards.html +18 -0
- etki/api/templates/case_chat.html +6 -0
- etki/api/templates/case_flow.html +39 -0
- etki/api/templates/casefile.html +11 -0
- etki/api/templates/casefile_body.html +88 -0
- etki/api/templates/clause_detail.html +59 -0
- etki/api/templates/deps_diff.html +84 -0
- etki/api/templates/deps_online.html +13 -0
- etki/api/templates/document_preview.html +13 -0
- etki/api/templates/index_history.html +13 -0
- etki/api/templates/login.html +98 -0
- etki/api/templates/macros.html +159 -0
- etki/api/templates/pmo_base.html +288 -0
- etki/api/templates/pool_breakdown.html +16 -0
- etki/api/templates/pre_analysis_saved.html +2 -0
- etki/api/templates/project_ask.html +27 -0
- etki/api/templates/project_detail.html +274 -0
- etki/api/templates/project_files.html +123 -0
- etki/api/templates/project_hafiza.html +62 -0
- etki/api/templates/project_history.html +37 -0
- etki/api/templates/project_modules.html +57 -0
- etki/api/templates/project_sankey.html +35 -0
- etki/api/templates/project_triyaj.html +58 -0
- etki/api/templates/projects.html +70 -0
- etki/api/templates/triage_result.html +62 -0
- etki/api/templates/yeni_proje.html +31 -0
- etki/api/web.py +2258 -0
- etki/auth.py +236 -0
- etki/config.py +303 -0
- etki/core/__init__.py +0 -0
- etki/core/enums.py +55 -0
- etki/core/models.py +269 -0
- etki/core/ports.py +212 -0
- etki/core/text.py +140 -0
- etki/domains.py +69 -0
- etki/engine/__init__.py +0 -0
- etki/engine/estimation.py +141 -0
- etki/engine/triage.py +1316 -0
- etki/engine/understanding.py +249 -0
- etki/extraction/__init__.py +0 -0
- etki/extraction/parsers.py +85 -0
- etki/extraction/scope_extractor.py +257 -0
- etki/graphquery.py +382 -0
- etki/hitl/__init__.py +0 -0
- etki/hitl/ingest.py +160 -0
- etki/hitl/service.py +197 -0
- etki/i18n/__init__.py +67 -0
- etki/i18n/catalog.py +1907 -0
- etki/index_tools.py +266 -0
- etki/indexing/__init__.py +0 -0
- etki/indexing/__main__.py +45 -0
- etki/indexing/engine.py +95 -0
- etki/kpi.py +92 -0
- etki/llm_profile.py +66 -0
- etki/llm_settings.py +65 -0
- etki/logging_config.py +26 -0
- etki/mcp_server.py +201 -0
- etki/persistence/__init__.py +0 -0
- etki/persistence/__main__.py +71 -0
- etki/persistence/db.py +42 -0
- etki/persistence/memory_repo.py +62 -0
- etki/persistence/models.py +79 -0
- etki/persistence/repository.py +156 -0
- etki/pilot/__init__.py +0 -0
- etki/pilot/__main__.py +97 -0
- etki/pilot/calibration.py +73 -0
- etki/pilot/shadow.py +77 -0
- etki/process_log.py +65 -0
- etki/projects_store.py +220 -0
- etki/reporting/__init__.py +0 -0
- etki/reporting/docx_report.py +66 -0
- etki/wiki/__init__.py +54 -0
- etki/wiki/__main__.py +75 -0
- etki-0.1.0a2.dist-info/METADATA +245 -0
- etki-0.1.0a2.dist-info/RECORD +125 -0
- etki-0.1.0a2.dist-info/WHEEL +4 -0
- etki-0.1.0a2.dist-info/licenses/LICENSE +201 -0
etki/__init__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""Dependency-free Python-AST code indexer (CodeRepositoryProvider).
|
|
2
|
+
|
|
3
|
+
Real static analysis via the stdlib `ast`: imports → dependencies, functions +
|
|
4
|
+
control structures → complexity. Produces the same normalized schema as Joern;
|
|
5
|
+
lets the demo run without Joern too (for Python sources only).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import ast
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from etki.adapters.code_index import CodeIndex, FileNode, IndexBackedCodeRepository
|
|
14
|
+
from etki.adapters.manifests import parse_manifests
|
|
15
|
+
|
|
16
|
+
_CONTROL = (ast.If, ast.For, ast.AsyncFor, ast.While, ast.Try, ast.With, ast.AsyncWith)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _imports(tree: ast.AST) -> list[str]:
|
|
20
|
+
names: set[str] = set()
|
|
21
|
+
for node in ast.walk(tree):
|
|
22
|
+
if isinstance(node, ast.Import):
|
|
23
|
+
for alias in node.names:
|
|
24
|
+
names.add(alias.name.split(".")[0])
|
|
25
|
+
elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0:
|
|
26
|
+
names.add(node.module.split(".")[0])
|
|
27
|
+
return sorted(names)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _api_uses(tree: ast.AST) -> dict[str, list[str]]:
|
|
31
|
+
"""Which SYMBOLS of each imported package the file actually touches — the
|
|
32
|
+
call-site surface a version change must be audited against.
|
|
33
|
+
|
|
34
|
+
Two capture paths: `from pkg import name` (the name itself) and attribute
|
|
35
|
+
access on an imported alias (`requests.get`, `np.array` → numpy.array).
|
|
36
|
+
Keys are TOP-LEVEL package names (alias-resolved); internal/stdlib keys are
|
|
37
|
+
filtered later in parse_code_index, same as plain imports."""
|
|
38
|
+
alias_to_pkg: dict[str, str] = {}
|
|
39
|
+
uses: dict[str, set[str]] = {}
|
|
40
|
+
for node in ast.walk(tree):
|
|
41
|
+
if isinstance(node, ast.Import):
|
|
42
|
+
for alias in node.names:
|
|
43
|
+
top = alias.name.split(".")[0]
|
|
44
|
+
alias_to_pkg[(alias.asname or alias.name).split(".")[0]] = top
|
|
45
|
+
elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0:
|
|
46
|
+
top = node.module.split(".")[0]
|
|
47
|
+
for alias in node.names:
|
|
48
|
+
if alias.name != "*":
|
|
49
|
+
uses.setdefault(top, set()).add(alias.name)
|
|
50
|
+
for node in ast.walk(tree):
|
|
51
|
+
if (
|
|
52
|
+
isinstance(node, ast.Attribute)
|
|
53
|
+
and isinstance(node.value, ast.Name)
|
|
54
|
+
and node.value.id in alias_to_pkg
|
|
55
|
+
):
|
|
56
|
+
uses.setdefault(alias_to_pkg[node.value.id], set()).add(node.attr)
|
|
57
|
+
return {pkg: sorted(symbols) for pkg, symbols in sorted(uses.items())}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _dotted_chain(node: ast.Attribute) -> list[str] | None:
|
|
61
|
+
"""`np.linalg.norm` → ["np", "linalg", "norm"]; None if the base isn't a Name."""
|
|
62
|
+
parts: list[str] = []
|
|
63
|
+
current: ast.expr = node
|
|
64
|
+
while isinstance(current, ast.Attribute):
|
|
65
|
+
parts.append(current.attr)
|
|
66
|
+
current = current.value
|
|
67
|
+
if isinstance(current, ast.Name):
|
|
68
|
+
parts.append(current.id)
|
|
69
|
+
return list(reversed(parts))
|
|
70
|
+
return None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _api_paths(tree: ast.AST) -> dict[str, list[str]]:
|
|
74
|
+
"""QUALIFIED use paths per package — the precise audit input for version
|
|
75
|
+
diffs (`from faker.providers.credit_card import CreditCard` →
|
|
76
|
+
"faker.providers.credit_card.CreditCard"; `np.linalg.norm` →
|
|
77
|
+
"numpy.linalg.norm", alias-resolved). Keys = top-level package; only
|
|
78
|
+
absolute imports (level == 0) — relative imports are internal."""
|
|
79
|
+
alias_to_module: dict[str, str] = {}
|
|
80
|
+
paths: dict[str, set[str]] = {}
|
|
81
|
+
for node in ast.walk(tree):
|
|
82
|
+
if isinstance(node, ast.Import):
|
|
83
|
+
for alias in node.names:
|
|
84
|
+
alias_to_module[(alias.asname or alias.name).split(".")[0]] = (
|
|
85
|
+
alias.name if alias.asname else alias.name.split(".")[0]
|
|
86
|
+
)
|
|
87
|
+
elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0:
|
|
88
|
+
top = node.module.split(".")[0]
|
|
89
|
+
for alias in node.names:
|
|
90
|
+
if alias.name != "*": # original name, never the asname
|
|
91
|
+
paths.setdefault(top, set()).add(f"{node.module}.{alias.name}")
|
|
92
|
+
for node in ast.walk(tree):
|
|
93
|
+
if isinstance(node, ast.Attribute):
|
|
94
|
+
chain = _dotted_chain(node)
|
|
95
|
+
if chain and chain[0] in alias_to_module:
|
|
96
|
+
resolved = [*alias_to_module[chain[0]].split("."), *chain[1:]]
|
|
97
|
+
paths.setdefault(resolved[0], set()).add(".".join(resolved))
|
|
98
|
+
return {pkg: sorted(qualified) for pkg, qualified in sorted(paths.items())}
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def build_ast_index(src_root: str | Path) -> CodeIndex:
|
|
102
|
+
root = Path(src_root)
|
|
103
|
+
files: list[FileNode] = []
|
|
104
|
+
for py in sorted(root.rglob("*.py")):
|
|
105
|
+
source = py.read_text(encoding="utf-8")
|
|
106
|
+
tree = ast.parse(source)
|
|
107
|
+
functions = [
|
|
108
|
+
node.name
|
|
109
|
+
for node in ast.walk(tree)
|
|
110
|
+
if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef)
|
|
111
|
+
]
|
|
112
|
+
control = sum(1 for node in ast.walk(tree) if isinstance(node, _CONTROL))
|
|
113
|
+
files.append(
|
|
114
|
+
FileNode(
|
|
115
|
+
path=str(py.relative_to(root)),
|
|
116
|
+
loc=len(source.splitlines()),
|
|
117
|
+
functions=functions,
|
|
118
|
+
control_structures=control,
|
|
119
|
+
imports=_imports(tree),
|
|
120
|
+
api_uses=_api_uses(tree),
|
|
121
|
+
api_paths=_api_paths(tree),
|
|
122
|
+
)
|
|
123
|
+
)
|
|
124
|
+
return CodeIndex(
|
|
125
|
+
root=str(root), producer="ast", files=files, dependencies=parse_manifests(root)
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class AstCodeRepositoryProvider(IndexBackedCodeRepository):
|
|
130
|
+
def __init__(self, src_root: str | Path, churn: dict[str, int] | None = None) -> None:
|
|
131
|
+
super().__init__(build_ast_index(src_root), churn, supports_incremental_diff=False)
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""Azure DevOps (Boards) WorkItemProvider — REST 7.x, PAT auth.
|
|
2
|
+
|
|
3
|
+
Effort comes from `Microsoft.VSTS.Scheduling.CompletedWork` (float HOURS on
|
|
4
|
+
task-level items) → normalized to `WorkItem.effort_seconds` inside the adapter.
|
|
5
|
+
Similar-item search runs a WIQL query over title/description, then batch-fetches
|
|
6
|
+
fields. Requires a live organization → integration is CI-skipped; pure parsing is
|
|
7
|
+
unit-tested. Config example:
|
|
8
|
+
|
|
9
|
+
connectors:
|
|
10
|
+
work_items:
|
|
11
|
+
adapter: azure_devops
|
|
12
|
+
options:
|
|
13
|
+
organization: myorg # dev.azure.com/myorg
|
|
14
|
+
project: MyProject
|
|
15
|
+
pat: env:AZDO_PAT # scope: Work Items (Read)
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import base64
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
import httpx
|
|
24
|
+
|
|
25
|
+
from etki.core.models import WorkItem
|
|
26
|
+
from etki.core.ports import Capabilities
|
|
27
|
+
|
|
28
|
+
_FIELDS = (
|
|
29
|
+
"System.Title,System.State,System.WorkItemType,System.Description,"
|
|
30
|
+
"Microsoft.VSTS.Scheduling.CompletedWork"
|
|
31
|
+
)
|
|
32
|
+
_API = "api-version=7.1"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class AzureDevOpsWorkItemProvider:
|
|
36
|
+
def __init__(
|
|
37
|
+
self, organization: str, project: str, pat: str, timeout: float = 30.0
|
|
38
|
+
) -> None:
|
|
39
|
+
self._base_url = f"https://dev.azure.com/{organization}/{project}"
|
|
40
|
+
token = base64.b64encode(f":{pat}".encode()).decode() # PAT: empty user + token
|
|
41
|
+
self._auth = f"Basic {token}"
|
|
42
|
+
self._timeout = timeout
|
|
43
|
+
|
|
44
|
+
def _headers(self) -> dict[str, str]:
|
|
45
|
+
return {"Authorization": self._auth, "Content-Type": "application/json"}
|
|
46
|
+
|
|
47
|
+
def _to_work_item(self, raw: dict[str, Any]) -> WorkItem:
|
|
48
|
+
fields = raw.get("fields") or {}
|
|
49
|
+
completed_hours = float(
|
|
50
|
+
fields.get("Microsoft.VSTS.Scheduling.CompletedWork") or 0.0
|
|
51
|
+
)
|
|
52
|
+
return WorkItem(
|
|
53
|
+
id=str(raw.get("id", "?")),
|
|
54
|
+
title=fields.get("System.Title", "") or "",
|
|
55
|
+
description=str(fields.get("System.Description") or ""),
|
|
56
|
+
category=str(fields.get("System.WorkItemType") or "") or None,
|
|
57
|
+
status=str(fields.get("System.State") or ""),
|
|
58
|
+
effort_seconds=int(completed_hours * 3600),
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
async def get_work_item(self, item_id: str) -> WorkItem:
|
|
62
|
+
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
|
63
|
+
response = await client.get(
|
|
64
|
+
f"{self._base_url}/_apis/wit/workitems/{item_id}?fields={_FIELDS}&{_API}",
|
|
65
|
+
headers=self._headers(),
|
|
66
|
+
)
|
|
67
|
+
response.raise_for_status()
|
|
68
|
+
return self._to_work_item(response.json())
|
|
69
|
+
|
|
70
|
+
async def find_similar(self, description: str, *, limit: int = 5) -> list[WorkItem]:
|
|
71
|
+
term = description.replace("'", " ").strip()[:200]
|
|
72
|
+
wiql = (
|
|
73
|
+
"SELECT [System.Id] FROM WorkItems "
|
|
74
|
+
f"WHERE [System.Title] CONTAINS '{term}' "
|
|
75
|
+
"ORDER BY [System.ChangedDate] DESC"
|
|
76
|
+
)
|
|
77
|
+
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
|
78
|
+
response = await client.post(
|
|
79
|
+
f"{self._base_url}/_apis/wit/wiql?$top={limit}&{_API}",
|
|
80
|
+
headers=self._headers(),
|
|
81
|
+
json={"query": wiql},
|
|
82
|
+
)
|
|
83
|
+
response.raise_for_status()
|
|
84
|
+
ids = [str(w["id"]) for w in response.json().get("workItems", [])][:limit]
|
|
85
|
+
if not ids:
|
|
86
|
+
return []
|
|
87
|
+
batch = await client.get(
|
|
88
|
+
f"{self._base_url}/_apis/wit/workitems?ids={','.join(ids)}"
|
|
89
|
+
f"&fields={_FIELDS}&{_API}",
|
|
90
|
+
headers=self._headers(),
|
|
91
|
+
)
|
|
92
|
+
batch.raise_for_status()
|
|
93
|
+
return [self._to_work_item(w) for w in batch.json().get("value", [])]
|
|
94
|
+
|
|
95
|
+
def capabilities(self) -> Capabilities:
|
|
96
|
+
return Capabilities(
|
|
97
|
+
supports_webhooks=True, # service hooks
|
|
98
|
+
supports_realtime=False,
|
|
99
|
+
supports_effort_tracking=True, # CompletedWork (hours → seconds here)
|
|
100
|
+
supports_incremental_diff=False,
|
|
101
|
+
)
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
"""Normalized 'code index' schema — the contract between any indexer (Joern, AST...)
|
|
2
|
+
and the core. Pure and testable: the indexer produces this schema,
|
|
3
|
+
`parse_code_index` converts it into a `CodeModule` graph.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from pathlib import PurePosixPath
|
|
9
|
+
|
|
10
|
+
from pydantic import BaseModel, Field
|
|
11
|
+
|
|
12
|
+
from etki.adapters.manifests import NOISE_IMPORTS
|
|
13
|
+
from etki.core.models import DeclaredDependency
|
|
14
|
+
from etki_api import Capabilities, Churn, CodeModule, CodeRepositoryProvider, Complexity
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class FileNode(BaseModel):
|
|
18
|
+
path: str # root-relative, e.g. "auth/service.py"
|
|
19
|
+
loc: int = 0
|
|
20
|
+
functions: list[str] = Field(default_factory=list)
|
|
21
|
+
control_structures: int = 0
|
|
22
|
+
imports: list[str] = Field(default_factory=list) # top-level imported names
|
|
23
|
+
# Per-package symbols the file touches (`requests` → ["get", "post"]) — the
|
|
24
|
+
# call-site surface for API-change checks. Filled by the ast producer;
|
|
25
|
+
# joern/graphify leave it empty (defaulted → schema stays compatible).
|
|
26
|
+
api_uses: dict[str, list[str]] = Field(default_factory=dict)
|
|
27
|
+
# QUALIFIED use paths ("faker" → ["faker.providers.credit_card.CreditCard"])
|
|
28
|
+
# — precise version-diff auditing incl. non-exported imports. ast only.
|
|
29
|
+
api_paths: dict[str, list[str]] = Field(default_factory=dict)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class CodeIndex(BaseModel):
|
|
33
|
+
root: str
|
|
34
|
+
producer: str = "unknown" # "joern" | "ast"
|
|
35
|
+
files: list[FileNode] = Field(default_factory=list)
|
|
36
|
+
# Declared manifest dependencies (filled by parse_manifests; defaulted so
|
|
37
|
+
# existing producers/JSON stay valid).
|
|
38
|
+
dependencies: list[DeclaredDependency] = Field(default_factory=list)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _module_of(path: str) -> str:
|
|
42
|
+
parts = PurePosixPath(path.replace("\\", "/")).parts
|
|
43
|
+
return parts[0] if parts else path
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def parse_code_index(index: CodeIndex) -> list[CodeModule]:
|
|
47
|
+
"""Groups files into modules by top-level directory; builds the dependency
|
|
48
|
+
graph from imports and complexity from the metrics."""
|
|
49
|
+
groups: dict[str, list[FileNode]] = {}
|
|
50
|
+
for node in index.files:
|
|
51
|
+
groups.setdefault(_module_of(node.path), []).append(node)
|
|
52
|
+
module_ids = set(groups)
|
|
53
|
+
|
|
54
|
+
modules: list[CodeModule] = []
|
|
55
|
+
dep_map: dict[str, list[str]] = {}
|
|
56
|
+
for module_id, files in groups.items():
|
|
57
|
+
loc = sum(f.loc for f in files)
|
|
58
|
+
functions = sorted({fn for f in files for fn in f.functions})
|
|
59
|
+
control = sum(f.control_structures for f in files)
|
|
60
|
+
imports = {imp for f in files for imp in f.imports}
|
|
61
|
+
depends_on = sorted((imports & module_ids) - {module_id})
|
|
62
|
+
# The complement of the internal-module intersection is the EXTERNAL
|
|
63
|
+
# usage surface (third-party packages) — previously discarded here.
|
|
64
|
+
packages = sorted(imports - module_ids - {module_id} - NOISE_IMPORTS)
|
|
65
|
+
package_apis: dict[str, set[str]] = {}
|
|
66
|
+
package_api_paths: dict[str, set[str]] = {}
|
|
67
|
+
for f in files:
|
|
68
|
+
for pkg, symbols in f.api_uses.items():
|
|
69
|
+
if pkg in packages: # external only — internal/stdlib filtered
|
|
70
|
+
package_apis.setdefault(pkg, set()).update(symbols)
|
|
71
|
+
for pkg, qualified in f.api_paths.items():
|
|
72
|
+
if pkg in packages:
|
|
73
|
+
package_api_paths.setdefault(pkg, set()).update(qualified)
|
|
74
|
+
dep_map[module_id] = depends_on
|
|
75
|
+
complexity = Complexity(loc=loc, cyclomatic=control + len(functions), files=len(files))
|
|
76
|
+
modules.append(
|
|
77
|
+
CodeModule(
|
|
78
|
+
id=module_id,
|
|
79
|
+
path=f"{module_id}/",
|
|
80
|
+
responsibilities=functions,
|
|
81
|
+
depends_on=depends_on,
|
|
82
|
+
packages=packages,
|
|
83
|
+
package_apis={p: sorted(s) for p, s in sorted(package_apis.items())},
|
|
84
|
+
package_api_paths={
|
|
85
|
+
p: sorted(s) for p, s in sorted(package_api_paths.items())
|
|
86
|
+
},
|
|
87
|
+
complexity=complexity,
|
|
88
|
+
churn=Churn(),
|
|
89
|
+
)
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
by_id = {m.id: m for m in modules}
|
|
93
|
+
for module_id, deps in dep_map.items():
|
|
94
|
+
for dep in deps:
|
|
95
|
+
if dep in by_id:
|
|
96
|
+
by_id[dep].depended_by.append(module_id)
|
|
97
|
+
for module in modules:
|
|
98
|
+
module.depended_by.sort()
|
|
99
|
+
return sorted(modules, key=lambda m: m.id)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def impacted_modules(
|
|
103
|
+
modules: list[CodeModule], module_hint: str | None, *, depth: int = 1
|
|
104
|
+
) -> list[CodeModule]:
|
|
105
|
+
"""Module(s) matching the hint + ``depth`` degrees of dependency propagation (BFS)."""
|
|
106
|
+
if not module_hint:
|
|
107
|
+
return []
|
|
108
|
+
hint = module_hint.lower()
|
|
109
|
+
by_id = {m.id: m for m in modules}
|
|
110
|
+
ordered: list[str] = [
|
|
111
|
+
m.id
|
|
112
|
+
for m in modules
|
|
113
|
+
if hint in m.id.lower() or any(hint in r.lower() for r in m.responsibilities)
|
|
114
|
+
]
|
|
115
|
+
seen = set(ordered)
|
|
116
|
+
frontier = list(ordered)
|
|
117
|
+
for _ in range(max(0, depth)):
|
|
118
|
+
nxt: list[str] = []
|
|
119
|
+
for mid in frontier:
|
|
120
|
+
module = by_id.get(mid)
|
|
121
|
+
if module is None:
|
|
122
|
+
continue
|
|
123
|
+
for dep in (*module.depends_on, *module.depended_by):
|
|
124
|
+
if dep in by_id and dep not in seen:
|
|
125
|
+
seen.add(dep)
|
|
126
|
+
ordered.append(dep)
|
|
127
|
+
nxt.append(dep)
|
|
128
|
+
frontier = nxt
|
|
129
|
+
return [by_id[mid] for mid in ordered]
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class StaticCodeRepository:
|
|
133
|
+
"""Online CodeRepositoryProvider serving a precomputed module graph
|
|
134
|
+
(read from the persisted Index — no Joern run at triage time)."""
|
|
135
|
+
|
|
136
|
+
def __init__(
|
|
137
|
+
self,
|
|
138
|
+
modules: list[CodeModule],
|
|
139
|
+
dependencies: list[DeclaredDependency] | None = None,
|
|
140
|
+
) -> None:
|
|
141
|
+
self._modules = list(modules)
|
|
142
|
+
self._dependencies = list(dependencies or [])
|
|
143
|
+
|
|
144
|
+
async def list_modules(self) -> list[CodeModule]:
|
|
145
|
+
return list(self._modules)
|
|
146
|
+
|
|
147
|
+
def list_dependencies(self) -> list[DeclaredDependency]:
|
|
148
|
+
# Structural degradation seam (hasattr, like WorkItemProvider.all_items):
|
|
149
|
+
# the CodeRepositoryProvider Protocol stays untouched.
|
|
150
|
+
return list(self._dependencies)
|
|
151
|
+
|
|
152
|
+
async def get_impacted(self, module_hint: str | None) -> list[CodeModule]:
|
|
153
|
+
# depth=1 already yields the seed + its direct dependency/dependent neighbors;
|
|
154
|
+
# preserves risk separation on small graphs (deeper propagation is possible via BFS).
|
|
155
|
+
return impacted_modules(self._modules, module_hint, depth=1)
|
|
156
|
+
|
|
157
|
+
def capabilities(self) -> Capabilities:
|
|
158
|
+
return Capabilities(supports_incremental_diff=True)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class MergedCodeRepository:
|
|
162
|
+
"""Merges multiple code repos into a single graph (multi-repo impact analysis).
|
|
163
|
+
|
|
164
|
+
With ≥2 repos, module ids and dependency edges are namespaced by repo name
|
|
165
|
+
(collisions prevented; intra-repo edges preserved).
|
|
166
|
+
"""
|
|
167
|
+
|
|
168
|
+
def __init__(self, providers: list[tuple[str, CodeRepositoryProvider]]) -> None:
|
|
169
|
+
self._providers = providers
|
|
170
|
+
|
|
171
|
+
async def list_modules(self) -> list[CodeModule]:
|
|
172
|
+
multi = len(self._providers) > 1
|
|
173
|
+
out: list[CodeModule] = []
|
|
174
|
+
for name, provider in self._providers:
|
|
175
|
+
for module in await provider.list_modules():
|
|
176
|
+
if multi:
|
|
177
|
+
prefix = f"{name}:"
|
|
178
|
+
module = module.model_copy(
|
|
179
|
+
update={
|
|
180
|
+
"id": prefix + module.id,
|
|
181
|
+
"depends_on": [prefix + d for d in module.depends_on],
|
|
182
|
+
"depended_by": [prefix + d for d in module.depended_by],
|
|
183
|
+
}
|
|
184
|
+
)
|
|
185
|
+
out.append(module)
|
|
186
|
+
return out
|
|
187
|
+
|
|
188
|
+
async def get_impacted(self, module_hint: str | None) -> list[CodeModule]:
|
|
189
|
+
return impacted_modules(await self.list_modules(), module_hint, depth=1)
|
|
190
|
+
|
|
191
|
+
def list_dependencies(self) -> list[DeclaredDependency]:
|
|
192
|
+
"""Union across repos. Package names stay GLOBAL (the same artifact
|
|
193
|
+
everywhere — unlike module ids); only the `manifest` provenance gets
|
|
194
|
+
the repo prefix when there is more than one repo."""
|
|
195
|
+
multi = len(self._providers) > 1
|
|
196
|
+
seen: set[tuple[str, str, str]] = set()
|
|
197
|
+
out: list[DeclaredDependency] = []
|
|
198
|
+
for name, provider in self._providers:
|
|
199
|
+
lister = getattr(provider, "list_dependencies", None)
|
|
200
|
+
for dep in lister() if lister else []:
|
|
201
|
+
key = (dep.ecosystem, dep.name, dep.raw_spec)
|
|
202
|
+
if key in seen:
|
|
203
|
+
continue
|
|
204
|
+
seen.add(key)
|
|
205
|
+
if multi:
|
|
206
|
+
dep = dep.model_copy(update={"manifest": f"{name}:{dep.manifest}"})
|
|
207
|
+
out.append(dep)
|
|
208
|
+
return out
|
|
209
|
+
|
|
210
|
+
def capabilities(self) -> Capabilities:
|
|
211
|
+
return Capabilities(supports_incremental_diff=True)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
class IndexBackedCodeRepository:
|
|
215
|
+
"""Common CodeRepositoryProvider base fed from a `CodeIndex`."""
|
|
216
|
+
|
|
217
|
+
def __init__(
|
|
218
|
+
self,
|
|
219
|
+
index: CodeIndex,
|
|
220
|
+
churn: dict[str, int] | None = None,
|
|
221
|
+
*,
|
|
222
|
+
supports_incremental_diff: bool = False,
|
|
223
|
+
) -> None:
|
|
224
|
+
self._modules = parse_code_index(index)
|
|
225
|
+
self._dependencies = list(index.dependencies)
|
|
226
|
+
if churn:
|
|
227
|
+
for module in self._modules:
|
|
228
|
+
module.churn.commits_last_6mo = churn.get(module.id, 0)
|
|
229
|
+
self._incremental = supports_incremental_diff
|
|
230
|
+
|
|
231
|
+
async def list_modules(self) -> list[CodeModule]:
|
|
232
|
+
return list(self._modules)
|
|
233
|
+
|
|
234
|
+
def list_dependencies(self) -> list[DeclaredDependency]:
|
|
235
|
+
return list(self._dependencies)
|
|
236
|
+
|
|
237
|
+
async def get_impacted(self, module_hint: str | None) -> list[CodeModule]:
|
|
238
|
+
return impacted_modules(self._modules, module_hint)
|
|
239
|
+
|
|
240
|
+
def capabilities(self) -> Capabilities:
|
|
241
|
+
return Capabilities(
|
|
242
|
+
supports_webhooks=False,
|
|
243
|
+
supports_realtime=False,
|
|
244
|
+
supports_effort_tracking=False,
|
|
245
|
+
supports_incremental_diff=self._incremental,
|
|
246
|
+
)
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Composite DocumentSourceProvider — merges multiple document sources behind one port.
|
|
2
|
+
|
|
3
|
+
The architecture's example: some documents live in FileSystem, some in SharePoint →
|
|
4
|
+
composite. The core sees a single DocumentSourceProvider; strong proof of pluggability.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from etki.core.models import DocumentRef
|
|
10
|
+
from etki.core.ports import Capabilities, DocumentSourceProvider
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class CompositeDocumentSourceProvider:
|
|
14
|
+
def __init__(self, sources: list[DocumentSourceProvider]) -> None:
|
|
15
|
+
self._sources = list(sources)
|
|
16
|
+
|
|
17
|
+
async def list_documents(self) -> list[DocumentRef]:
|
|
18
|
+
docs: list[DocumentRef] = []
|
|
19
|
+
for idx, source in enumerate(self._sources):
|
|
20
|
+
for doc in await source.list_documents():
|
|
21
|
+
docs.append(doc.model_copy(update={"id": f"{idx}:{doc.id}"}))
|
|
22
|
+
return docs
|
|
23
|
+
|
|
24
|
+
async def fetch_content(self, document_id: str) -> bytes:
|
|
25
|
+
prefix, _, real_id = document_id.partition(":")
|
|
26
|
+
try:
|
|
27
|
+
idx = int(prefix)
|
|
28
|
+
except ValueError as exc:
|
|
29
|
+
raise KeyError(f"Composite id beklenen formatta değil: {document_id}") from exc
|
|
30
|
+
if not 0 <= idx < len(self._sources):
|
|
31
|
+
raise KeyError(f"Geçersiz kaynak indeksi: {idx}")
|
|
32
|
+
return await self._sources[idx].fetch_content(real_id)
|
|
33
|
+
|
|
34
|
+
def capabilities(self) -> Capabilities:
|
|
35
|
+
caps = [s.capabilities() for s in self._sources]
|
|
36
|
+
if not caps:
|
|
37
|
+
return Capabilities()
|
|
38
|
+
# Graceful degradation: a capability is guaranteed only if ALL children support it.
|
|
39
|
+
return Capabilities(
|
|
40
|
+
supports_webhooks=all(c.supports_webhooks for c in caps),
|
|
41
|
+
supports_realtime=all(c.supports_realtime for c in caps),
|
|
42
|
+
supports_effort_tracking=all(c.supports_effort_tracking for c in caps),
|
|
43
|
+
supports_incremental_diff=all(c.supports_incremental_diff for c in caps),
|
|
44
|
+
)
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""Confluence Cloud DocumentSourceProvider — pages of a space as contract documents.
|
|
2
|
+
|
|
3
|
+
Auth mirrors the Jira adapter (same Atlassian account): email + API token, basic
|
|
4
|
+
auth. Pages are listed via the REST content API (paginated); `fetch_content`
|
|
5
|
+
returns the page's **storage-format HTML converted to plain text** (UTF-8 bytes)
|
|
6
|
+
so scope extraction sees clean clause lines. Requires a live site → integration
|
|
7
|
+
is CI-skipped; HTML→text conversion is unit-tested. Config example:
|
|
8
|
+
|
|
9
|
+
connectors:
|
|
10
|
+
documents:
|
|
11
|
+
adapter: confluence
|
|
12
|
+
options:
|
|
13
|
+
base_url: https://yoursite.atlassian.net/wiki
|
|
14
|
+
email: you@company.com
|
|
15
|
+
api_token: env:CONFLUENCE_TOKEN
|
|
16
|
+
space_key: CONTRACTS
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import html as html_lib
|
|
22
|
+
import re
|
|
23
|
+
from datetime import datetime
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
import httpx
|
|
27
|
+
|
|
28
|
+
from etki.core.models import DocumentRef
|
|
29
|
+
from etki.core.ports import Capabilities
|
|
30
|
+
|
|
31
|
+
_PAGE_SIZE = 50
|
|
32
|
+
# Block-level closers become newlines so clause lines stay separate lines.
|
|
33
|
+
_BLOCK_RE = re.compile(r"(?i)<\s*(?:br\s*/?|/p|/h[1-6]|/li|/tr|/div)\s*>")
|
|
34
|
+
_TAG_RE = re.compile(r"<[^>]+>")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def storage_to_text(storage_html: str) -> str:
|
|
38
|
+
"""Confluence storage-format HTML → plain text (newline per block, entities unescaped)."""
|
|
39
|
+
text = _BLOCK_RE.sub("\n", storage_html)
|
|
40
|
+
text = _TAG_RE.sub(" ", text)
|
|
41
|
+
text = html_lib.unescape(text)
|
|
42
|
+
lines = [re.sub(r"[ \t]+", " ", line).strip() for line in text.splitlines()]
|
|
43
|
+
return "\n".join(line for line in lines if line)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class ConfluenceDocumentSourceProvider:
|
|
47
|
+
def __init__(
|
|
48
|
+
self,
|
|
49
|
+
base_url: str,
|
|
50
|
+
email: str,
|
|
51
|
+
api_token: str,
|
|
52
|
+
space_key: str,
|
|
53
|
+
timeout: float = 30.0,
|
|
54
|
+
) -> None:
|
|
55
|
+
self._base_url = base_url.rstrip("/")
|
|
56
|
+
self._auth = (email, api_token)
|
|
57
|
+
self._space_key = space_key
|
|
58
|
+
self._timeout = timeout
|
|
59
|
+
|
|
60
|
+
def _to_ref(self, page: dict[str, Any]) -> DocumentRef:
|
|
61
|
+
version = page.get("version") or {}
|
|
62
|
+
when = version.get("when")
|
|
63
|
+
return DocumentRef(
|
|
64
|
+
id=str(page.get("id", "?")),
|
|
65
|
+
name=str(page.get("title") or "untitled"), # no extension → parsed as text
|
|
66
|
+
path=f"{self._base_url}{(page.get('_links') or {}).get('webui', '')}",
|
|
67
|
+
mime="text/plain",
|
|
68
|
+
modified_at=datetime.fromisoformat(when) if when else None,
|
|
69
|
+
source="confluence",
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
async def list_documents(self) -> list[DocumentRef]:
|
|
73
|
+
docs: list[DocumentRef] = []
|
|
74
|
+
start = 0
|
|
75
|
+
async with httpx.AsyncClient(timeout=self._timeout, auth=self._auth) as client:
|
|
76
|
+
while True:
|
|
77
|
+
response = await client.get(
|
|
78
|
+
f"{self._base_url}/rest/api/content",
|
|
79
|
+
params={
|
|
80
|
+
"spaceKey": self._space_key,
|
|
81
|
+
"type": "page",
|
|
82
|
+
"status": "current",
|
|
83
|
+
"start": start,
|
|
84
|
+
"limit": _PAGE_SIZE,
|
|
85
|
+
"expand": "version",
|
|
86
|
+
},
|
|
87
|
+
)
|
|
88
|
+
response.raise_for_status()
|
|
89
|
+
results = response.json().get("results", [])
|
|
90
|
+
docs.extend(self._to_ref(p) for p in results)
|
|
91
|
+
if len(results) < _PAGE_SIZE:
|
|
92
|
+
return docs
|
|
93
|
+
start += _PAGE_SIZE
|
|
94
|
+
|
|
95
|
+
async def fetch_content(self, document_id: str) -> bytes:
|
|
96
|
+
async with httpx.AsyncClient(timeout=self._timeout, auth=self._auth) as client:
|
|
97
|
+
response = await client.get(
|
|
98
|
+
f"{self._base_url}/rest/api/content/{document_id}",
|
|
99
|
+
params={"expand": "body.storage"},
|
|
100
|
+
)
|
|
101
|
+
response.raise_for_status()
|
|
102
|
+
storage = (
|
|
103
|
+
(response.json().get("body") or {}).get("storage") or {}
|
|
104
|
+
).get("value", "")
|
|
105
|
+
return storage_to_text(storage).encode("utf-8")
|
|
106
|
+
|
|
107
|
+
def capabilities(self) -> Capabilities:
|
|
108
|
+
return Capabilities(
|
|
109
|
+
supports_webhooks=False, # Cloud webhooks need a Connect/Forge app — not implemented
|
|
110
|
+
supports_realtime=False,
|
|
111
|
+
supports_effort_tracking=False,
|
|
112
|
+
supports_incremental_diff=False,
|
|
113
|
+
)
|