concierge-graph 3.8.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agents/__init__.py +14 -0
- agents/revisor_critico.py +610 -0
- concierge_graph-3.8.2.dist-info/METADATA +327 -0
- concierge_graph-3.8.2.dist-info/RECORD +37 -0
- concierge_graph-3.8.2.dist-info/WHEEL +5 -0
- concierge_graph-3.8.2.dist-info/entry_points.txt +3 -0
- concierge_graph-3.8.2.dist-info/licenses/LICENSE +21 -0
- concierge_graph-3.8.2.dist-info/top_level.txt +8 -0
- core/__init__.py +23 -0
- core/config.py +192 -0
- core/hybrid_search.py +288 -0
- core/memory_extractor.py +245 -0
- core/middleware.py +723 -0
- core/probabilistic_retriever.py +99 -0
- core/project_index.py +316 -0
- core/vector_backend.py +471 -0
- ingestion/__init__.py +25 -0
- ingestion/crawler.py +722 -0
- ingestion/orchestrator.py +920 -0
- ingestion/parser.py +984 -0
- ingestion/summarizer.py +948 -0
- interface/__init__.py +16 -0
- interface/action_hooks.py +261 -0
- interface/cli.py +391 -0
- interface/mcp_server.py +1737 -0
- main.py +281 -0
- scripts/bootstrap_core_memory.py +93 -0
- services/__init__.py +13 -0
- services/janitor.py +762 -0
- storage/__init__.py +31 -0
- storage/base_backend.py +241 -0
- storage/connection.py +310 -0
- storage/logic.py +703 -0
- storage/schema.py +390 -0
- storage/semantic_logic.py +125 -0
- storage/store.py +802 -0
- storage/vector_store.py +759 -0
ingestion/crawler.py
ADDED
|
@@ -0,0 +1,722 @@
|
|
|
1
|
+
"""
|
|
2
|
+
ingestion/crawler.py — Grafo Concierge v3.8.0 (Absolute Solidity)
|
|
3
|
+
|
|
4
|
+
Intelligent filesystem scanning for the Apex Ingestion Engine.
|
|
5
|
+
|
|
6
|
+
Responsibilities:
|
|
7
|
+
- Recursively traverse project directories.
|
|
8
|
+
- Respect ignore patterns (.gitignore + IGNORE_DIRS from config).
|
|
9
|
+
- Calculate SHA256 of each file for delta detection.
|
|
10
|
+
- Compare hashes with SqliteStore to skip unmodified files.
|
|
11
|
+
- Classify files by type (code, doc, config, conversation).
|
|
12
|
+
- Detect deleted files for Garbage Collection.
|
|
13
|
+
|
|
14
|
+
Integration:
|
|
15
|
+
- SqliteStore.find_node_by_hash(project_uuid, hash) → checks if already processed.
|
|
16
|
+
- SqliteStore.get_nodes_by_project(project_uuid) → lists existing nodes for GC.
|
|
17
|
+
- Result is a CrawlReport consumed by the Parser/Orchestrator.
|
|
18
|
+
|
|
19
|
+
Identity Preservation (Path-Agnostic ID):
|
|
20
|
+
The doc_id of each node is derived from the content (hash), not the path.
|
|
21
|
+
If the file moves to another folder, the hash changes → new node is created.
|
|
22
|
+
If the file is renamed without content changes → same hash → reuses the node.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import fnmatch
|
|
28
|
+
import hashlib
|
|
29
|
+
import logging
|
|
30
|
+
import os
|
|
31
|
+
from storage import SqliteStore
|
|
32
|
+
from dataclasses import dataclass, field
|
|
33
|
+
from enum import Enum
|
|
34
|
+
from pathlib import Path
|
|
35
|
+
from typing import Optional
|
|
36
|
+
|
|
37
|
+
logger = logging.getLogger("grafo-concierge.crawler")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
# ---------------------------------------------------------------------------
|
|
41
|
+
# File classification
|
|
42
|
+
# ---------------------------------------------------------------------------
|
|
43
|
+
|
|
44
|
+
class FileCategory(str, Enum):
|
|
45
|
+
"""File categories automatically detected by the Crawler.
|
|
46
|
+
|
|
47
|
+
Mapping per spec v3.8:
|
|
48
|
+
code → .py, .js, .ts, .go, .rs, .java, .cpp, .c, .rb
|
|
49
|
+
doc → .md, .txt, .rst, .adoc
|
|
50
|
+
config → .json, .yaml, .yml, .toml, .env, .ini, .cfg
|
|
51
|
+
conversation → .log, .chat
|
|
52
|
+
unknown → unmapped extensions
|
|
53
|
+
"""
|
|
54
|
+
CODE = "code"
|
|
55
|
+
DOC = "doc"
|
|
56
|
+
CONFIG = "config"
|
|
57
|
+
CONVERSATION = "conversation"
|
|
58
|
+
UNKNOWN = "unknown"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
# Extension -> category mapping (per API v3.8 table)
|
|
62
|
+
EXTENSION_MAP: dict[str, FileCategory] = {
|
|
63
|
+
# Code
|
|
64
|
+
".py": FileCategory.CODE,
|
|
65
|
+
".js": FileCategory.CODE,
|
|
66
|
+
".ts": FileCategory.CODE,
|
|
67
|
+
".tsx": FileCategory.CODE,
|
|
68
|
+
".jsx": FileCategory.CODE,
|
|
69
|
+
".go": FileCategory.CODE,
|
|
70
|
+
".rs": FileCategory.CODE,
|
|
71
|
+
".java": FileCategory.CODE,
|
|
72
|
+
".cpp": FileCategory.CODE,
|
|
73
|
+
".c": FileCategory.CODE,
|
|
74
|
+
".h": FileCategory.CODE,
|
|
75
|
+
".hpp": FileCategory.CODE,
|
|
76
|
+
".rb": FileCategory.CODE,
|
|
77
|
+
".cs": FileCategory.CODE,
|
|
78
|
+
".swift": FileCategory.CODE,
|
|
79
|
+
".kt": FileCategory.CODE,
|
|
80
|
+
".php": FileCategory.CODE,
|
|
81
|
+
".lua": FileCategory.CODE,
|
|
82
|
+
".sh": FileCategory.CODE,
|
|
83
|
+
".bash": FileCategory.CODE,
|
|
84
|
+
".ps1": FileCategory.CODE,
|
|
85
|
+
".sql": FileCategory.CODE,
|
|
86
|
+
".r": FileCategory.CODE,
|
|
87
|
+
".scala": FileCategory.CODE,
|
|
88
|
+
".ex": FileCategory.CODE,
|
|
89
|
+
".exs": FileCategory.CODE,
|
|
90
|
+
# Doc
|
|
91
|
+
".md": FileCategory.DOC,
|
|
92
|
+
".txt": FileCategory.DOC,
|
|
93
|
+
".rst": FileCategory.DOC,
|
|
94
|
+
".adoc": FileCategory.DOC,
|
|
95
|
+
".org": FileCategory.DOC,
|
|
96
|
+
# Config
|
|
97
|
+
".json": FileCategory.CONFIG,
|
|
98
|
+
".yaml": FileCategory.CONFIG,
|
|
99
|
+
".yml": FileCategory.CONFIG,
|
|
100
|
+
".toml": FileCategory.CONFIG,
|
|
101
|
+
".env": FileCategory.CONFIG,
|
|
102
|
+
".ini": FileCategory.CONFIG,
|
|
103
|
+
".cfg": FileCategory.CONFIG,
|
|
104
|
+
".xml": FileCategory.CONFIG,
|
|
105
|
+
".properties": FileCategory.CONFIG,
|
|
106
|
+
# Conversation
|
|
107
|
+
".log": FileCategory.CONVERSATION,
|
|
108
|
+
".chat": FileCategory.CONVERSATION,
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
# ---------------------------------------------------------------------------
|
|
113
|
+
# CrawlResult — individual result of a scanned file
|
|
114
|
+
# ---------------------------------------------------------------------------
|
|
115
|
+
|
|
116
|
+
@dataclass
|
|
117
|
+
class CrawlResult:
|
|
118
|
+
"""Result of the scan of a single file.
|
|
119
|
+
|
|
120
|
+
Attributes:
|
|
121
|
+
absolute_path: Absolute path in the filesystem.
|
|
122
|
+
relative_path: Relative path to the project's source_path.
|
|
123
|
+
file_hash: SHA256 of the file content.
|
|
124
|
+
category: Automatic classification (code, doc, config, conversation).
|
|
125
|
+
extension: File extension (e.g. '.py').
|
|
126
|
+
size_bytes: Size in bytes.
|
|
127
|
+
is_new: True if the hash does not exist in SqliteStore (new or modified file).
|
|
128
|
+
existing_node_id: If not new, the ID of the existing node in SQLite.
|
|
129
|
+
"""
|
|
130
|
+
absolute_path: str
|
|
131
|
+
relative_path: str
|
|
132
|
+
file_hash: str
|
|
133
|
+
category: FileCategory
|
|
134
|
+
extension: str
|
|
135
|
+
size_bytes: int
|
|
136
|
+
is_new: bool = True
|
|
137
|
+
existing_node_id: Optional[int] = None
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
@dataclass
|
|
141
|
+
class CrawlReport:
|
|
142
|
+
"""Consolidated report of a crawl operation.
|
|
143
|
+
|
|
144
|
+
Attributes:
|
|
145
|
+
new_files: New or modified files (for processing).
|
|
146
|
+
unchanged_files: Files whose hash has not changed (skip).
|
|
147
|
+
deleted_node_ids: SQLite node IDs whose files no longer exist (GC).
|
|
148
|
+
categories: Count per category.
|
|
149
|
+
total_scanned: Total scanned files.
|
|
150
|
+
"""
|
|
151
|
+
new_files: list[CrawlResult] = field(default_factory=list)
|
|
152
|
+
unchanged_files: list[CrawlResult] = field(default_factory=list)
|
|
153
|
+
deleted_node_ids: list[int] = field(default_factory=list)
|
|
154
|
+
categories: dict[str, int] = field(default_factory=dict)
|
|
155
|
+
total_scanned: int = 0
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
# ---------------------------------------------------------------------------
|
|
159
|
+
# GitignoreParser — robust parser for .gitignore
|
|
160
|
+
# ---------------------------------------------------------------------------
|
|
161
|
+
|
|
162
|
+
class GitignoreParser:
|
|
163
|
+
"""Parser for .gitignore with support for common patterns.
|
|
164
|
+
|
|
165
|
+
Supports:
|
|
166
|
+
- Comments (# ...) and blank lines.
|
|
167
|
+
- Negation (! pattern → do not ignore).
|
|
168
|
+
- Explicit directories (dir/ → directories only).
|
|
169
|
+
- Wildcards (*, **, ?).
|
|
170
|
+
- Anchored patterns (starting with /).
|
|
171
|
+
|
|
172
|
+
Limitations:
|
|
173
|
+
- Does not support nested .gitignore in subdirectories (root only).
|
|
174
|
+
- Complex patterns with ranges [a-z] are treated as simple globs.
|
|
175
|
+
"""
|
|
176
|
+
|
|
177
|
+
def __init__(self) -> None:
|
|
178
|
+
"""Initializes the parser with empty lists."""
|
|
179
|
+
self._patterns: list[str] = []
|
|
180
|
+
self._negations: list[str] = []
|
|
181
|
+
self._dir_only_patterns: list[str] = []
|
|
182
|
+
|
|
183
|
+
def add_patterns(self, patterns: list[str]) -> None:
|
|
184
|
+
"""Adds a list of ignore patterns programmatically.
|
|
185
|
+
|
|
186
|
+
Allows loading default safety patterns (DEFAULT_IGNORE_PATTERNS)
|
|
187
|
+
without depending on a file on disk. Follows the same semantics as .gitignore.
|
|
188
|
+
|
|
189
|
+
Args:
|
|
190
|
+
patterns: List of patterns in .gitignore format.
|
|
191
|
+
"""
|
|
192
|
+
for raw in patterns:
|
|
193
|
+
stripped = raw.strip()
|
|
194
|
+
if not stripped or stripped.startswith("#"):
|
|
195
|
+
continue
|
|
196
|
+
if stripped.startswith("!"):
|
|
197
|
+
negation = stripped[1:].strip()
|
|
198
|
+
if negation:
|
|
199
|
+
self._negations.append(negation)
|
|
200
|
+
continue
|
|
201
|
+
if stripped.endswith("/"):
|
|
202
|
+
self._dir_only_patterns.append(stripped.rstrip("/"))
|
|
203
|
+
self._patterns.append(stripped.rstrip("/"))
|
|
204
|
+
self._patterns.append(stripped.rstrip("/") + "/**")
|
|
205
|
+
continue
|
|
206
|
+
if stripped.startswith("/"):
|
|
207
|
+
stripped = stripped[1:]
|
|
208
|
+
self._patterns.append(stripped)
|
|
209
|
+
|
|
210
|
+
logger.debug(
|
|
211
|
+
"GitignoreParser.add_patterns: +%d entradas → %d padrões totais.",
|
|
212
|
+
len(patterns), len(self._patterns),
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
def load(self, gitignore_path: str) -> None:
|
|
216
|
+
"""Loads and parses a .gitignore file.
|
|
217
|
+
|
|
218
|
+
Ignores blank lines and comments.
|
|
219
|
+
Negation patterns (!) are stored separately.
|
|
220
|
+
Directory patterns (ending in /) are stored separately.
|
|
221
|
+
|
|
222
|
+
Args:
|
|
223
|
+
gitignore_path: Absolute path to the .gitignore.
|
|
224
|
+
"""
|
|
225
|
+
try:
|
|
226
|
+
with open(gitignore_path, "r", encoding="utf-8", errors="replace") as f:
|
|
227
|
+
raw_lines = f.readlines()
|
|
228
|
+
except (OSError, IOError) as e:
|
|
229
|
+
logger.warning("Could not read .gitignore at %s: %s", gitignore_path, e)
|
|
230
|
+
return
|
|
231
|
+
|
|
232
|
+
for line in raw_lines:
|
|
233
|
+
# Remove trailing whitespace (keeps leading for space patterns)
|
|
234
|
+
stripped = line.rstrip()
|
|
235
|
+
|
|
236
|
+
# Ignores empty lines and comments
|
|
237
|
+
if not stripped or stripped.startswith("#"):
|
|
238
|
+
continue
|
|
239
|
+
|
|
240
|
+
# Remove trailing escaped space (\\ at the end)
|
|
241
|
+
if stripped.endswith("\\ "):
|
|
242
|
+
stripped = stripped[:-2] + " "
|
|
243
|
+
|
|
244
|
+
# Negation patterns
|
|
245
|
+
if stripped.startswith("!"):
|
|
246
|
+
negation = stripped[1:].strip()
|
|
247
|
+
if negation:
|
|
248
|
+
self._negations.append(negation)
|
|
249
|
+
continue
|
|
250
|
+
|
|
251
|
+
# Directory patterns (ending in /)
|
|
252
|
+
if stripped.endswith("/"):
|
|
253
|
+
self._dir_only_patterns.append(stripped.rstrip("/"))
|
|
254
|
+
# Also adds as normal pattern for directory contents
|
|
255
|
+
self._patterns.append(stripped.rstrip("/"))
|
|
256
|
+
self._patterns.append(stripped.rstrip("/") + "/**")
|
|
257
|
+
continue
|
|
258
|
+
|
|
259
|
+
# Remove leading slash (anchors to root, but we treat as relative)
|
|
260
|
+
if stripped.startswith("/"):
|
|
261
|
+
stripped = stripped[1:]
|
|
262
|
+
|
|
263
|
+
self._patterns.append(stripped)
|
|
264
|
+
|
|
265
|
+
logger.debug(
|
|
266
|
+
"GitignoreParser: %d patterns loaded, %d negations, %d dir-only",
|
|
267
|
+
len(self._patterns), len(self._negations), len(self._dir_only_patterns),
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
def should_ignore(self, relative_path: str, is_dir: bool = False) -> bool:
|
|
271
|
+
"""Verifies if a relative path should be ignored.
|
|
272
|
+
|
|
273
|
+
Logic follows Git semantics:
|
|
274
|
+
1. If the path matches any ignore pattern → True.
|
|
275
|
+
2. If the path matches a negation pattern → False (override).
|
|
276
|
+
3. Directory-only patterns only apply to directories.
|
|
277
|
+
|
|
278
|
+
Args:
|
|
279
|
+
relative_path: Relative path to the project root (forward slashes).
|
|
280
|
+
is_dir: True if the path is a directory.
|
|
281
|
+
|
|
282
|
+
Returns:
|
|
283
|
+
True if it should be ignored.
|
|
284
|
+
"""
|
|
285
|
+
# Normalize to forward slashes
|
|
286
|
+
normalized = relative_path.replace("\\", "/")
|
|
287
|
+
# Extract basename for simple matching (e.g. 'node_modules')
|
|
288
|
+
basename = normalized.rsplit("/", 1)[-1] if "/" in normalized else normalized
|
|
289
|
+
|
|
290
|
+
# Check negations first (override)
|
|
291
|
+
for neg_pattern in self._negations:
|
|
292
|
+
if self._matches(normalized, neg_pattern) or self._matches(basename, neg_pattern):
|
|
293
|
+
return False
|
|
294
|
+
|
|
295
|
+
# Check directory patterns
|
|
296
|
+
if is_dir:
|
|
297
|
+
for dir_pattern in self._dir_only_patterns:
|
|
298
|
+
if self._matches(normalized, dir_pattern) or self._matches(basename, dir_pattern):
|
|
299
|
+
return True
|
|
300
|
+
|
|
301
|
+
# Check general patterns
|
|
302
|
+
for pattern in self._patterns:
|
|
303
|
+
if self._matches(normalized, pattern) or self._matches(basename, pattern):
|
|
304
|
+
return True
|
|
305
|
+
|
|
306
|
+
return False
|
|
307
|
+
|
|
308
|
+
@staticmethod
|
|
309
|
+
def _matches(path: str, pattern: str) -> bool:
|
|
310
|
+
"""Checks if a path matches a glob pattern.
|
|
311
|
+
|
|
312
|
+
Supports:
|
|
313
|
+
- fnmatch wildcards (*, ?, [seq])
|
|
314
|
+
- ** (recursive glob — match of any depth)
|
|
315
|
+
|
|
316
|
+
Args:
|
|
317
|
+
path: Normalized path (forward slashes).
|
|
318
|
+
pattern: Glob pattern.
|
|
319
|
+
|
|
320
|
+
Returns:
|
|
321
|
+
True if the path matches the pattern.
|
|
322
|
+
"""
|
|
323
|
+
# ** → any subdirectory
|
|
324
|
+
if "**" in pattern:
|
|
325
|
+
# Transforms 'dir/**/file' into recursive match
|
|
326
|
+
# Splits by '**' and checks the parts
|
|
327
|
+
parts = pattern.split("**")
|
|
328
|
+
if len(parts) == 2:
|
|
329
|
+
prefix = parts[0].rstrip("/")
|
|
330
|
+
suffix = parts[1].lstrip("/")
|
|
331
|
+
if prefix and not path.startswith(prefix):
|
|
332
|
+
return False
|
|
333
|
+
if suffix and not fnmatch.fnmatch(path.rsplit("/", 1)[-1], suffix):
|
|
334
|
+
return False
|
|
335
|
+
if prefix and suffix:
|
|
336
|
+
return True
|
|
337
|
+
if not prefix and suffix:
|
|
338
|
+
return fnmatch.fnmatch(path.rsplit("/", 1)[-1], suffix)
|
|
339
|
+
if prefix and not suffix:
|
|
340
|
+
return path.startswith(prefix)
|
|
341
|
+
return True # ** alone → ignores everything
|
|
342
|
+
|
|
343
|
+
# Pattern contains / → match against full path
|
|
344
|
+
if "/" in pattern:
|
|
345
|
+
return fnmatch.fnmatch(path, pattern)
|
|
346
|
+
|
|
347
|
+
# Simple pattern → match against each component of the path
|
|
348
|
+
components = path.split("/")
|
|
349
|
+
for component in components:
|
|
350
|
+
if fnmatch.fnmatch(component, pattern):
|
|
351
|
+
return True
|
|
352
|
+
|
|
353
|
+
return False
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
# ---------------------------------------------------------------------------
|
|
357
|
+
# ProjectCrawler — Scanning engine
|
|
358
|
+
# ---------------------------------------------------------------------------
|
|
359
|
+
|
|
360
|
+
class ProjectCrawler:
|
|
361
|
+
"""Intelligent filesystem scanning with Delta Detection.
|
|
362
|
+
|
|
363
|
+
Flow:
|
|
364
|
+
1. Loads ignore patterns (.gitignore + IGNORE_DIRS).
|
|
365
|
+
2. Recursively traverses the source_path.
|
|
366
|
+
3. Calculates SHA256 of each file.
|
|
367
|
+
4. Queries SqliteStore to check if the hash already exists.
|
|
368
|
+
5. Classifies file by extension.
|
|
369
|
+
6. Detects orphan nodes (deleted files) for Garbage Collection.
|
|
370
|
+
|
|
371
|
+
Args:
|
|
372
|
+
sqlite_store: SqliteStore instance for hash queries.
|
|
373
|
+
ignore_dirs: List of extra directories to ignore (merge with DEFAULT_IGNORE_DIRS).
|
|
374
|
+
"""
|
|
375
|
+
|
|
376
|
+
# Ignored directories by default (per ConciergeConfig.IGNORE_DIRS)
|
|
377
|
+
DEFAULT_IGNORE_DIRS: set[str] = {
|
|
378
|
+
".git", "node_modules", ".next", "dist", "build",
|
|
379
|
+
"__pycache__", ".venv", "venv", ".env", ".idea",
|
|
380
|
+
".vscode", ".mypy_cache", ".pytest_cache", "coverage",
|
|
381
|
+
".tox", "egg-info", ".eggs", ".cache", ".gradle",
|
|
382
|
+
"target", "vendor", "bower_components",
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
# Ignored FILE patterns by default (zero-config security)
|
|
386
|
+
DEFAULT_IGNORE_PATTERNS: list[str] = [
|
|
387
|
+
# Security — Credentials and Keys
|
|
388
|
+
".env", ".env.*",
|
|
389
|
+
"*.pem", "*.key", "*.cert", "*.der", "*.pfx", "*.p12",
|
|
390
|
+
"id_rsa", "id_dsa", "id_ed25519",
|
|
391
|
+
# Lock files — Pure noise
|
|
392
|
+
"package-lock.json", "yarn.lock", "pnpm-lock.yaml",
|
|
393
|
+
"poetry.lock", "Cargo.lock", "composer.lock", "Gemfile.lock",
|
|
394
|
+
# Logs and junk text
|
|
395
|
+
"*.log", "*.txt",
|
|
396
|
+
# Local databases
|
|
397
|
+
"*.db", "*.sqlite", "*.sqlite3",
|
|
398
|
+
# Binaries and compilation
|
|
399
|
+
"*.exe", "*.dll", "*.so", "*.dylib", "*.bin", "*.o", "*.a",
|
|
400
|
+
"*.pyc", "*.pyo", "*.class", "*.wasm",
|
|
401
|
+
# Compressed files
|
|
402
|
+
"*.zip", "*.tar", "*.tar.gz", "*.tgz", "*.rar", "*.7z", "*.bz2",
|
|
403
|
+
# Media and Images
|
|
404
|
+
"*.jpg", "*.jpeg", "*.png", "*.gif", "*.ico", "*.svg",
|
|
405
|
+
"*.mp3", "*.mp4", "*.wav", "*.avi", "*.mov", "*.webp",
|
|
406
|
+
"*.ttf", "*.woff", "*.woff2", "*.eot",
|
|
407
|
+
# OS junk
|
|
408
|
+
".DS_Store", "Thumbs.db", "desktop.ini",
|
|
409
|
+
]
|
|
410
|
+
|
|
411
|
+
# Maximum file size for ingestion (10 MB)
|
|
412
|
+
MAX_FILE_SIZE_BYTES: int = 10 * 1024 * 1024
|
|
413
|
+
|
|
414
|
+
# Read block size for hash (64 KB — memory efficiency)
|
|
415
|
+
_HASH_BLOCK_SIZE: int = 65536
|
|
416
|
+
|
|
417
|
+
def __init__(
|
|
418
|
+
self,
|
|
419
|
+
sqlite_store: "SqliteStore",
|
|
420
|
+
ignore_dirs: Optional[set[str]] = None,
|
|
421
|
+
) -> None:
|
|
422
|
+
"""Initializes the Crawler.
|
|
423
|
+
|
|
424
|
+
Args:
|
|
425
|
+
sqlite_store: SQLite facade for querying existing hashes.
|
|
426
|
+
ignore_dirs: Extra directories to ignore (merge with DEFAULT_IGNORE_DIRS).
|
|
427
|
+
"""
|
|
428
|
+
self._store = sqlite_store
|
|
429
|
+
self._ignore_dirs = self.DEFAULT_IGNORE_DIRS.copy()
|
|
430
|
+
if ignore_dirs:
|
|
431
|
+
self._ignore_dirs.update(ignore_dirs)
|
|
432
|
+
|
|
433
|
+
# GitignoreParser will be loaded by crawl() if .gitignore exists
|
|
434
|
+
self._gitignore: Optional[GitignoreParser] = None
|
|
435
|
+
|
|
436
|
+
logger.info(
|
|
437
|
+
"ProjectCrawler initialized: %d active ignore patterns.",
|
|
438
|
+
len(self._ignore_dirs),
|
|
439
|
+
)
|
|
440
|
+
|
|
441
|
+
def crawl(
|
|
442
|
+
self,
|
|
443
|
+
source_path: str,
|
|
444
|
+
project_uuid: str,
|
|
445
|
+
) -> CrawlReport:
|
|
446
|
+
"""Executes a complete scan of a project directory.
|
|
447
|
+
|
|
448
|
+
Flow:
|
|
449
|
+
1. Validates source_path.
|
|
450
|
+
2. Loads .gitignore (if it exists).
|
|
451
|
+
3. Traverses files recursively.
|
|
452
|
+
4. For each file: hash → classify → delta check.
|
|
453
|
+
5. Detects orphan nodes (Garbage Collection).
|
|
454
|
+
6. Returns CrawlReport with new_files, unchanged, and deleted.
|
|
455
|
+
|
|
456
|
+
Args:
|
|
457
|
+
source_path: Root directory of the project to scan.
|
|
458
|
+
project_uuid: Project UUID (for node filtering in SQLite).
|
|
459
|
+
|
|
460
|
+
Returns:
|
|
461
|
+
CrawlReport with the complete scan inventory.
|
|
462
|
+
|
|
463
|
+
Raises:
|
|
464
|
+
FileNotFoundError: If source_path does not exist.
|
|
465
|
+
NotADirectoryError: If source_path is not a directory.
|
|
466
|
+
"""
|
|
467
|
+
root = Path(source_path).resolve()
|
|
468
|
+
|
|
469
|
+
# --- Input validation ---
|
|
470
|
+
if not root.exists():
|
|
471
|
+
raise FileNotFoundError(f"source_path does not exist: {source_path}")
|
|
472
|
+
if not root.is_dir():
|
|
473
|
+
raise NotADirectoryError(f"source_path is not a directory: {source_path}")
|
|
474
|
+
|
|
475
|
+
logger.info("Crawl started: %s (project: %s)", root, project_uuid)
|
|
476
|
+
|
|
477
|
+
# --- Load ignore patterns (3 layers) ---
|
|
478
|
+
self._gitignore = GitignoreParser()
|
|
479
|
+
|
|
480
|
+
# Layer 1: Default safety patterns (zero-config)
|
|
481
|
+
self._gitignore.add_patterns(self.DEFAULT_IGNORE_PATTERNS)
|
|
482
|
+
logger.info("Default safety patterns loaded: %d rules.", len(self.DEFAULT_IGNORE_PATTERNS))
|
|
483
|
+
|
|
484
|
+
# Layer 2: Project .gitignore
|
|
485
|
+
gitignore_file = root / ".gitignore"
|
|
486
|
+
if gitignore_file.is_file():
|
|
487
|
+
self._gitignore.load(str(gitignore_file))
|
|
488
|
+
logger.info(".gitignore found and loaded: %s", gitignore_file)
|
|
489
|
+
|
|
490
|
+
# Layer 3: .conciergeignore (Grafo Concierge specific extra rules)
|
|
491
|
+
concierge_ignore_file = root / ".conciergeignore"
|
|
492
|
+
if concierge_ignore_file.is_file():
|
|
493
|
+
self._gitignore.load(str(concierge_ignore_file))
|
|
494
|
+
logger.info(".conciergeignore found and loaded: %s", concierge_ignore_file)
|
|
495
|
+
|
|
496
|
+
# --- Traverse recursively ---
|
|
497
|
+
report = CrawlReport()
|
|
498
|
+
current_hashes: set[str] = set()
|
|
499
|
+
current_files: set[str] = set()
|
|
500
|
+
|
|
501
|
+
for dirpath, dirnames, filenames in os.walk(str(root), topdown=True):
|
|
502
|
+
current_dir = Path(dirpath)
|
|
503
|
+
relative_dir = current_dir.relative_to(root)
|
|
504
|
+
|
|
505
|
+
# Filter directories IN-PLACE (os.walk topdown=True respects this)
|
|
506
|
+
dirnames[:] = [
|
|
507
|
+
d for d in dirnames
|
|
508
|
+
if not self._should_ignore_dir(d, str(relative_dir / d))
|
|
509
|
+
]
|
|
510
|
+
|
|
511
|
+
for filename in filenames:
|
|
512
|
+
filepath = current_dir / filename
|
|
513
|
+
relative_filepath = str(relative_dir / filename).replace("\\", "/")
|
|
514
|
+
|
|
515
|
+
# Skip hidden files (start with .)
|
|
516
|
+
if filename.startswith("."):
|
|
517
|
+
continue
|
|
518
|
+
|
|
519
|
+
# Check .gitignore
|
|
520
|
+
if self._gitignore and self._gitignore.should_ignore(relative_filepath, is_dir=False):
|
|
521
|
+
logger.debug("Ignored (.gitignore): %s", relative_filepath)
|
|
522
|
+
continue
|
|
523
|
+
|
|
524
|
+
# Check if it is a regular and readable file
|
|
525
|
+
if not filepath.is_file():
|
|
526
|
+
continue
|
|
527
|
+
|
|
528
|
+
# Check maximum size
|
|
529
|
+
try:
|
|
530
|
+
size_bytes = filepath.stat().st_size
|
|
531
|
+
except OSError as e:
|
|
532
|
+
logger.warning("Could not read stat of %s: %s", filepath, e)
|
|
533
|
+
continue
|
|
534
|
+
|
|
535
|
+
if size_bytes > self.MAX_FILE_SIZE_BYTES:
|
|
536
|
+
logger.debug(
|
|
537
|
+
"File exceeds MAX_FILE_SIZE (%d bytes): %s",
|
|
538
|
+
size_bytes, relative_filepath,
|
|
539
|
+
)
|
|
540
|
+
continue
|
|
541
|
+
|
|
542
|
+
# Skip empty files (0 bytes)
|
|
543
|
+
if size_bytes == 0:
|
|
544
|
+
continue
|
|
545
|
+
|
|
546
|
+
# --- Compute SHA256 ---
|
|
547
|
+
file_hash = self.compute_file_hash(str(filepath))
|
|
548
|
+
if file_hash is None:
|
|
549
|
+
# Reading error — skip (already logged by compute_file_hash)
|
|
550
|
+
continue
|
|
551
|
+
|
|
552
|
+
current_hashes.add(file_hash)
|
|
553
|
+
current_files.add(relative_filepath)
|
|
554
|
+
|
|
555
|
+
# --- Classify ---
|
|
556
|
+
category = self.classify_file(filename)
|
|
557
|
+
extension = Path(filename).suffix.lower()
|
|
558
|
+
|
|
559
|
+
# --- Delta Check via SqliteStore ---
|
|
560
|
+
existing_node = self._store.find_node_by_hash(project_uuid, file_hash)
|
|
561
|
+
|
|
562
|
+
result = CrawlResult(
|
|
563
|
+
absolute_path=str(filepath),
|
|
564
|
+
relative_path=relative_filepath,
|
|
565
|
+
file_hash=file_hash,
|
|
566
|
+
category=category,
|
|
567
|
+
extension=extension,
|
|
568
|
+
size_bytes=size_bytes,
|
|
569
|
+
is_new=(existing_node is None),
|
|
570
|
+
existing_node_id=existing_node["id"] if existing_node else None,
|
|
571
|
+
)
|
|
572
|
+
|
|
573
|
+
if result.is_new:
|
|
574
|
+
report.new_files.append(result)
|
|
575
|
+
else:
|
|
576
|
+
report.unchanged_files.append(result)
|
|
577
|
+
|
|
578
|
+
# Count per category
|
|
579
|
+
cat_key = category.value
|
|
580
|
+
report.categories[cat_key] = report.categories.get(cat_key, 0) + 1
|
|
581
|
+
report.total_scanned += 1
|
|
582
|
+
|
|
583
|
+
# --- Detect orphan nodes for Garbage Collection ---
|
|
584
|
+
report.deleted_node_ids = self._detect_deleted_nodes(project_uuid, current_files)
|
|
585
|
+
|
|
586
|
+
logger.info(
|
|
587
|
+
"Crawl completed: %d scanned, %d new, %d unchanged, %d deleted (GC).",
|
|
588
|
+
report.total_scanned,
|
|
589
|
+
len(report.new_files),
|
|
590
|
+
len(report.unchanged_files),
|
|
591
|
+
len(report.deleted_node_ids),
|
|
592
|
+
)
|
|
593
|
+
|
|
594
|
+
return report
|
|
595
|
+
|
|
596
|
+
def compute_file_hash(self, filepath: str) -> Optional[str]:
|
|
597
|
+
"""Calculates SHA256 of a file's content.
|
|
598
|
+
|
|
599
|
+
Reads the file in 64KB blocks for memory efficiency.
|
|
600
|
+
Returns None in case of read error (Semantic Fallback).
|
|
601
|
+
|
|
602
|
+
Args:
|
|
603
|
+
filepath: Absolute path to the file.
|
|
604
|
+
|
|
605
|
+
Returns:
|
|
606
|
+
Hexadecimal SHA256 hash (64 characters) or None if it failed.
|
|
607
|
+
"""
|
|
608
|
+
sha256 = hashlib.sha256()
|
|
609
|
+
try:
|
|
610
|
+
with open(filepath, "rb") as f:
|
|
611
|
+
while True:
|
|
612
|
+
block = f.read(self._HASH_BLOCK_SIZE)
|
|
613
|
+
if not block:
|
|
614
|
+
break
|
|
615
|
+
sha256.update(block)
|
|
616
|
+
return sha256.hexdigest()
|
|
617
|
+
except (OSError, IOError, PermissionError) as e:
|
|
618
|
+
logger.warning("Failed to calculate hash of %s: %s", filepath, e)
|
|
619
|
+
return None
|
|
620
|
+
|
|
621
|
+
def classify_file(self, filename: str) -> FileCategory:
|
|
622
|
+
"""Classifies a file by extension.
|
|
623
|
+
|
|
624
|
+
Args:
|
|
625
|
+
filename: File name (basename, e.g. 'main.py').
|
|
626
|
+
|
|
627
|
+
Returns:
|
|
628
|
+
Corresponding FileCategory (or UNKNOWN for unmapped extensions).
|
|
629
|
+
"""
|
|
630
|
+
ext = Path(filename).suffix.lower()
|
|
631
|
+
return EXTENSION_MAP.get(ext, FileCategory.UNKNOWN)
|
|
632
|
+
|
|
633
|
+
def _should_ignore_dir(self, dirname: str, relative_path: str) -> bool:
|
|
634
|
+
"""Checks if a directory should be ignored.
|
|
635
|
+
|
|
636
|
+
Evaluates against:
|
|
637
|
+
- DEFAULT_IGNORE_DIRS + customized ignore_dirs.
|
|
638
|
+
- Patterns from .gitignore.
|
|
639
|
+
- Hidden directories (start with . except .github, .husky).
|
|
640
|
+
|
|
641
|
+
Args:
|
|
642
|
+
dirname: Directory name (basename).
|
|
643
|
+
relative_path: Relative path to root.
|
|
644
|
+
|
|
645
|
+
Returns:
|
|
646
|
+
True if it should be ignored.
|
|
647
|
+
"""
|
|
648
|
+
# Directories from ignore list
|
|
649
|
+
if dirname in self._ignore_dirs:
|
|
650
|
+
return True
|
|
651
|
+
|
|
652
|
+
# Hidden directories (except useful exceptions)
|
|
653
|
+
ALLOWED_HIDDEN = {".github", ".husky", ".circleci", ".gitlab"}
|
|
654
|
+
if dirname.startswith(".") and dirname not in ALLOWED_HIDDEN:
|
|
655
|
+
return True
|
|
656
|
+
|
|
657
|
+
# Check .gitignore
|
|
658
|
+
if self._gitignore and self._gitignore.should_ignore(
|
|
659
|
+
relative_path.replace("\\", "/"), is_dir=True
|
|
660
|
+
):
|
|
661
|
+
return True
|
|
662
|
+
|
|
663
|
+
return False
|
|
664
|
+
|
|
665
|
+
def _detect_deleted_nodes(
|
|
666
|
+
self,
|
|
667
|
+
project_uuid: str,
|
|
668
|
+
current_files: set[str],
|
|
669
|
+
) -> list[int]:
|
|
670
|
+
"""Detects nodes in SQLite whose files no longer exist on disk.
|
|
671
|
+
|
|
672
|
+
Compares the relative paths of project nodes in SQLite with the list of files
|
|
673
|
+
existing on disk. Nodes belonging to files that are no longer in current_files
|
|
674
|
+
are marked for Garbage Collection.
|
|
675
|
+
|
|
676
|
+
SECURITY: Only nodes that are not directories or projects are deleted.
|
|
677
|
+
|
|
678
|
+
Args:
|
|
679
|
+
project_uuid: UUID of the project.
|
|
680
|
+
current_files: Set of relative paths of files existing on disk.
|
|
681
|
+
|
|
682
|
+
Returns:
|
|
683
|
+
List of orphan node_ids for removal.
|
|
684
|
+
"""
|
|
685
|
+
orphan_ids: list[int] = []
|
|
686
|
+
|
|
687
|
+
try:
|
|
688
|
+
# Fetch all project nodes from SQLite
|
|
689
|
+
all_nodes = self._store.get_nodes_by_project(project_uuid)
|
|
690
|
+
for node in all_nodes:
|
|
691
|
+
# Only nodes that are not directories or projects
|
|
692
|
+
if node.get("type") in ("directory", "cluster", "project"):
|
|
693
|
+
continue
|
|
694
|
+
|
|
695
|
+
label = node.get("label", "")
|
|
696
|
+
if not label:
|
|
697
|
+
continue
|
|
698
|
+
|
|
699
|
+
# The label is formatted as 'rel_path::symbol_name' or 'rel_path'
|
|
700
|
+
rel_path = label.split("::")[0]
|
|
701
|
+
|
|
702
|
+
# If the file no longer exists on disk → orphan node
|
|
703
|
+
if rel_path not in current_files:
|
|
704
|
+
orphan_ids.append(node["id"])
|
|
705
|
+
logger.debug(
|
|
706
|
+
"Orphan node detected (GC): id=%d, label=%s, deleted file=%s",
|
|
707
|
+
node["id"], label, rel_path,
|
|
708
|
+
)
|
|
709
|
+
|
|
710
|
+
except Exception as e:
|
|
711
|
+
logger.error("Falha na detecção de nós deletados (GC): %s", e)
|
|
712
|
+
|
|
713
|
+
if orphan_ids:
|
|
714
|
+
logger.warning(
|
|
715
|
+
"Garbage Collection: %d nós órfãos detectados no projeto %s.",
|
|
716
|
+
len(orphan_ids), project_uuid,
|
|
717
|
+
)
|
|
718
|
+
else:
|
|
719
|
+
logger.debug("Garbage Collection: nenhum nó órfão detectado.")
|
|
720
|
+
|
|
721
|
+
return orphan_ids
|
|
722
|
+
|