concierge-graph 3.8.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
ingestion/crawler.py ADDED
@@ -0,0 +1,722 @@
1
+ """
2
+ ingestion/crawler.py — Grafo Concierge v3.8.0 (Absolute Solidity)
3
+
4
+ Intelligent filesystem scanning for the Apex Ingestion Engine.
5
+
6
+ Responsibilities:
7
+ - Recursively traverse project directories.
8
+ - Respect ignore patterns (.gitignore + IGNORE_DIRS from config).
9
+ - Calculate SHA256 of each file for delta detection.
10
+ - Compare hashes with SqliteStore to skip unmodified files.
11
+ - Classify files by type (code, doc, config, conversation).
12
+ - Detect deleted files for Garbage Collection.
13
+
14
+ Integration:
15
+ - SqliteStore.find_node_by_hash(project_uuid, hash) → checks if already processed.
16
+ - SqliteStore.get_nodes_by_project(project_uuid) → lists existing nodes for GC.
17
+ - Result is a CrawlReport consumed by the Parser/Orchestrator.
18
+
19
+ Identity Preservation (Path-Agnostic ID):
20
+ The doc_id of each node is derived from the content (hash), not the path.
21
+ If the file moves to another folder, the hash changes → new node is created.
22
+ If the file is renamed without content changes → same hash → reuses the node.
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ import fnmatch
28
+ import hashlib
29
+ import logging
30
+ import os
31
+ from storage import SqliteStore
32
+ from dataclasses import dataclass, field
33
+ from enum import Enum
34
+ from pathlib import Path
35
+ from typing import Optional
36
+
37
+ logger = logging.getLogger("grafo-concierge.crawler")
38
+
39
+
40
+ # ---------------------------------------------------------------------------
41
+ # File classification
42
+ # ---------------------------------------------------------------------------
43
+
44
+ class FileCategory(str, Enum):
45
+ """File categories automatically detected by the Crawler.
46
+
47
+ Mapping per spec v3.8:
48
+ code → .py, .js, .ts, .go, .rs, .java, .cpp, .c, .rb
49
+ doc → .md, .txt, .rst, .adoc
50
+ config → .json, .yaml, .yml, .toml, .env, .ini, .cfg
51
+ conversation → .log, .chat
52
+ unknown → unmapped extensions
53
+ """
54
+ CODE = "code"
55
+ DOC = "doc"
56
+ CONFIG = "config"
57
+ CONVERSATION = "conversation"
58
+ UNKNOWN = "unknown"
59
+
60
+
61
+ # Extension -> category mapping (per API v3.8 table)
62
+ EXTENSION_MAP: dict[str, FileCategory] = {
63
+ # Code
64
+ ".py": FileCategory.CODE,
65
+ ".js": FileCategory.CODE,
66
+ ".ts": FileCategory.CODE,
67
+ ".tsx": FileCategory.CODE,
68
+ ".jsx": FileCategory.CODE,
69
+ ".go": FileCategory.CODE,
70
+ ".rs": FileCategory.CODE,
71
+ ".java": FileCategory.CODE,
72
+ ".cpp": FileCategory.CODE,
73
+ ".c": FileCategory.CODE,
74
+ ".h": FileCategory.CODE,
75
+ ".hpp": FileCategory.CODE,
76
+ ".rb": FileCategory.CODE,
77
+ ".cs": FileCategory.CODE,
78
+ ".swift": FileCategory.CODE,
79
+ ".kt": FileCategory.CODE,
80
+ ".php": FileCategory.CODE,
81
+ ".lua": FileCategory.CODE,
82
+ ".sh": FileCategory.CODE,
83
+ ".bash": FileCategory.CODE,
84
+ ".ps1": FileCategory.CODE,
85
+ ".sql": FileCategory.CODE,
86
+ ".r": FileCategory.CODE,
87
+ ".scala": FileCategory.CODE,
88
+ ".ex": FileCategory.CODE,
89
+ ".exs": FileCategory.CODE,
90
+ # Doc
91
+ ".md": FileCategory.DOC,
92
+ ".txt": FileCategory.DOC,
93
+ ".rst": FileCategory.DOC,
94
+ ".adoc": FileCategory.DOC,
95
+ ".org": FileCategory.DOC,
96
+ # Config
97
+ ".json": FileCategory.CONFIG,
98
+ ".yaml": FileCategory.CONFIG,
99
+ ".yml": FileCategory.CONFIG,
100
+ ".toml": FileCategory.CONFIG,
101
+ ".env": FileCategory.CONFIG,
102
+ ".ini": FileCategory.CONFIG,
103
+ ".cfg": FileCategory.CONFIG,
104
+ ".xml": FileCategory.CONFIG,
105
+ ".properties": FileCategory.CONFIG,
106
+ # Conversation
107
+ ".log": FileCategory.CONVERSATION,
108
+ ".chat": FileCategory.CONVERSATION,
109
+ }
110
+
111
+
112
+ # ---------------------------------------------------------------------------
113
+ # CrawlResult — individual result of a scanned file
114
+ # ---------------------------------------------------------------------------
115
+
116
+ @dataclass
117
+ class CrawlResult:
118
+ """Result of the scan of a single file.
119
+
120
+ Attributes:
121
+ absolute_path: Absolute path in the filesystem.
122
+ relative_path: Relative path to the project's source_path.
123
+ file_hash: SHA256 of the file content.
124
+ category: Automatic classification (code, doc, config, conversation).
125
+ extension: File extension (e.g. '.py').
126
+ size_bytes: Size in bytes.
127
+ is_new: True if the hash does not exist in SqliteStore (new or modified file).
128
+ existing_node_id: If not new, the ID of the existing node in SQLite.
129
+ """
130
+ absolute_path: str
131
+ relative_path: str
132
+ file_hash: str
133
+ category: FileCategory
134
+ extension: str
135
+ size_bytes: int
136
+ is_new: bool = True
137
+ existing_node_id: Optional[int] = None
138
+
139
+
140
+ @dataclass
141
+ class CrawlReport:
142
+ """Consolidated report of a crawl operation.
143
+
144
+ Attributes:
145
+ new_files: New or modified files (for processing).
146
+ unchanged_files: Files whose hash has not changed (skip).
147
+ deleted_node_ids: SQLite node IDs whose files no longer exist (GC).
148
+ categories: Count per category.
149
+ total_scanned: Total scanned files.
150
+ """
151
+ new_files: list[CrawlResult] = field(default_factory=list)
152
+ unchanged_files: list[CrawlResult] = field(default_factory=list)
153
+ deleted_node_ids: list[int] = field(default_factory=list)
154
+ categories: dict[str, int] = field(default_factory=dict)
155
+ total_scanned: int = 0
156
+
157
+
158
+ # ---------------------------------------------------------------------------
159
+ # GitignoreParser — robust parser for .gitignore
160
+ # ---------------------------------------------------------------------------
161
+
162
+ class GitignoreParser:
163
+ """Parser for .gitignore with support for common patterns.
164
+
165
+ Supports:
166
+ - Comments (# ...) and blank lines.
167
+ - Negation (! pattern → do not ignore).
168
+ - Explicit directories (dir/ → directories only).
169
+ - Wildcards (*, **, ?).
170
+ - Anchored patterns (starting with /).
171
+
172
+ Limitations:
173
+ - Does not support nested .gitignore in subdirectories (root only).
174
+ - Complex patterns with ranges [a-z] are treated as simple globs.
175
+ """
176
+
177
+ def __init__(self) -> None:
178
+ """Initializes the parser with empty lists."""
179
+ self._patterns: list[str] = []
180
+ self._negations: list[str] = []
181
+ self._dir_only_patterns: list[str] = []
182
+
183
+ def add_patterns(self, patterns: list[str]) -> None:
184
+ """Adds a list of ignore patterns programmatically.
185
+
186
+ Allows loading default safety patterns (DEFAULT_IGNORE_PATTERNS)
187
+ without depending on a file on disk. Follows the same semantics as .gitignore.
188
+
189
+ Args:
190
+ patterns: List of patterns in .gitignore format.
191
+ """
192
+ for raw in patterns:
193
+ stripped = raw.strip()
194
+ if not stripped or stripped.startswith("#"):
195
+ continue
196
+ if stripped.startswith("!"):
197
+ negation = stripped[1:].strip()
198
+ if negation:
199
+ self._negations.append(negation)
200
+ continue
201
+ if stripped.endswith("/"):
202
+ self._dir_only_patterns.append(stripped.rstrip("/"))
203
+ self._patterns.append(stripped.rstrip("/"))
204
+ self._patterns.append(stripped.rstrip("/") + "/**")
205
+ continue
206
+ if stripped.startswith("/"):
207
+ stripped = stripped[1:]
208
+ self._patterns.append(stripped)
209
+
210
+ logger.debug(
211
+ "GitignoreParser.add_patterns: +%d entradas → %d padrões totais.",
212
+ len(patterns), len(self._patterns),
213
+ )
214
+
215
+ def load(self, gitignore_path: str) -> None:
216
+ """Loads and parses a .gitignore file.
217
+
218
+ Ignores blank lines and comments.
219
+ Negation patterns (!) are stored separately.
220
+ Directory patterns (ending in /) are stored separately.
221
+
222
+ Args:
223
+ gitignore_path: Absolute path to the .gitignore.
224
+ """
225
+ try:
226
+ with open(gitignore_path, "r", encoding="utf-8", errors="replace") as f:
227
+ raw_lines = f.readlines()
228
+ except (OSError, IOError) as e:
229
+ logger.warning("Could not read .gitignore at %s: %s", gitignore_path, e)
230
+ return
231
+
232
+ for line in raw_lines:
233
+ # Remove trailing whitespace (keeps leading for space patterns)
234
+ stripped = line.rstrip()
235
+
236
+ # Ignores empty lines and comments
237
+ if not stripped or stripped.startswith("#"):
238
+ continue
239
+
240
+ # Remove trailing escaped space (\\ at the end)
241
+ if stripped.endswith("\\ "):
242
+ stripped = stripped[:-2] + " "
243
+
244
+ # Negation patterns
245
+ if stripped.startswith("!"):
246
+ negation = stripped[1:].strip()
247
+ if negation:
248
+ self._negations.append(negation)
249
+ continue
250
+
251
+ # Directory patterns (ending in /)
252
+ if stripped.endswith("/"):
253
+ self._dir_only_patterns.append(stripped.rstrip("/"))
254
+ # Also adds as normal pattern for directory contents
255
+ self._patterns.append(stripped.rstrip("/"))
256
+ self._patterns.append(stripped.rstrip("/") + "/**")
257
+ continue
258
+
259
+ # Remove leading slash (anchors to root, but we treat as relative)
260
+ if stripped.startswith("/"):
261
+ stripped = stripped[1:]
262
+
263
+ self._patterns.append(stripped)
264
+
265
+ logger.debug(
266
+ "GitignoreParser: %d patterns loaded, %d negations, %d dir-only",
267
+ len(self._patterns), len(self._negations), len(self._dir_only_patterns),
268
+ )
269
+
270
+ def should_ignore(self, relative_path: str, is_dir: bool = False) -> bool:
271
+ """Verifies if a relative path should be ignored.
272
+
273
+ Logic follows Git semantics:
274
+ 1. If the path matches any ignore pattern → True.
275
+ 2. If the path matches a negation pattern → False (override).
276
+ 3. Directory-only patterns only apply to directories.
277
+
278
+ Args:
279
+ relative_path: Relative path to the project root (forward slashes).
280
+ is_dir: True if the path is a directory.
281
+
282
+ Returns:
283
+ True if it should be ignored.
284
+ """
285
+ # Normalize to forward slashes
286
+ normalized = relative_path.replace("\\", "/")
287
+ # Extract basename for simple matching (e.g. 'node_modules')
288
+ basename = normalized.rsplit("/", 1)[-1] if "/" in normalized else normalized
289
+
290
+ # Check negations first (override)
291
+ for neg_pattern in self._negations:
292
+ if self._matches(normalized, neg_pattern) or self._matches(basename, neg_pattern):
293
+ return False
294
+
295
+ # Check directory patterns
296
+ if is_dir:
297
+ for dir_pattern in self._dir_only_patterns:
298
+ if self._matches(normalized, dir_pattern) or self._matches(basename, dir_pattern):
299
+ return True
300
+
301
+ # Check general patterns
302
+ for pattern in self._patterns:
303
+ if self._matches(normalized, pattern) or self._matches(basename, pattern):
304
+ return True
305
+
306
+ return False
307
+
308
+ @staticmethod
309
+ def _matches(path: str, pattern: str) -> bool:
310
+ """Checks if a path matches a glob pattern.
311
+
312
+ Supports:
313
+ - fnmatch wildcards (*, ?, [seq])
314
+ - ** (recursive glob — match of any depth)
315
+
316
+ Args:
317
+ path: Normalized path (forward slashes).
318
+ pattern: Glob pattern.
319
+
320
+ Returns:
321
+ True if the path matches the pattern.
322
+ """
323
+ # ** → any subdirectory
324
+ if "**" in pattern:
325
+ # Transforms 'dir/**/file' into recursive match
326
+ # Splits by '**' and checks the parts
327
+ parts = pattern.split("**")
328
+ if len(parts) == 2:
329
+ prefix = parts[0].rstrip("/")
330
+ suffix = parts[1].lstrip("/")
331
+ if prefix and not path.startswith(prefix):
332
+ return False
333
+ if suffix and not fnmatch.fnmatch(path.rsplit("/", 1)[-1], suffix):
334
+ return False
335
+ if prefix and suffix:
336
+ return True
337
+ if not prefix and suffix:
338
+ return fnmatch.fnmatch(path.rsplit("/", 1)[-1], suffix)
339
+ if prefix and not suffix:
340
+ return path.startswith(prefix)
341
+ return True # ** alone → ignores everything
342
+
343
+ # Pattern contains / → match against full path
344
+ if "/" in pattern:
345
+ return fnmatch.fnmatch(path, pattern)
346
+
347
+ # Simple pattern → match against each component of the path
348
+ components = path.split("/")
349
+ for component in components:
350
+ if fnmatch.fnmatch(component, pattern):
351
+ return True
352
+
353
+ return False
354
+
355
+
356
+ # ---------------------------------------------------------------------------
357
+ # ProjectCrawler — Scanning engine
358
+ # ---------------------------------------------------------------------------
359
+
360
+ class ProjectCrawler:
361
+ """Intelligent filesystem scanning with Delta Detection.
362
+
363
+ Flow:
364
+ 1. Loads ignore patterns (.gitignore + IGNORE_DIRS).
365
+ 2. Recursively traverses the source_path.
366
+ 3. Calculates SHA256 of each file.
367
+ 4. Queries SqliteStore to check if the hash already exists.
368
+ 5. Classifies file by extension.
369
+ 6. Detects orphan nodes (deleted files) for Garbage Collection.
370
+
371
+ Args:
372
+ sqlite_store: SqliteStore instance for hash queries.
373
+ ignore_dirs: List of extra directories to ignore (merge with DEFAULT_IGNORE_DIRS).
374
+ """
375
+
376
+ # Ignored directories by default (per ConciergeConfig.IGNORE_DIRS)
377
+ DEFAULT_IGNORE_DIRS: set[str] = {
378
+ ".git", "node_modules", ".next", "dist", "build",
379
+ "__pycache__", ".venv", "venv", ".env", ".idea",
380
+ ".vscode", ".mypy_cache", ".pytest_cache", "coverage",
381
+ ".tox", "egg-info", ".eggs", ".cache", ".gradle",
382
+ "target", "vendor", "bower_components",
383
+ }
384
+
385
+ # Ignored FILE patterns by default (zero-config security)
386
+ DEFAULT_IGNORE_PATTERNS: list[str] = [
387
+ # Security — Credentials and Keys
388
+ ".env", ".env.*",
389
+ "*.pem", "*.key", "*.cert", "*.der", "*.pfx", "*.p12",
390
+ "id_rsa", "id_dsa", "id_ed25519",
391
+ # Lock files — Pure noise
392
+ "package-lock.json", "yarn.lock", "pnpm-lock.yaml",
393
+ "poetry.lock", "Cargo.lock", "composer.lock", "Gemfile.lock",
394
+ # Logs and junk text
395
+ "*.log", "*.txt",
396
+ # Local databases
397
+ "*.db", "*.sqlite", "*.sqlite3",
398
+ # Binaries and compilation
399
+ "*.exe", "*.dll", "*.so", "*.dylib", "*.bin", "*.o", "*.a",
400
+ "*.pyc", "*.pyo", "*.class", "*.wasm",
401
+ # Compressed files
402
+ "*.zip", "*.tar", "*.tar.gz", "*.tgz", "*.rar", "*.7z", "*.bz2",
403
+ # Media and Images
404
+ "*.jpg", "*.jpeg", "*.png", "*.gif", "*.ico", "*.svg",
405
+ "*.mp3", "*.mp4", "*.wav", "*.avi", "*.mov", "*.webp",
406
+ "*.ttf", "*.woff", "*.woff2", "*.eot",
407
+ # OS junk
408
+ ".DS_Store", "Thumbs.db", "desktop.ini",
409
+ ]
410
+
411
+ # Maximum file size for ingestion (10 MB)
412
+ MAX_FILE_SIZE_BYTES: int = 10 * 1024 * 1024
413
+
414
+ # Read block size for hash (64 KB — memory efficiency)
415
+ _HASH_BLOCK_SIZE: int = 65536
416
+
417
+ def __init__(
418
+ self,
419
+ sqlite_store: "SqliteStore",
420
+ ignore_dirs: Optional[set[str]] = None,
421
+ ) -> None:
422
+ """Initializes the Crawler.
423
+
424
+ Args:
425
+ sqlite_store: SQLite facade for querying existing hashes.
426
+ ignore_dirs: Extra directories to ignore (merge with DEFAULT_IGNORE_DIRS).
427
+ """
428
+ self._store = sqlite_store
429
+ self._ignore_dirs = self.DEFAULT_IGNORE_DIRS.copy()
430
+ if ignore_dirs:
431
+ self._ignore_dirs.update(ignore_dirs)
432
+
433
+ # GitignoreParser will be loaded by crawl() if .gitignore exists
434
+ self._gitignore: Optional[GitignoreParser] = None
435
+
436
+ logger.info(
437
+ "ProjectCrawler initialized: %d active ignore patterns.",
438
+ len(self._ignore_dirs),
439
+ )
440
+
441
+ def crawl(
442
+ self,
443
+ source_path: str,
444
+ project_uuid: str,
445
+ ) -> CrawlReport:
446
+ """Executes a complete scan of a project directory.
447
+
448
+ Flow:
449
+ 1. Validates source_path.
450
+ 2. Loads .gitignore (if it exists).
451
+ 3. Traverses files recursively.
452
+ 4. For each file: hash → classify → delta check.
453
+ 5. Detects orphan nodes (Garbage Collection).
454
+ 6. Returns CrawlReport with new_files, unchanged, and deleted.
455
+
456
+ Args:
457
+ source_path: Root directory of the project to scan.
458
+ project_uuid: Project UUID (for node filtering in SQLite).
459
+
460
+ Returns:
461
+ CrawlReport with the complete scan inventory.
462
+
463
+ Raises:
464
+ FileNotFoundError: If source_path does not exist.
465
+ NotADirectoryError: If source_path is not a directory.
466
+ """
467
+ root = Path(source_path).resolve()
468
+
469
+ # --- Input validation ---
470
+ if not root.exists():
471
+ raise FileNotFoundError(f"source_path does not exist: {source_path}")
472
+ if not root.is_dir():
473
+ raise NotADirectoryError(f"source_path is not a directory: {source_path}")
474
+
475
+ logger.info("Crawl started: %s (project: %s)", root, project_uuid)
476
+
477
+ # --- Load ignore patterns (3 layers) ---
478
+ self._gitignore = GitignoreParser()
479
+
480
+ # Layer 1: Default safety patterns (zero-config)
481
+ self._gitignore.add_patterns(self.DEFAULT_IGNORE_PATTERNS)
482
+ logger.info("Default safety patterns loaded: %d rules.", len(self.DEFAULT_IGNORE_PATTERNS))
483
+
484
+ # Layer 2: Project .gitignore
485
+ gitignore_file = root / ".gitignore"
486
+ if gitignore_file.is_file():
487
+ self._gitignore.load(str(gitignore_file))
488
+ logger.info(".gitignore found and loaded: %s", gitignore_file)
489
+
490
+ # Layer 3: .conciergeignore (Grafo Concierge specific extra rules)
491
+ concierge_ignore_file = root / ".conciergeignore"
492
+ if concierge_ignore_file.is_file():
493
+ self._gitignore.load(str(concierge_ignore_file))
494
+ logger.info(".conciergeignore found and loaded: %s", concierge_ignore_file)
495
+
496
+ # --- Traverse recursively ---
497
+ report = CrawlReport()
498
+ current_hashes: set[str] = set()
499
+ current_files: set[str] = set()
500
+
501
+ for dirpath, dirnames, filenames in os.walk(str(root), topdown=True):
502
+ current_dir = Path(dirpath)
503
+ relative_dir = current_dir.relative_to(root)
504
+
505
+ # Filter directories IN-PLACE (os.walk topdown=True respects this)
506
+ dirnames[:] = [
507
+ d for d in dirnames
508
+ if not self._should_ignore_dir(d, str(relative_dir / d))
509
+ ]
510
+
511
+ for filename in filenames:
512
+ filepath = current_dir / filename
513
+ relative_filepath = str(relative_dir / filename).replace("\\", "/")
514
+
515
+ # Skip hidden files (start with .)
516
+ if filename.startswith("."):
517
+ continue
518
+
519
+ # Check .gitignore
520
+ if self._gitignore and self._gitignore.should_ignore(relative_filepath, is_dir=False):
521
+ logger.debug("Ignored (.gitignore): %s", relative_filepath)
522
+ continue
523
+
524
+ # Check if it is a regular and readable file
525
+ if not filepath.is_file():
526
+ continue
527
+
528
+ # Check maximum size
529
+ try:
530
+ size_bytes = filepath.stat().st_size
531
+ except OSError as e:
532
+ logger.warning("Could not read stat of %s: %s", filepath, e)
533
+ continue
534
+
535
+ if size_bytes > self.MAX_FILE_SIZE_BYTES:
536
+ logger.debug(
537
+ "File exceeds MAX_FILE_SIZE (%d bytes): %s",
538
+ size_bytes, relative_filepath,
539
+ )
540
+ continue
541
+
542
+ # Skip empty files (0 bytes)
543
+ if size_bytes == 0:
544
+ continue
545
+
546
+ # --- Compute SHA256 ---
547
+ file_hash = self.compute_file_hash(str(filepath))
548
+ if file_hash is None:
549
+ # Reading error — skip (already logged by compute_file_hash)
550
+ continue
551
+
552
+ current_hashes.add(file_hash)
553
+ current_files.add(relative_filepath)
554
+
555
+ # --- Classify ---
556
+ category = self.classify_file(filename)
557
+ extension = Path(filename).suffix.lower()
558
+
559
+ # --- Delta Check via SqliteStore ---
560
+ existing_node = self._store.find_node_by_hash(project_uuid, file_hash)
561
+
562
+ result = CrawlResult(
563
+ absolute_path=str(filepath),
564
+ relative_path=relative_filepath,
565
+ file_hash=file_hash,
566
+ category=category,
567
+ extension=extension,
568
+ size_bytes=size_bytes,
569
+ is_new=(existing_node is None),
570
+ existing_node_id=existing_node["id"] if existing_node else None,
571
+ )
572
+
573
+ if result.is_new:
574
+ report.new_files.append(result)
575
+ else:
576
+ report.unchanged_files.append(result)
577
+
578
+ # Count per category
579
+ cat_key = category.value
580
+ report.categories[cat_key] = report.categories.get(cat_key, 0) + 1
581
+ report.total_scanned += 1
582
+
583
+ # --- Detect orphan nodes for Garbage Collection ---
584
+ report.deleted_node_ids = self._detect_deleted_nodes(project_uuid, current_files)
585
+
586
+ logger.info(
587
+ "Crawl completed: %d scanned, %d new, %d unchanged, %d deleted (GC).",
588
+ report.total_scanned,
589
+ len(report.new_files),
590
+ len(report.unchanged_files),
591
+ len(report.deleted_node_ids),
592
+ )
593
+
594
+ return report
595
+
596
+ def compute_file_hash(self, filepath: str) -> Optional[str]:
597
+ """Calculates SHA256 of a file's content.
598
+
599
+ Reads the file in 64KB blocks for memory efficiency.
600
+ Returns None in case of read error (Semantic Fallback).
601
+
602
+ Args:
603
+ filepath: Absolute path to the file.
604
+
605
+ Returns:
606
+ Hexadecimal SHA256 hash (64 characters) or None if it failed.
607
+ """
608
+ sha256 = hashlib.sha256()
609
+ try:
610
+ with open(filepath, "rb") as f:
611
+ while True:
612
+ block = f.read(self._HASH_BLOCK_SIZE)
613
+ if not block:
614
+ break
615
+ sha256.update(block)
616
+ return sha256.hexdigest()
617
+ except (OSError, IOError, PermissionError) as e:
618
+ logger.warning("Failed to calculate hash of %s: %s", filepath, e)
619
+ return None
620
+
621
+ def classify_file(self, filename: str) -> FileCategory:
622
+ """Classifies a file by extension.
623
+
624
+ Args:
625
+ filename: File name (basename, e.g. 'main.py').
626
+
627
+ Returns:
628
+ Corresponding FileCategory (or UNKNOWN for unmapped extensions).
629
+ """
630
+ ext = Path(filename).suffix.lower()
631
+ return EXTENSION_MAP.get(ext, FileCategory.UNKNOWN)
632
+
633
+ def _should_ignore_dir(self, dirname: str, relative_path: str) -> bool:
634
+ """Checks if a directory should be ignored.
635
+
636
+ Evaluates against:
637
+ - DEFAULT_IGNORE_DIRS + customized ignore_dirs.
638
+ - Patterns from .gitignore.
639
+ - Hidden directories (start with . except .github, .husky).
640
+
641
+ Args:
642
+ dirname: Directory name (basename).
643
+ relative_path: Relative path to root.
644
+
645
+ Returns:
646
+ True if it should be ignored.
647
+ """
648
+ # Directories from ignore list
649
+ if dirname in self._ignore_dirs:
650
+ return True
651
+
652
+ # Hidden directories (except useful exceptions)
653
+ ALLOWED_HIDDEN = {".github", ".husky", ".circleci", ".gitlab"}
654
+ if dirname.startswith(".") and dirname not in ALLOWED_HIDDEN:
655
+ return True
656
+
657
+ # Check .gitignore
658
+ if self._gitignore and self._gitignore.should_ignore(
659
+ relative_path.replace("\\", "/"), is_dir=True
660
+ ):
661
+ return True
662
+
663
+ return False
664
+
665
+ def _detect_deleted_nodes(
666
+ self,
667
+ project_uuid: str,
668
+ current_files: set[str],
669
+ ) -> list[int]:
670
+ """Detects nodes in SQLite whose files no longer exist on disk.
671
+
672
+ Compares the relative paths of project nodes in SQLite with the list of files
673
+ existing on disk. Nodes belonging to files that are no longer in current_files
674
+ are marked for Garbage Collection.
675
+
676
+ SECURITY: Only nodes that are not directories or projects are deleted.
677
+
678
+ Args:
679
+ project_uuid: UUID of the project.
680
+ current_files: Set of relative paths of files existing on disk.
681
+
682
+ Returns:
683
+ List of orphan node_ids for removal.
684
+ """
685
+ orphan_ids: list[int] = []
686
+
687
+ try:
688
+ # Fetch all project nodes from SQLite
689
+ all_nodes = self._store.get_nodes_by_project(project_uuid)
690
+ for node in all_nodes:
691
+ # Only nodes that are not directories or projects
692
+ if node.get("type") in ("directory", "cluster", "project"):
693
+ continue
694
+
695
+ label = node.get("label", "")
696
+ if not label:
697
+ continue
698
+
699
+ # The label is formatted as 'rel_path::symbol_name' or 'rel_path'
700
+ rel_path = label.split("::")[0]
701
+
702
+ # If the file no longer exists on disk → orphan node
703
+ if rel_path not in current_files:
704
+ orphan_ids.append(node["id"])
705
+ logger.debug(
706
+ "Orphan node detected (GC): id=%d, label=%s, deleted file=%s",
707
+ node["id"], label, rel_path,
708
+ )
709
+
710
+ except Exception as e:
711
+ logger.error("Falha na detecção de nós deletados (GC): %s", e)
712
+
713
+ if orphan_ids:
714
+ logger.warning(
715
+ "Garbage Collection: %d nós órfãos detectados no projeto %s.",
716
+ len(orphan_ids), project_uuid,
717
+ )
718
+ else:
719
+ logger.debug("Garbage Collection: nenhum nó órfão detectado.")
720
+
721
+ return orphan_ids
722
+