fluidattacks_core_filesystem 12.0.0__py3-none-any.whl

fluidattacks_core/filesystem/__init__.py

@@ -0,0 +1,18 @@
+from .detection import detect_languages_in_dir
+from .language import Language
+from .partitioning import build_part_tree, decide_part_tree
+from .parts import Part, PartIdentity, PartTree
+from .projects import detect_root_language, find_parent_project, find_projects
+
+__all__ = [
+    "Language",
+    "Part",
+    "PartIdentity",
+    "PartTree",
+    "build_part_tree",
+    "decide_part_tree",
+    "detect_languages_in_dir",
+    "detect_root_language",
+    "find_parent_project",
+    "find_projects",
+]

fluidattacks_core/filesystem/defaults.py

@@ -0,0 +1,517 @@
+from typing import TypedDict
+
+from .language import Language
+
+__all__ = [
+    "CONFIG_MARKERS",
+    "EXCLUDE_DIRS",
+    "EXCLUDE_FILES",
+    "EXTENDED_EXCLUDE_DIRS",
+    "MANIFEST_MARKERS",
+    "SKIP_DIRS",
+    "TEST_DATA_DIRS",
+    "TEST_FILES",
+    "TEST_GLOBS",
+    "VCS_DIRS",
+    "Language",
+    "MarkerSet",
+]
+
+
+class MarkerSet(TypedDict):
+    names: set[str]
+    globs: list[str]
+
+
+CONFIG_MARKERS: dict[Language, MarkerSet] = {
+    Language.Java: {
+        "names": {
+            "pom.xml",
+            "settings.gradle",
+            "settings.gradle.kts",
+            "build.gradle",
+            "build.gradle.kts",
+            "gradle.properties",
+            "gradlew",
+            "gradlew.bat",
+            "mvnw",
+            "mvnw.cmd",
+            "application.properties",
+            "application.yml",
+            "application.yaml",
+            "log4j.properties",
+            "log4j2.xml",
+            "logback.xml",
+            "web.xml",
+            "hibernate.cfg.xml",
+            "persistence.xml",
+            "MANIFEST.MF",
+            "sonar-project.properties",
+            "checkstyle.xml",
+            "spotbugs.xml",
+            "pmd.xml",
+            "findbugs.xml",
+            "junit-platform.properties",
+            "testng.xml",
+            "spring.factories",
+            "spring.profiles",
+            "bootstrap.properties",
+            "bootstrap.yml",
+        },
+        "globs": [
+            "*.gradle",
+            "*.gradle.kts",
+            "application-*.properties",
+            "application-*.yml",
+            "application-*.yaml",
+            "logback-*.xml",
+            "log4j2-*.xml",
+        ],
+    },
+    Language.Python: {
+        "names": {
+            "pyproject.toml",
+            "requirements.txt",
+            "requirements.in",
+            "requirements-dev.txt",
+            "requirements-test.txt",
+            "constraints.txt",
+            "setup.py",
+            "setup.cfg",
+            "Pipfile",
+            "Pipfile.lock",
+            "poetry.lock",
+            "poetry.toml",
+            "hatch.toml",
+            "uv.lock",
+            "uv.toml",
+            "tox.ini",
+            "pytest.ini",
+            "mypy.ini",
+            ".pylintrc",
+            "pylintrc",
+            ".flake8",
+            ".isort.cfg",
+            "ruff.toml",
+            ".ruff.toml",
+            ".coveragerc",
+            "coverage.ini",
+            "noxfile.py",
+            "pyrightconfig.json",
+            "MANIFEST.in",
+            "py.typed",
+            "tox.toml",
+        },
+        "globs": [
+            "requirements*.txt",
+            "requirements*.in",
+            "constraints*.txt",
+            "setup.cfg",
+            "*.ini",
+        ],
+    },
+    Language.JavaScript: {
+        "names": {
+            "package.json",
+            "package-lock.json",
+            "yarn.lock",
+            "pnpm-lock.yaml",
+            "bun.lockb",
+            "bunfig.toml",
+            "nx.json",
+            "lerna.json",
+            "turbo.json",
+            "angular.json",
+            "workspace.json",
+            "gulpfile.js",
+            "gulpfile.ts",
+            "gruntfile.js",
+            "gruntfile.coffee",
+            "karma.conf.js",
+            "karma.conf.coffee",
+            "protractor.conf.js",
+            "cypress.json",
+            "cypress.config.js",
+            "playwright.config.js",
+            "playwright.config.ts",
+            "vitest.config.js",
+            "vitest.config.ts",
+            "storybook",
+            ".storybook",
+        },
+        "globs": [
+            "vite.config.*",
+            "next.config.*",
+            "nuxt.config.*",
+            "svelte.config.*",
+            "astro.config.*",
+            "rollup.config.*",
+            "webpack.config.*",
+            "babel.config.*",
+            ".babelrc*",
+            "eslint.config.*",
+            ".eslintrc*",
+            ".prettierrc*",
+            "prettier.config.*",
+            "jest.config.*",
+            "vitest.config.*",
+            "cypress.config.*",
+            "playwright.config.*",
+            "karma.conf.*",
+            "protractor.conf.*",
+            "*.config.js",
+            "*.config.ts",
+            "*.config.mjs",
+            "*.config.cjs",
+        ],
+    },
+    Language.TypeScript: {
+        "names": {
+            "tsconfig.json",
+            "tsconfig.base.json",
+            "tsconfig.app.json",
+            "tsconfig.build.json",
+            "tsconfig.node.json",
+            "tsconfig.lib.json",
+            "tsconfig.spec.json",
+            "tsconfig.test.json",
+            "deno.json",
+            "deno.jsonc",
+            "tsconfig.eslint.json",
+        },
+        "globs": [
+            "tsconfig.*.json",
+            "tsconfig-*.json",
+        ],
+    },
+    Language.PHP: {
+        "names": {
+            "composer.json",
+            "composer.lock",
+            "php.ini",
+            "phpunit.xml",
+            "phpunit.xml.dist",
+            "phpcs.xml",
+            "phpcs.xml.dist",
+            "phpmd.xml",
+            "phpstan.neon",
+            "phpstan.neon.dist",
+            "psalm.xml",
+            "psalm.xml.dist",
+            ".php-cs-fixer.php",
+            ".php_cs",
+            ".php_cs.dist",
+            "codeception.yml",
+            "symfony.lock",
+            "artisan",
+            "wp-config.php",
+            "config.php",
+        },
+        "globs": [
+            "phpspec.yml*",
+        ],
+    },
+    Language.CSharp: {
+        "names": {
+            "app.config",
+            "web.config",
+            "packages.config",
+            "nuget.config",
+            "NuGet.Config",
+            "Directory.Build.props",
+            "Directory.Build.targets",
+            "Directory.Packages.props",
+            "global.json",
+            "appsettings.json",
+        },
+        "globs": [
+            "*.sln",
+            "*.csproj",
+            "*.fsproj",
+            "*.vbproj",
+            "appsettings.*.json",
+        ],
+    },
+    Language.Go: {
+        "names": {
+            "go.mod",
+            "go.sum",
+            "go.work",
+            "go.work.sum",
+            ".golangci.yml",
+            ".golangci.yaml",
+            "Gopkg.toml",
+            "Gopkg.lock",
+            "glide.yaml",
+            "glide.lock",
+            ".air.toml",
+            "air.toml",
+            "goreleaser.yml",
+            "goreleaser.yaml",
+        },
+        "globs": [
+            "*.mk",
+        ],
+    },
+    Language.Ruby: {
+        "names": {
+            "Gemfile",
+            "Gemfile.lock",
+            ".ruby-version",
+            ".rubocop.yml",
+            ".reek",
+            ".rspec",
+            ".overcommit.yml",
+            "Rakefile",
+            "Capfile",
+            "Guardfile",
+            "config.ru",
+            "database.yml",
+            "boot.rb",
+        },
+        "globs": [],
+    },
+    Language.Kotlin: {
+        "names": {
+            "build.gradle.kts",
+            "settings.gradle.kts",
+        },
+        "globs": [],
+    },
+    Language.Swift: {
+        "names": {
+            "Package.swift",
+            "Package.resolved",
+            "Config.xcconfig",
+            "Info.plist",
+            "project.pbxproj",
+            ".swiftlint.yml",
+            ".swiftformat",
+            "Cartfile",
+            "Cartfile.resolved",
+            "Podfile",
+            "Podfile.lock",
+        },
+        "globs": [],
+    },
+    Language.Scala: {
+        "names": {
+            "build.sbt",
+            "build.scala",
+            "project/build.properties",
+            "project/plugins.sbt",
+            "project/Dependencies.scala",
+            "project/Assembly.scala",
+            "mill",
+            "mill.bat",
+            "millw",
+            "millw.bat",
+        },
+        "globs": [
+            "*.sbt",
+            "project/*.scala",
+            "project/*.sbt",
+        ],
+    },
+    Language.Rust: {
+        "names": {
+            "Cargo.toml",
+            "Cargo.lock",
+            "rust-toolchain",
+            "rust-toolchain.toml",
+            "rustfmt.toml",
+            "clippy.toml",
+            "Cargo.toml.orig",
+        },
+        "globs": [],
+    },
+    Language.Dart: {
+        "names": {
+            "pubspec.yaml",
+            "pubspec.lock",
+            "analysis_options.yaml",
+            "dart_test.yaml",
+            "build.yaml",
+            "melos.yaml",
+            "melos.yml",
+        },
+        "globs": [
+            "pubspec*.yaml",
+            "pubspec*.yml",
+        ],
+    },
+}
+
+VCS_DIRS = [
+    ".git",
+    ".hg",
+    ".svn",
+]
+
+
+SKIP_DIRS = {
+    *VCS_DIRS,
+    ".idea",
+    ".vscode",
+    "__pycache__",
+    ".mypy_cache",
+    ".pytest_cache",
+    "*cache*",
+    "node_modules",
+    ".venv",
+    "venv",
+    "dist",
+    "build",
+    "target",
+    "*test*",
+    "out",
+    "bin",
+    "obj",
+}
+
+
+TEST_GLOBS = {
+    Language.Python: [
+        "test*",  # Python
+        "*test.py",  # Python
+    ],
+    Language.JavaScript: [
+        "*.spec.js",  # JavaScript
+        "*.test.js",  # JavaScript
+    ],
+    Language.TypeScript: [
+        "*.spec.ts",  # TypeScript
+        "*.test.ts",  # TypeScript
+    ],
+    Language.Java: [
+        "*Test.java",  # Java
+    ],
+    Language.Kotlin: [
+        "*Test.kt",  # Kotlin
+    ],
+    Language.Go: [
+        "*_test.go",  # Go
+    ],
+    Language.Ruby: [
+        "*_test.rb",  # Ruby
+    ],
+    Language.CSharp: [
+        "*.Tests.cs",  # C#
+    ],
+    Language.Swift: [
+        "*Test.swift",  # Swift
+    ],
+}
+
+
+EXCLUDE_DIRS = [
+    "assets",
+    "target",
+    "build",
+    ".gradle",
+    ".mvn",
+    ".idea",
+    "out",
+    "tests",
+    "__tests__",
+    "spec",
+    ".venv",
+    "__pycache__",
+    "dist",
+    ".tox",
+    ".pytest_cache",
+    ".mypy_cache",
+    "node_modules",
+    ".cache",
+    "vendor",
+    ".composer",
+    "bin",
+    "obj",
+    ".vs",
+    "packages",
+    ".vscode",
+    "pkg",
+    ".bundle",
+    "tmp",
+    ".build",
+    "Carthage",
+    "Pods",
+    "DerivedData",
+    "cache",
+    ".ruff_cache",
+    ".metals",
+    ".bloop",
+    "wwwroot",
+    "plugin",
+    "plugins",
+    "libraries",
+    ".stack-work",
+    "dist-newstyle",
+    ".dart_tool",
+    "deps",
+    "_build",
+    "checkouts",
+]
+
+
+EXCLUDE_FILES = [
+    "*.min.js",
+    "*.min.css",
+    "*.jar",
+    "*.war",
+    "*.ear",
+    "*.jmod",
+    "*.nupkg",
+    "*.gem",
+    "*.phar*",
+    "*.so",
+    "*.dll",
+    "*.dylib",
+    "*.a",
+    "*.lib",
+    "*.rlib",
+    "*.rs.bk",
+    "*mock*",
+]
+
+
+TEST_FILES = [x for item in TEST_GLOBS.values() for x in item]
+
+
+MANIFEST_MARKERS: dict[Language, frozenset[str]] = {
+    Language.Java: frozenset({"pom.xml", "build.gradle", "settings.gradle"}),
+    Language.Python: frozenset({"pyproject.toml", "setup.py", "Pipfile", "requirements.txt"}),
+    Language.JavaScript: frozenset({"package.json"}),
+    Language.TypeScript: frozenset({"tsconfig.json"}),
+    Language.PHP: frozenset({"composer.json"}),
+    Language.CSharp: frozenset({"Directory.Build.props", "Directory.Build.targets", "global.json"}),
+    Language.Go: frozenset({"go.mod", "Gopkg.toml", "glide.yaml"}),
+    Language.Ruby: frozenset({"Gemfile"}),
+    Language.Kotlin: frozenset({"build.gradle.kts", "settings.gradle.kts"}),
+    Language.Swift: frozenset({"Package.swift", "Podfile", "Cartfile", "project.pbxproj"}),
+    Language.Rust: frozenset({"Cargo.toml"}),
+    Language.Dart: frozenset({"pubspec.yaml"}),
+    Language.Scala: frozenset({"build.sbt"}),
+}
+
+
+TEST_DATA_DIRS = [
+    "test-data",
+    "testdata",
+    "test_data",
+    "fixtures",
+    "__fixtures__",
+    "fixture",
+    "mocks",
+    "__mocks__",
+    "*mock*",
+    "testfiles",
+    "test_files",
+    "sample_data",
+    "samples",
+    "examples",
+    "test",
+]
+
+
+EXTENDED_EXCLUDE_DIRS = [*EXCLUDE_DIRS, *TEST_DATA_DIRS, *VCS_DIRS]

fluidattacks_core/filesystem/detection.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+from fnmatch import fnmatch
+from typing import TYPE_CHECKING
+
+from .defaults import CONFIG_MARKERS, MANIFEST_MARKERS, MarkerSet
+from .language import Language
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+__all__ = ["detect_anchor_language", "detect_languages_in_dir", "is_config_marker"]
+
+_FOLDED_MARKER_NAMES = frozenset(
+    name.casefold() for markers in CONFIG_MARKERS.values() for name in markers["names"]
+)
+
+_MARKER_GLOBS = tuple(pat for markers in CONFIG_MARKERS.values() for pat in markers["globs"])
+
+_FOLDED_MANIFESTS = {
+    language: frozenset(name.casefold() for name in names)
+    for language, names in MANIFEST_MARKERS.items()
+}
+
+_LANGUAGE_PRIORITIES = {
+    Language.TypeScript: 10,
+    Language.JavaScript: 5,
+    Language.Kotlin: 8,
+    Language.Java: 6,
+    Language.Rust: 9,
+    Language.C: 4,
+    Language.Scala: 7,
+    Language.Dart: 8,
+    Language.Python: 7,
+    Language.Go: 8,
+    Language.PHP: 6,
+    Language.Ruby: 6,
+    Language.CSharp: 7,
+    Language.Swift: 8,
+}
+
+_LANGUAGE_CONFLICTS = {
+    Language.TypeScript: {Language.JavaScript},
+    Language.JavaScript: {Language.TypeScript, Language.Dart},
+    Language.Kotlin: {Language.Java, Language.Scala},
+    Language.Java: {Language.Kotlin, Language.Scala},
+    Language.Scala: {Language.Java, Language.Kotlin},
+    Language.Rust: {Language.C},
+    Language.C: {Language.Rust},
+    Language.Dart: {Language.JavaScript},
+}
+
+
+def _get_language_conflicts(lang: Language) -> set[Language]:
+    return _LANGUAGE_CONFLICTS.get(lang, set())
+
+
+def _calculate_language_score(names: set[str], markers: MarkerSet) -> int:
+    exacts = {n.casefold() for n in markers["names"]}
+    patterns = markers["globs"]
+    score = len([n for n in exacts if n in names]) * 10
+    glob_matches = sum(1 for f in names if any(fnmatch(f, pat) for pat in patterns))
+    return score + glob_matches * 5
+
+
+def _resolve_language_conflicts(
+    hits: list[Language],
+    language_scores: dict[Language, int],
+) -> list[Language]:
+    if len(hits) <= 1:
+        return hits
+    ordered = sorted(
+        hits,
+        key=lambda lang: (_LANGUAGE_PRIORITIES.get(lang, 0), language_scores.get(lang, 0)),
+        reverse=True,
+    )
+    resolved_hits: list[Language] = []
+    for lang in ordered:
+        conflicts = _get_language_conflicts(lang)
+        if not any(conflict in resolved_hits for conflict in conflicts):
+            resolved_hits.append(lang)
+    return resolved_hits
+
+
+def detect_languages_in_dir(files: Iterable[str]) -> list[Language]:
+    names = {f.casefold() for f in files}
+    hits: list[Language] = []
+    language_scores: dict[Language, int] = {}
+
+    for lang, markers in CONFIG_MARKERS.items():
+        score = _calculate_language_score(names, markers)
+        if score > 0:
+            language_scores[lang] = score
+            hits.append(lang)
+
+    return _resolve_language_conflicts(hits, language_scores)
+
+
+def is_config_marker(name: str) -> bool:
+    return name.casefold() in _FOLDED_MARKER_NAMES or any(
+        fnmatch(name, pat) for pat in _MARKER_GLOBS
+    )
+
+
+def detect_anchor_language(files: Iterable[str]) -> Language | None:
+    names = tuple(files)
+    folded = {name.casefold() for name in names}
+    anchors = {language for language, manifests in _FOLDED_MANIFESTS.items() if manifests & folded}
+    if not anchors:
+        return None
+    ranked = tuple(lang for lang in detect_languages_in_dir(names) if lang in anchors) or tuple(
+        sorted(anchors, key=lambda language: language.value)
+    )
+    return ranked[0] if len(ranked) == 1 else Language.Unknown

fluidattacks_core/filesystem/language.py

@@ -0,0 +1,27 @@
+from enum import Enum
+
+
+class Language(Enum):
+    Swift = "swift"
+    CSharp = "csharp"
+    Go = "golang"
+    Rust = "rust"
+    Dart = "dart"
+    FuzzyTestLang = "fuzzy_test_lang"
+    Java = "Java"
+    Python = "Python"
+    PHP = "PHP"
+    Ruby = "Ruby"
+    C = "C"
+    Kotlin = "Kotlin"
+    Ghidra = "Ghidra"
+    JavaScript = "JavaScript"
+    TypeScript = "TypeScript"
+    LLVM = "LLVM"
+    NewC = "newc"
+    Scala = "Scala"
+    Unknown = "unknown"
+
+    def __str__(self) -> str:
+        """Render as the bare value, as sifts and explores consumers expect."""
+        return str(self.value)

fluidattacks_core/filesystem/partitioning.py

@@ -0,0 +1,254 @@
+from __future__ import annotations
+
+import os
+import stat
+from collections import Counter
+from collections.abc import Sequence
+from dataclasses import dataclass
+from fnmatch import fnmatch
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from .defaults import EXTENDED_EXCLUDE_DIRS
+from .detection import detect_anchor_language, detect_languages_in_dir, is_config_marker
+from .parts import Part, PartIdentity, PartTree
+
+if TYPE_CHECKING:
+    from collections.abc import Collection, Mapping
+
+__all__ = ["DirListing", "build_part_tree", "decide_part_tree"]
+
+DirListing = Sequence[tuple[Path, tuple[str, ...]]]
+
+PotentialPart = tuple[Path, frozenset[Path], str | None]
+
+_ROOT_PATH_MARKER = Path()
+
+
+def _generate_part_name(path: Path, root_path: Path) -> str:
+    if path == root_path:
+        return "."
+    try:
+        relative = path.relative_to(root_path)
+    except ValueError:
+        return path.name
+    return "/".join(relative.parts)
+
+
+def _relativize(path: Path, root_path: Path) -> Path:
+    try:
+        return path.relative_to(root_path)
+    except ValueError:
+        return path
+
+
+def _find_config_files(dir_path: Path, files: Sequence[str]) -> frozenset[Path]:
+    return frozenset(dir_path / name for name in files if is_config_marker(name))
+
+
+def _owning_part(dir_path: Path, part_paths: frozenset[Path], root_path: Path) -> Path | None:
+    current = dir_path
+    while True:
+        if current in part_paths:
+            return current
+        if current in (root_path, current.parent):
+            return None
+        current = current.parent
+
+
+def _root_filenames(listing: DirListing, root_path: Path) -> tuple[str, ...]:
+    for dir_path, filenames in listing:
+        if dir_path == root_path:
+            return filenames
+    return ()
+
+
+def _discover_potential_parts(listing: DirListing) -> tuple[PotentialPart, ...]:
+    return tuple(
+        (dir_path, _find_config_files(dir_path, filenames), language.value)
+        for dir_path, filenames in listing
+        if (language := detect_anchor_language(filenames)) is not None
+    )
+
+
+def _build_hierarchy(
+    potential_parts: Sequence[PotentialPart],
+    root_path: Path,
+) -> Mapping[Path, tuple[Path, ...]]:
+    sorted_parts = sorted(potential_parts, key=lambda part: len(part[0].parts))
+    children: dict[Path, list[Path]] = {part[0]: [] for part in sorted_parts}
+
+    for part_path, _, _ in sorted_parts:
+        if part_path == root_path:
+            continue
+        current = part_path.parent
+        while True:
+            if current in children:
+                children[current].append(part_path)
+                break
+            if current == current.parent:
+                break
+            current = current.parent
+
+    return {path: tuple(child_paths) for path, child_paths in children.items()}
+
+
+def _assign_files_to_parts(
+    part_paths: frozenset[Path],
+    listing: DirListing,
+    root_path: Path,
+) -> Mapping[Path, frozenset[Path]]:
+    owned: dict[Path, set[Path]] = {part_path: set() for part_path in part_paths}
+
+    for dir_path, filenames in listing:
+        owner = _owning_part(dir_path, part_paths, root_path)
+        if owner is None:
+            continue
+        owned[owner].update(_relativize(dir_path / name, root_path) for name in filenames)
+
+    return {part_path: frozenset(files) for part_path, files in owned.items()}
+
+
+def _collect_files_for_all(listing: DirListing, root_path: Path) -> frozenset[Path]:
+    return frozenset(
+        _relativize(dir_path / name, root_path)
+        for dir_path, filenames in listing
+        for name in filenames
+    )
+
+
+def _find_all_config_files(listing: DirListing) -> frozenset[Path]:
+    return frozenset(
+        path for dir_path, filenames in listing for path in _find_config_files(dir_path, filenames)
+    )
+
+
+@dataclass(frozen=True)
+class _BuildContext:
+    parent_map: Mapping[Path, tuple[Path, ...]]
+    parts_info: Mapping[Path, tuple[frozenset[Path], str | None]]
+    root_path: Path
+    files_by_part: Mapping[Path, frozenset[Path]]
+
+
+def _build_flat_parts(
+    ctx: _BuildContext,
+) -> tuple[Mapping[str, Part], Mapping[str, tuple[str, ...]]]:
+    parts: dict[str, Part] = {}
+    children: dict[str, tuple[str, ...]] = {}
+
+    for part_path, child_paths in ctx.parent_map.items():
+        config_files, language = ctx.parts_info.get(part_path, (frozenset(), None))
+        name = _generate_part_name(part_path, ctx.root_path)
+        parts[name] = Part(
+            identity=PartIdentity(name=name, path=_relativize(part_path, ctx.root_path)),
+            files=ctx.files_by_part.get(part_path, frozenset()),
+            config_files=frozenset(_relativize(path, ctx.root_path) for path in config_files),
+            detected_language=language,
+        )
+        children[name] = tuple(
+            _generate_part_name(child, ctx.root_path) for child in sorted(child_paths)
+        )
+
+    return parts, children
+
+
+def _validate_tree(tree: PartTree) -> None:
+    all_parts = tree.get_all_parts()
+
+    names = Counter(part.name for part in all_parts)
+    repeated = sorted(name for name, count in names.items() if count > 1)
+    if repeated:
+        msg = f"Duplicate part names found: {repeated}"
+        raise ValueError(msg)
+
+    files = Counter(path for part in all_parts for path in part.files)
+    shared = sorted(str(path) for path, count in files.items() if count > 1)
+    if shared:
+        msg = f"Files assigned to multiple parts: {shared[:5]}"
+        raise ValueError(msg)
+
+
+def _build_single_root_tree(root_path: Path, listing: DirListing) -> PartTree:
+    config_files = _find_all_config_files(listing)
+    languages = detect_languages_in_dir(path.name for path in config_files)
+    root_part = Part(
+        identity=PartIdentity(name=".", path=_ROOT_PATH_MARKER),
+        files=_collect_files_for_all(listing, root_path),
+        config_files=frozenset(_relativize(path, root_path) for path in config_files),
+        detected_language=languages[0].value if languages else None,
+    )
+    return PartTree(
+        root_path=root_path,
+        parts={".": root_part},
+        children_map={".": ()},
+        root_name=".",
+    )
+
+
+def decide_part_tree(root_path: Path, listing: DirListing) -> PartTree:
+    potential_parts = _discover_potential_parts(listing)
+    if not potential_parts:
+        return _build_single_root_tree(root_path, listing)
+
+    if all(path != root_path for path, _, _ in potential_parts):
+        root_files = _root_filenames(listing, root_path)
+        anchor = detect_anchor_language(root_files)
+        potential_parts = (
+            (
+                root_path,
+                _find_config_files(root_path, root_files),
+                anchor.value if anchor else None,
+            ),
+            *potential_parts,
+        )
+
+    parent_map = _build_hierarchy(potential_parts, root_path)
+    context = _BuildContext(
+        parent_map=parent_map,
+        parts_info={path: (configs, lang) for path, configs, lang in potential_parts},
+        root_path=root_path,
+        files_by_part=_assign_files_to_parts(frozenset(parent_map), listing, root_path),
+    )
+    parts, children_map = _build_flat_parts(context)
+
+    tree = PartTree(
+        root_path=root_path,
+        parts=parts,
+        children_map=children_map,
+        root_name=".",
+    )
+    _validate_tree(tree)
+    return tree
+
+
+def _is_member_file(path: Path) -> bool:
+    try:
+        return stat.S_ISREG(path.lstat().st_mode)
+    except OSError:
+        return False
+
+
+def _walk_listing(root_path: Path, exclude_dirs: Collection[str]) -> DirListing:
+    listing: list[tuple[Path, tuple[str, ...]]] = []
+    for current, dirs, filenames in os.walk(root_path):
+        dirs[:] = sorted(d for d in dirs if not any(fnmatch(d, pat) for pat in exclude_dirs))
+        directory = Path(current)
+        kept = (name for name in filenames if _is_member_file(directory / name))
+        listing.append((directory, tuple(sorted(kept))))
+    return tuple(listing)
+
+
+def build_part_tree(root_path: Path, extra_excludes: Sequence[str] | None = None) -> PartTree:
+    resolved = Path(root_path).resolve()
+    if not resolved.is_dir():
+        msg = f"Root path must be a directory: {root_path}"
+        raise ValueError(msg)
+
+    exclusions = (*EXTENDED_EXCLUDE_DIRS, *(extra_excludes or ()))
+    listing = _walk_listing(resolved, exclusions)
+    if not listing:
+        msg = f"Root path could not be read: {root_path}"
+        raise OSError(msg)
+
+    return decide_part_tree(resolved, listing)

fluidattacks_core/filesystem/parts.py

@@ -0,0 +1,178 @@
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from pathlib import Path
+
+__all__ = ["Part", "PartIdentity", "PartTree"]
+
+_SAMPLE_FILE_LIMIT = 10
+
+
+@dataclass(frozen=True)
+class PartIdentity:
+    name: str
+    path: Path
+
+
+@dataclass(frozen=True)
+class Part:
+    identity: PartIdentity
+    files: frozenset[Path] = field(default_factory=frozenset)
+    config_files: frozenset[Path] = field(default_factory=frozenset)
+    detected_language: str | None = None
+
+    @property
+    def name(self) -> str:
+        return self.identity.name
+
+    @property
+    def path(self) -> Path:
+        return self.identity.path
+
+    def file_count(self) -> int:
+        return len(self.files)
+
+    def all_files(self) -> frozenset[Path]:
+        return self.files
+
+
+@dataclass(frozen=True)
+class PartTree:
+    root_path: Path
+    parts: Mapping[str, Part] = field(default_factory=dict)
+    children_map: Mapping[str, tuple[str, ...]] = field(default_factory=dict)
+    root_name: str = "."
+
+    @property
+    def root(self) -> Part | None:
+        return self.parts.get(self.root_name)
+
+    def __len__(self) -> int:
+        """Count every part in the tree, not just the root's children."""
+        return len(self.parts)
+
+    def get_part(self, name: str) -> Part | None:
+        return self.parts.get(name)
+
+    def get_all_parts(self) -> tuple[Part, ...]:
+        return self._collect_parts_dfs(self.root_name)
+
+    def get_part_by_path(self, path: Path) -> Part | None:
+        for part in self.parts.values():
+            if part.path == path:
+                return part
+        return None
+
+    def get_children(self, part_name: str) -> tuple[Part, ...]:
+        child_names = self.children_map.get(part_name, ())
+        return tuple(self.parts[name] for name in child_names if name in self.parts)
+
+    def get_parent(self, part_name: str) -> Part | None:
+        for parent_name, child_names in self.children_map.items():
+            if part_name in child_names:
+                return self.parts.get(parent_name)
+        return None
+
+    def recursive_file_count(self, part_name: str) -> int:
+        part = self.parts.get(part_name)
+        if part is None:
+            return 0
+        count = len(part.files)
+        for child_name in self.children_map.get(part_name, ()):
+            count += self.recursive_file_count(child_name)
+        return count
+
+    def recursive_all_files(self, part_name: str) -> frozenset[Path]:
+        part = self.parts.get(part_name)
+        if part is None:
+            return frozenset()
+        all_files: set[Path] = set(part.files)
+        for child_name in self.children_map.get(part_name, ()):
+            all_files.update(self.recursive_all_files(child_name))
+        return frozenset(all_files)
+
+    def get_part_context(self, part_name: str) -> str:
+        part = self.parts.get(part_name)
+        if part is None:
+            return ""
+        lines = [
+            f"# Part: {part.name}",
+            f"**Path**: `{part.path}`",
+            "",
+        ]
+        if part.detected_language:
+            lines.extend([f"**Language**: {part.detected_language}", ""])
+
+        if part.config_files:
+            lines.append("**Config files**:")
+            lines.extend(f"- `{config_file}`" for config_file in sorted(part.config_files))
+            lines.append("")
+
+        children = self.get_children(part_name)
+        total_file_count = self.recursive_file_count(part_name)
+
+        lines.extend(
+            [
+                f"**Direct files**: {len(part.files)}",
+                f"**Total files** (including children): {total_file_count}",
+                "",
+            ]
+        )
+
+        if children:
+            lines.append("**Child parts**:")
+            lines.extend(
+                f"- `{child.name}` ({self.recursive_file_count(child.name)} files)"
+                for child in children
+            )
+            lines.append("")
+
+        sample_files = sorted(part.files)[:_SAMPLE_FILE_LIMIT]
+        if sample_files:
+            lines.append("**Sample files**:")
+            lines.extend(f"- `{f}`" for f in sample_files)
+            if len(part.files) > _SAMPLE_FILE_LIMIT:
+                lines.append(f"- ... and {len(part.files) - _SAMPLE_FILE_LIMIT} more")
+
+        return "\n".join(lines)
+
+    def to_tree_string(self) -> str:
+        rendered = self._format_part_recursive(self.root_name, prefix="", is_last=True)
+        return "\n".join((f"{self.root_path}", *rendered))
+
+    def _collect_parts_dfs(self, part_name: str) -> tuple[Part, ...]:
+        part = self.parts.get(part_name)
+        if part is None:
+            return ()
+        return (
+            part,
+            *(
+                descendant
+                for child_name in self.children_map.get(part_name, ())
+                for descendant in self._collect_parts_dfs(child_name)
+            ),
+        )
+
+    def _format_part_recursive(
+        self, part_name: str, *, prefix: str, is_last: bool
+    ) -> tuple[str, ...]:
+        part = self.parts.get(part_name)
+        if part is None:
+            return ()
+        connector = "└── " if is_last else "├── "
+        lang_info = f" [{part.detected_language}]" if part.detected_language else ""
+        file_info = f" ({self.recursive_file_count(part_name)} files)"
+        child_prefix = prefix + ("    " if is_last else "│   ")
+        child_names = self.children_map.get(part_name, ())
+
+        return (
+            f"{prefix}{connector}{part.name}{lang_info}{file_info}",
+            *(
+                line
+                for index, child_name in enumerate(child_names)
+                for line in self._format_part_recursive(
+                    child_name,
+                    prefix=child_prefix,
+                    is_last=index == len(child_names) - 1,
+                )
+            ),
+        )

fluidattacks_core/filesystem/projects.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+import os
+from fnmatch import fnmatch
+from pathlib import Path
+
+from .defaults import (
+    EXCLUDE_DIRS,
+    EXCLUDE_FILES,
+    SKIP_DIRS,
+    TEST_FILES,
+)
+from .detection import detect_anchor_language, detect_languages_in_dir
+from .language import Language
+
+__all__ = ["detect_root_language", "find_parent_project", "find_projects"]
+
+
+def _file_names(directory: Path) -> list[str]:
+    try:
+        entries = list(os.scandir(directory))
+    except OSError:
+        return []
+    return [e.name for e in entries if e.is_file(follow_symlinks=False)]
+
+
+def _optimize_exclusions(exclusions: list[str]) -> list[str]:
+    if not exclusions:
+        return []
+
+    paths = sorted([Path(p) for p in set(exclusions)])
+
+    optimized: list[Path] = []
+    for p in paths:
+        if not any(p.is_relative_to(opt_p) for opt_p in optimized):
+            optimized.append(p)
+
+    return sorted([p.as_posix() for p in optimized])
+
+
+def _collect_files(
+    entries: list[os.DirEntry[str]],
+    root_dir: Path,
+    excluded: set[str],
+) -> list[str]:
+    files: list[str] = []
+    for e in entries:
+        if not e.is_file(follow_symlinks=False):
+            continue
+        name = e.name
+        if any(fnmatch(name, pat) for pat in TEST_FILES) or any(
+            fnmatch(name, pat) for pat in EXCLUDE_FILES
+        ):
+            excluded.add(Path(e.path).relative_to(root_dir).as_posix())
+            continue
+        files.append(name)
+    return files
+
+
+def _scan_dir(
+    dir_path: Path,
+    root_dir: Path,
+    excluded_by_skip: set[str],
+) -> list[tuple[Path, Language, list[str]]]:
+    try:
+        entries = list(os.scandir(dir_path))
+    except OSError:
+        return []
+
+    files = _collect_files(entries, root_dir, excluded_by_skip)
+    language = detect_anchor_language(files)
+    subprojects: list[tuple[Path, Language, list[str]]] = []
+
+    for sd in sorted(
+        (e for e in entries if e.is_dir(follow_symlinks=False)),
+        key=lambda e: e.name,
+    ):
+        rel_sd = Path(sd.path).relative_to(root_dir).as_posix()
+        skip_dir = (
+            sd.name.startswith(".tox")
+            or any(fnmatch(sd.name, pat) for pat in SKIP_DIRS)
+            or any(fnmatch(rel_sd, pat) for pat in EXCLUDE_DIRS)
+        )
+        if skip_dir:
+            excluded_by_skip.add(rel_sd)
+            continue
+        subprojects.extend(_scan_dir(Path(sd.path), root_dir, excluded_by_skip))
+
+    if language is not None:
+        exclusions = _optimize_exclusions(
+            [sp[0].relative_to(dir_path).as_posix() for sp in subprojects]
+        )
+        return [(dir_path, language, exclusions), *subprojects]
+
+    return subprojects
+
+
+def _enrich(
+    projects: list[tuple[Path, Language, list[str]]],
+    root_dir: Path,
+    excluded_by_skip: set[str],
+) -> list[tuple[Path, str, list[str]]]:
+    enriched: list[tuple[Path, str, list[str]]] = []
+    for p_path, p_lang, p_excls in projects:
+        extra = [
+            (root_dir / rel).relative_to(p_path).as_posix()
+            for rel in excluded_by_skip
+            if (root_dir / rel).is_relative_to(p_path)
+        ]
+        final_excls = _optimize_exclusions([*p_excls, *extra]) if extra else p_excls
+        enriched.append((p_path, p_lang.value, final_excls))
+    return enriched
+
+
+def find_projects(root_dir: str | Path) -> list[tuple[Path, str, list[str]]]:
+    root_dir = Path(root_dir).resolve()
+
+    excluded_by_skip: set[str] = set()
+    scanned = _scan_dir(root_dir, root_dir, excluded_by_skip)
+    contained = sorted((p for p in scanned if p[0].is_relative_to(root_dir)), key=lambda p: p[0])
+    enriched_projects = _enrich(contained, root_dir, excluded_by_skip)
+
+    root_exclusions = [p[0].relative_to(root_dir).as_posix() for p in enriched_projects]
+    root_exclusions.extend(excluded_by_skip)
+
+    return [
+        (root_dir, "root", _optimize_exclusions(root_exclusions)),
+        *enriched_projects,
+    ]
+
+
+def detect_root_language(root_dir: str | Path) -> Language:
+    langs = detect_languages_in_dir(_file_names(Path(root_dir)))
+    return langs[0] if len(langs) == 1 else Language.Unknown
+
+
+def find_parent_project(
+    file_path: str | Path,
+    base_dir: Path,
+) -> tuple[Path, str] | None:
+    base = Path(base_dir).resolve()
+    start = Path(file_path).resolve()
+    current = start if start.is_dir() else start.parent
+
+    if not current.is_relative_to(base):
+        return None
+
+    while True:
+        language = detect_anchor_language(_file_names(current))
+        if language is not None:
+            return (current, language.value)
+        if current in (base, current.parent):
+            return None
+        current = current.parent

fluidattacks_core_filesystem-12.0.0.dist-info/METADATA

@@ -0,0 +1,13 @@
+Metadata-Version: 2.5
+Name: fluidattacks_core_filesystem
+Version: 12.0.0
+Summary: Fluid Attacks Core Filesystem Library
+Author-email: Development <development@fluidattacks.com>
+License: MPL-2.0
+Classifier: Development Status :: 1 - Planning
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.11

fluidattacks_core_filesystem-12.0.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+fluidattacks_core/filesystem/__init__.py,sha256=g7DDWi0WEYVmNFPu2ZSqNy2--gQzELpvwPdZH2_hG14,498
+fluidattacks_core/filesystem/defaults.py,sha256=vu9XB4La9bjVRYaRyckmUh731DojJml1udvRu5oa0nU,11740
+fluidattacks_core/filesystem/detection.py,sha256=ave7Y67w2FYTLzCEM5UcFzT2sdaqQ1vc3jc4A84zcoY,3607
+fluidattacks_core/filesystem/language.py,sha256=B_ESvmw56rjQw4C-e3Xk4j-iT_eBI-89FyAtZV7ZOeQ,592
+fluidattacks_core/filesystem/partitioning.py,sha256=Q0rHNO1miDj1u3yYnbFZ5-FHxCso5XTxCmdiAlQsWBk,8468
+fluidattacks_core/filesystem/parts.py,sha256=meyUBR9HdGLr3rGihRcF7J8KcBslC2cuvTi1-3kV7Lk,5838
+fluidattacks_core/filesystem/projects.py,sha256=VTbSZQzE7889_j6_JFomB-nk4ecLWWCOoMlq89NTx6Q,4671
+fluidattacks_core/filesystem/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fluidattacks_core_filesystem-12.0.0.dist-info/METADATA,sha256=Q6FkoLFxZgt9cHFYXTSGL8OBSk3MCXGl1FqEluALdTU,539
+fluidattacks_core_filesystem-12.0.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+fluidattacks_core_filesystem-12.0.0.dist-info/RECORD,,

fluidattacks_core_filesystem-12.0.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any