oat-py 1.0.0__py3-none-any.whl

oat/__init__.py

@@ -0,0 +1,4 @@
+# OAT - OpenHarmony OSS Audit Tool (Python Edition)
+# Copyright (c) 2024 Huawei Device Co., Ltd.
+# Licensed under the Apache License, Version 2.0
+__version__ = "2.0.0"

oat/__main__.py

@@ -0,0 +1,40 @@
+"""
+Entry point: python -m oat  or  oat (if installed via pip)
+"""
+import sys
+import time
+from pathlib import Path
+
+
+def main(argv=None):
+    from oat.cli import parse_args
+    args = parse_args(argv)
+
+    start = time.time()
+    print(f"[OAT] OpenHarmony OSS Audit Tool - Python Edition")
+    print(f"[OAT] Scanning: {args.s}")
+
+    from oat.config.loader import build_task
+    from oat.analysis.pipeline import run_pipeline
+    from oat.reporter.plain_reporter import PlainReporter
+
+    task = build_task(args)
+
+    report_model = run_pipeline(task)
+
+    reporter = PlainReporter(task, report_model)
+    reporter.write()
+
+    elapsed = time.time() - start
+    total = report_model.total_issues
+    print(f"[OAT] Scan complete in {elapsed:.1f}s. Issues found: {total}")
+    if total > 0:
+        print(f"[OAT] Report: {task.report_dir}")
+        sys.exit(1)
+    else:
+        print("[OAT] No compliance issues found.")
+        sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

oat/analysis/document.py

@@ -0,0 +1,36 @@
+"""
+FileDocument — mirrors OatFileDocument.java.
+Holds analysis results for a single file.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List, Optional
+
+
+@dataclass
+class FileDocument:
+    path: Path
+    rel_path: str = ""          # relative to src_dir, forward slashes
+
+    # Set by file_type analyser
+    file_type: str = "text"     # 'text' | 'binary' | 'archive'
+
+    # Set by header analyser
+    license: str = "NoLicenseHeader"
+    copyright_owners: List[str] = field(default_factory=list)
+
+    # Set by policy_verifier
+    issues: List["Issue"] = field(default_factory=list)
+
+    @property
+    def has_issues(self) -> bool:
+        return bool(self.issues)
+
+
+@dataclass
+class Issue:
+    issue_type: str             # 'filetype' | 'license' | 'copyright' | 'filename'
+    description: str
+    detail: str = ""

oat/analysis/file_type.py

@@ -0,0 +1,136 @@
+"""
+File type analyser — mirrors OatFileTypeAnalyser.java + Apache Rat BinaryGuesser.
+Detects binary and archive files.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+# Archive file extensions (lowercase)
+_ARCHIVE_EXTS = {
+    ".jar", ".gz", ".zip", ".tar", ".bz2", ".rar", ".war", ".7z",
+    ".rpm", ".deb", ".img", ".apk", ".ipa", ".whl", ".egg",
+    ".tar.gz", ".tar.bz2", ".tar.xz", ".tgz", ".tbz2",
+}
+
+# Binary/pre-built file extensions (lowercase).
+# Mirrors OatFileUtils.PREBUILD_FILE_EXTENSION + Apache Rat BinaryGuesser extension lists.
+_BINARY_EXTS = {
+    ".so", ".dll", ".exe", ".elf", ".bin", ".a", ".o", ".class",
+    ".pyc", ".pyd", ".pyo", ".hap", ".scr", ".lib", ".pdb",
+    ".obj", ".ko", ".d.ts", ".exp",
+}
+
+# Keystore / certificate binary extensions from Apache Rat BinaryGuesser.KEYSTORE_EXTENSIONS.
+# These are always treated as binary regardless of content.
+_KEYSTORE_EXTS = {
+    ".jks", ".keystore", ".pem", ".crl", ".truststore", ".cert", ".ks",
+}
+
+# Image extensions from Apache Rat BinaryGuesser.IMAGE_EXTENSIONS.
+_IMAGE_EXTS = {
+    ".png", ".pdf", ".gif", ".giff", ".tif", ".tiff", ".jpg", ".jpeg",
+    ".ico", ".icns", ".psd", ".webp", ".bmp", ".svg",
+}
+
+# Audio / media extensions from Apache Rat BinaryGuesser.AUDIO_EXTENSIONS.
+_AUDIO_EXTS = {
+    ".aif", ".iff", ".m3u", ".mid", ".mp3", ".mpa", ".wav", ".wma",
+}
+
+# Font / layout binary extensions
+_FONT_EXTS = {
+    ".woff", ".woff2", ".ttf", ".eot", ".otf",
+}
+
+# Extensions that Apache Rat BinaryGuesser.NON_BINARY_EXTENSIONS explicitly lists as text.
+# Files with these extensions are NEVER treated as binary by content-sniffing.
+_NON_BINARY_EXTS = {
+    ".ac", ".am", ".bat", ".cat", ".cgi", ".classpath", ".cmd", ".config",
+    ".cpp", ".css", ".cwiki", ".data", ".dcl", ".dtd", ".egrm", ".ent",
+    ".ft", ".fn", ".fv", ".grm", ".go", ".htaccess", ".html", ".ihtml",
+    ".in", ".jmx", ".jsp", ".js", ".json", ".junit", ".jx",
+    ".manifest", ".md", ".mf", ".meta", ".mod",
+    ".pen", ".pl", ".pm", ".pod", ".pom", ".project", ".properties",
+    ".py", ".rb", ".rdf", ".rnc", ".rng", ".rnx", ".roles", ".rss",
+    ".sh", ".sql", ".svg", ".tld", ".txt", ".types",
+    ".vm", ".vsl", ".wsdd", ".wsdl", ".xargs", ".xcat", ".xconf",
+    ".xegrm", ".xgrm", ".xlex", ".xlog", ".xmap", ".xml",
+    ".xroles", ".xsamples", ".xsd", ".xsl", ".xslt", ".xsp", ".xul",
+    ".xweb", ".xwelcome",
+    # Additional common text extensions not in Apache Rat 0.13 but treated as text by OAT
+    ".c", ".h", ".cc", ".hh", ".java", ".ts", ".cs", ".rb", ".php",
+    ".yml", ".yaml", ".toml", ".rst", ".tex",
+    ".gradle", ".cmake", ".mk", ".makefile",
+    ".proto", ".thrift", ".avro",
+    ".ini", ".cfg", ".conf",
+}
+
+# Probe this many bytes for content-based binary detection
+_PROBE_BYTES = 512
+
+# Apache Rat HIGH_BYTES_RATIO threshold: if >30% of chars are non-ASCII → binary
+_HIGH_BYTE_THRESHOLD = 30
+
+
+def is_archive(path: Path) -> bool:
+    """Return True if the file is a known archive type."""
+    name = path.name.lower()
+    # Check compound extensions first
+    for ext in (".tar.gz", ".tar.bz2", ".tar.xz", ".so.gz"):
+        if name.endswith(ext):
+            return True
+    return path.suffix.lower() in _ARCHIVE_EXTS
+
+
+def is_binary(path: Path) -> bool:
+    """Return True if the file should be treated as binary.
+
+    Mirrors Apache Rat BinaryGuesser.isBinary() logic used by OAT Java:
+    1. Known binary extensions → binary
+    2. Known keystore/image/audio/font extensions → binary
+    3. Explicitly listed non-binary extensions → NOT binary (skip content check)
+    4. Content check: if >30% of bytes in first 512 bytes are >127 → binary
+    5. Null byte in first 512 bytes → binary
+    """
+    ext = path.suffix.lower()
+    name_lower = path.name.lower()
+
+    # Step 1: known binary extensions
+    if ext in _BINARY_EXTS:
+        return True
+
+    # Step 2: keystore / image / audio / font → binary
+    if ext in _KEYSTORE_EXTS or ext in _IMAGE_EXTS or ext in _AUDIO_EXTS or ext in _FONT_EXTS:
+        return True
+
+    # Step 3: explicitly non-binary extension → skip content check
+    if ext in _NON_BINARY_EXTS:
+        return False
+
+    # Step 4+5: content-based detection
+    try:
+        with open(path, "rb") as f:
+            chunk = f.read(_PROBE_BYTES)
+        if not chunk:
+            return False
+        # Null byte → binary
+        if b"\x00" in chunk:
+            return True
+        # High-byte ratio check (mirrors Apache Rat BinaryGuesser HIGH_BYTES_RATIO=30)
+        high = sum(1 for b in chunk if b > 127)
+        if high * 100 // len(chunk) > _HIGH_BYTE_THRESHOLD:
+            return True
+    except OSError:
+        pass
+
+    return False
+
+
+def get_file_type(path: Path) -> str:
+    """Return 'archive', 'binary', or 'text'."""
+    if is_archive(path):
+        return "archive"
+    if is_binary(path):
+        return "binary"
+    return "text"

oat/analysis/header_reader.py

@@ -0,0 +1,114 @@
+"""
+Reads the header of a text file and drives license / copyright matchers.
+Mirrors OatHeaderMatchAnalyser.java + OatFileAnalyser.java.
+
+Performance note: read_header_with_type() performs a single file open for both
+binary detection and header extraction, avoiding the two-open pattern used when
+calling get_file_type() + read_header() separately.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+from oat.matchers.copyright_matcher import match_copyright
+from oat.matchers.license_matcher import match_license
+
+# Maximum number of header lines to read
+HEADER_LINES = 100
+# Maximum file size to scan (10 MB)
+MAX_FILE_SIZE = 10 * 1024 * 1024
+
+# Encodings to try, in order
+_ENCODINGS = ["utf-8", "latin-1", "gbk", "utf-16"]
+
+# Probe bytes for binary detection (mirrors Apache Rat BinaryGuesser)
+_PROBE_BYTES = 512
+_HIGH_BYTE_THRESHOLD = 30  # >30% non-ASCII → binary
+
+
+def read_header(path: Path) -> Optional[str]:
+    """
+    Read the first HEADER_LINES lines of a text file.
+    Returns None if the file cannot be decoded.
+    """
+    if path.stat().st_size > MAX_FILE_SIZE:
+        return None
+
+    for enc in _ENCODINGS:
+        try:
+            with open(path, encoding=enc, errors="strict") as f:
+                lines = []
+                for i, line in enumerate(f):
+                    if i >= HEADER_LINES:
+                        break
+                    lines.append(line)
+            return "".join(lines)
+        except (UnicodeDecodeError, LookupError):
+            continue
+    return None
+
+
+def read_header_with_type(path: Path) -> Tuple[str, Optional[str]]:
+    """
+    Single-pass file read: returns (file_type, header_text_or_None).
+
+    file_type is 'text', 'binary', or 'size_exceeded'.
+    This avoids the double-open that occurs when get_file_type() reads 512 bytes
+    and then read_header() opens the file again for HEADER_LINES lines.
+
+    Callers that need the full file_type taxonomy (archive detection etc.) should
+    still use get_file_type() for the extension-based checks before calling this.
+    Only the content-sniffing part is merged here.
+    """
+    try:
+        size = path.stat().st_size
+    except OSError:
+        return "text", None
+
+    if size > MAX_FILE_SIZE:
+        return "text", None  # oversized → treat as text but skip header
+
+    # --- Read raw bytes once ---
+    try:
+        with open(path, "rb") as f:
+            raw = f.read(MAX_FILE_SIZE)
+    except OSError:
+        return "text", None
+
+    # --- Binary sniffing (mirrors file_type.is_binary content check) ---
+    probe = raw[:_PROBE_BYTES]
+    if probe:
+        if b"\x00" in probe:
+            return "binary", None
+        high = sum(1 for b in probe if b > 127)
+        if high * 100 // len(probe) > _HIGH_BYTE_THRESHOLD:
+            return "binary", None
+
+    # --- Decode header lines ---
+    for enc in _ENCODINGS:
+        try:
+            text = raw.decode(enc, errors="strict")
+            lines = text.splitlines(keepends=True)[:HEADER_LINES]
+            return "text", "".join(lines)
+        except (UnicodeDecodeError, LookupError):
+            continue
+
+    return "text", None
+
+
+def analyse_header(
+    path: Path,
+    custom_texts: Optional[Dict[str, List[str]]] = None,
+):
+    """
+    Returns (license_str, copyright_owners_list).
+    license_str is 'NoLicenseHeader' / 'InvalidLicense' / actual ID.
+    """
+    header = read_header(path)
+    if header is None:
+        return "NoLicenseHeader", []
+
+    license_str = match_license(header, custom_texts)
+    copyright_owners = match_copyright(header)
+    return license_str, copyright_owners

oat/analysis/pipeline.py

@@ -0,0 +1,122 @@
+"""
+Analysis pipeline — mirrors OatComplianceExecutor.java + OatDefaultTaskProcessor.java.
+Orchestrates: walk → file_type → header → policy_verify → collect results.
+"""
+from __future__ import annotations
+
+import os
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import List
+
+from oat.analysis.document import FileDocument
+from oat.analysis.file_type import get_file_type, is_archive
+from oat.analysis.header_reader import analyse_header, read_header_with_type
+from oat.analysis.policy_verifier import verify
+from oat.config.schema import OatConfig
+from oat.reporter.model import ReportModel
+from oat.walker.directory_walker import collect_files
+
+
+def run_pipeline(cfg: OatConfig) -> ReportModel:
+    """Main entry point. Returns a populated ReportModel."""
+    files = collect_files(cfg)
+    total = len(files)
+    print(f"[OAT] Found {total} files to scan.")
+
+    prj = cfg.active_project
+    policy = prj.policy if prj else None
+    file_filter = prj.file_filter if prj else None
+    all_filters = cfg.file_filters  # pass named filter dict for per-policyitem lookup
+
+    # Merge custom license texts: global + project-level
+    custom_texts = dict(cfg.global_license_texts)
+    if prj and prj.custom_license_texts:
+        for k, v in prj.custom_license_texts.items():
+            existing = custom_texts.get(k, [])
+            custom_texts[k] = existing + v
+
+    # Names defined by project-level licensematcher entries (e.g. "cann License").
+    # When the header reader identifies a file using one of these matchers, the file
+    # is considered to have a valid licence (mirrors Java behaviour).
+    custom_license_names: set = set(prj.custom_license_texts.keys()) if prj else set()
+
+    src_path = Path(cfg.src_dir)
+
+    env_val = os.getenv("OAT_MAX_WORKERS", "").strip()
+    if env_val.isdigit() and int(env_val) > 0:
+        max_workers = int(env_val)
+    else:
+        max_workers = min(32, (os.cpu_count() or 1) * 4)
+    print(f"[OAT] Using {max_workers} worker threads (cpu_count={os.cpu_count()}).")
+    docs: List[FileDocument] = []
+
+    with ThreadPoolExecutor(max_workers=max_workers) as pool:
+        futures = {
+            pool.submit(
+                _analyse_file,
+                f,
+                src_path,
+                policy,
+                file_filter,
+                all_filters,
+                custom_texts,
+                custom_license_names,
+            ): f
+            for f in files
+        }
+        done = 0
+        for future in as_completed(futures):
+            doc = future.result()
+            docs.append(doc)
+            done += 1
+            if done % 100 == 0 or done == total:
+                print(f"[OAT] Progress: {done}/{total}", end="\r", flush=True)
+
+    if total > 0:
+        print()  # newline after progress
+
+    return ReportModel(cfg=cfg, documents=docs)
+
+
+def _analyse_file(path, src_path, policy, file_filter, all_filters, custom_texts, custom_license_names) -> FileDocument:
+    rel = path.relative_to(src_path).as_posix()
+    doc = FileDocument(path=path, rel_path=rel)
+
+    # Fast path: extension-based archive detection (no I/O needed)
+    if is_archive(path):
+        doc.file_type = "archive"
+    else:
+        # Single I/O: binary sniff + header read in one pass.
+        # is_binary() for known extensions is pure extension lookup (no I/O);
+        # for unknown extensions it reads 512 bytes.  read_header_with_type()
+        # reads the full file once and combines both steps.
+        from oat.analysis.file_type import _BINARY_EXTS, _KEYSTORE_EXTS, _IMAGE_EXTS, _AUDIO_EXTS, _FONT_EXTS, _NON_BINARY_EXTS  # noqa: PLC0415
+        ext = path.suffix.lower()
+        if ext in _BINARY_EXTS or ext in _KEYSTORE_EXTS or ext in _IMAGE_EXTS or ext in _AUDIO_EXTS or ext in _FONT_EXTS:
+            # Known binary by extension — no need to read content
+            doc.file_type = "binary"
+        elif ext in _NON_BINARY_EXTS:
+            # Known text by extension — still need header; use plain read_header path
+            doc.file_type = "text"
+            doc.license, doc.copyright_owners = analyse_header(path, custom_texts or None)
+        else:
+            # Unknown extension: single open covers binary sniff + header extraction
+            ft, header_text = read_header_with_type(path)
+            doc.file_type = ft
+            if ft == "text" and header_text is not None:
+                from oat.matchers.license_matcher import match_license
+                from oat.matchers.copyright_matcher import match_copyright
+                doc.license = match_license(header_text, custom_texts or None)
+                doc.copyright_owners = match_copyright(header_text)
+            elif ft == "text":
+                doc.license = "NoLicenseHeader"
+
+    # Header analysis for known-text files (handled inline above for unknown-ext)
+    # This branch covers ext in _NON_BINARY_EXTS (already done) — nothing extra needed.
+
+    # Policy verification
+    if policy:
+        verify(doc, policy, file_filter, all_filters, custom_license_names)
+
+    return doc