dalla-data-processing 0.0.1__py3-none-any.whl

dalla/deduplication/onion/src_sc/version.c

@@ -0,0 +1,17 @@
+/*********************************************************************
+ * Copyright (c) 2011-2016 Jan Pomikalek, Milos Jakubicek            *
+ * All rights reserved.                                              *
+ *                                                                   *
+ * This software is licensed as described in the file COPYING, which *
+ * you should have received as part of this distribution.            *
+ *********************************************************************/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "version.h"
+
+void print_version(const char* progname) {
+    printf("%s: onion v%s\n\n", progname, VERSION);
+    printf("Copyright (c) 2011-2020 Lexical Computing Limited and Lexical Computing CZ s.r.o.\n");
+}

dalla/deduplication/onion/src_sc/version.h

@@ -0,0 +1,10 @@
+/*********************************************************************
+ * Copyright (c) 2011-2016 Jan Pomikalek, Milos Jakubicek            *
+ * All rights reserved.                                              *
+ *                                                                   *
+ * This software is licensed as described in the file COPYING, which *
+ * you should have received as part of this distribution.            *
+ *********************************************************************/
+
+#define VERSION "1.4"
+void print_version(const char* progname);

dalla/deduplication/onion_wrapper.py

@@ -0,0 +1,223 @@
+"""
+Wrapper for the onion C binary deduplication tool.
+Handles execution of the onion binary and management of temporary files.
+"""
+
+import shutil
+import subprocess
+from pathlib import Path
+
+from dalla.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def find_onion_binary() -> Path | None:
+    """
+    Find the onion binary in the system.
+
+    Searches in:
+    1. System PATH
+    2. Bundled platform-specific binary in package
+    3. Generic bundled binary (symlink)
+    4. Local build directories
+
+    Returns:
+        Path to onion binary or None if not found
+    """
+    import platform
+
+    onion_path = shutil.which("onion")
+    if onion_path:
+        logger.info(f"Found onion in system PATH: {onion_path}")
+        return Path(onion_path)
+
+    package_dir = Path(__file__).parent
+    bin_dir = package_dir / "bin"
+
+    system = platform.system().lower()
+    machine = platform.machine().lower()
+
+    if system == "linux":
+        platform_name = "linux"
+    elif system == "darwin":
+        platform_name = "darwin"
+    elif system == "windows":
+        platform_name = "windows"
+    else:
+        platform_name = system
+
+    platform_binary = bin_dir / f"onion-{platform_name}-{machine}"
+    if platform_binary.exists() and platform_binary.is_file():
+        logger.info(f"Found bundled platform-specific binary: {platform_binary}")
+        return platform_binary
+
+    generic_binary = bin_dir / "onion"
+    if generic_binary.exists() and generic_binary.is_file():
+        logger.info(f"Found bundled onion binary: {generic_binary}")
+        return generic_binary
+
+    Path(__file__).parents[2]
+    local_builds = [
+        package_dir / "onion" / "src_sc" / "onion",
+        package_dir / "onion" / "src" / "onion",
+    ]
+
+    for local_bin in local_builds:
+        if local_bin.exists() and local_bin.is_file():
+            logger.info(f"Found local onion binary: {local_bin}")
+            return local_bin
+
+    logger.warning("Onion binary not found in PATH or package")
+    return None
+
+
+def run_onion(
+    file_list_path: Path,
+    output_dir: Path,
+    dataset_name: str = "dataset",
+    onion_binary: Path | None = None,
+) -> tuple[bool, Path | None]:
+    """
+    Run the onion deduplication binary.
+
+    Args:
+        file_list_path: Path to file containing list of files to process
+        output_dir: Directory for output CSV files
+        dataset_name: Name of the dataset (for output files)
+        onion_binary: Path to onion binary (auto-detected if None)
+
+    Returns:
+        Tuple of (success: bool, output_csv_dir: Path)
+    """
+    if onion_binary is None:
+        onion_binary = find_onion_binary()
+
+    if onion_binary is None:
+        raise FileNotFoundError(
+            "Onion binary not found. Please install onion or provide path to binary.\n"
+            "Installation: https://corpus.tools/wiki/Onion"
+        )
+
+    onion_binary = Path(onion_binary)
+    if not onion_binary.exists():
+        raise FileNotFoundError(f"Onion binary not found at: {onion_binary}")
+
+    output_dir = Path(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    # onion expects: ./onion -D <dataset-name> -L <file-list> -O <output-dir>
+    cmd = [
+        str(onion_binary),
+        "-D",
+        dataset_name,
+        "-L",
+        str(file_list_path),
+        "-O",
+        str(output_dir),
+    ]
+
+    logger.info(f"Running onion: {' '.join(cmd)}")
+    logger.info(f"Processing file list: {file_list_path}")
+    logger.info(f"Output directory: {output_dir}")
+
+    try:
+        # Run onion
+        result = subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+            check=False,
+            timeout=21600,  # 6 hour timeout
+        )
+
+        if result.returncode != 0:
+            logger.error(f"Onion failed with return code {result.returncode}")
+            logger.error(f"STDOUT: {result.stdout}")
+            logger.error(f"STDERR: {result.stderr}")
+            return False, None
+
+        logger.info("Onion execution completed successfully")
+        if result.stdout:
+            logger.debug(f"Onion output: {result.stdout}")
+
+        return True, output_dir
+
+    except subprocess.TimeoutExpired:
+        logger.error("Onion execution timed out after 1 hour")
+        return False, None
+    except Exception as e:
+        logger.error(f"Error running onion: {e}")
+        return False, None
+
+
+def compile_onion(source_dir: Path) -> Path | None:
+    """
+    Compile the onion binary from source.
+
+    Args:
+        source_dir: Directory containing onion source code (with Makefile)
+
+    Returns:
+        Path to compiled binary or None if compilation failed
+    """
+    source_dir = Path(source_dir)
+
+    if not source_dir.exists():
+        logger.error(f"Source directory not found: {source_dir}")
+        return None
+
+    makefile = source_dir / "Makefile.g"
+    if not makefile.exists():
+        makefile = source_dir / "Makefile"
+
+    if not makefile.exists():
+        logger.error(f"No Makefile found in {source_dir}")
+        return None
+
+    logger.info(f"Compiling onion from source: {source_dir}")
+
+    try:
+        # Clean
+        subprocess.run(
+            ["make", "clean"],
+            cwd=source_dir,
+            capture_output=True,
+            check=False,
+        )
+
+        # Compile
+        makefile_flag = "-f Makefile.g" if (source_dir / "Makefile.g").exists() else ""
+        compile_cmd = f"make onion {makefile_flag}".split()
+
+        result = subprocess.run(
+            compile_cmd,
+            cwd=source_dir,
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+
+        if result.returncode != 0:
+            logger.error(f"Compilation failed: {result.stderr}")
+            return None
+
+        # Check for binary
+        binary_path = source_dir / "onion"
+        if binary_path.exists():
+            logger.info(f"Successfully compiled onion: {binary_path}")
+            return binary_path
+        else:
+            logger.error("Compilation succeeded but binary not found")
+            return None
+
+    except Exception as e:
+        logger.error(f"Error during compilation: {e}")
+        return None
+
+
+__all__ = [
+    "find_onion_binary",
+    "run_onion",
+    "compile_onion",
+]

dalla/deduplication/postprocessing.py

@@ -0,0 +1,216 @@
+"""
+Postprocessing utilities for onion output.
+
+Parses CSV files from onion and extracts duplicate information.
+"""
+
+import contextlib
+import csv
+from collections import defaultdict
+from pathlib import Path
+
+from tqdm import tqdm
+
+from dalla.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def parse_onion_csv(csv_file: Path) -> list[tuple[str, int]]:
+    """
+    Parse a single onion CSV file.
+
+    Onion CSV format:
+    - First column: file path
+    - Second column: line number (index in file list)
+
+    Args:
+        csv_file: Path to CSV file
+
+    Returns:
+        List of (file_path, line_number) tuples
+    """
+    entries = []
+
+    try:
+        with open(csv_file, encoding="utf-8") as f:
+            reader = csv.reader(f)
+            for row in reader:
+                if len(row) >= 2:
+                    file_path = row[0].strip().strip('"')
+                    line_number = int(row[1])
+                    entries.append((file_path, line_number))
+    except Exception as e:
+        logger.warning(f"Error parsing CSV {csv_file}: {e}")
+
+    return entries
+
+
+def extract_duplicates_from_csvs(
+    csv_dir: Path,
+    file_list_path: Path,
+) -> dict[str, set[str]]:
+    """
+    Extract duplicate groups from onion CSV outputs.
+
+    Each CSV file represents a group of duplicates.
+
+    Args:
+        csv_dir: Directory containing onion output CSVs
+        file_list_path: Path to original file list (to resolve line numbers)
+
+    Returns:
+        Dictionary mapping each file to its set of duplicates
+    """
+    csv_dir = Path(csv_dir)
+
+    file_list = []
+    with open(file_list_path, encoding="utf-8") as f:
+        file_list = [line.strip() for line in f if line.strip()]
+
+    logger.info(f"Loaded {len(file_list)} files from file list")
+
+    csv_files = list(csv_dir.glob("*.csv"))
+    logger.info(f"Found {len(csv_files)} CSV files in {csv_dir}")
+
+    duplicate_groups = []
+
+    for csv_file in tqdm(csv_files, desc="Parsing duplicate CSVs", unit="file"):
+        entries = parse_onion_csv(csv_file)
+
+        if not entries:
+            logger.debug(f"CSV {csv_file.name} has no entries")
+            continue
+
+        logger.debug(f"CSV {csv_file.name}: {len(entries)} entries")
+
+        group = set()
+        for _, line_num in entries:
+            if 1 <= line_num <= len(file_list):
+                resolved_path = file_list[line_num - 1]
+                group.add(resolved_path)
+                logger.debug(f"  Line {line_num} -> {Path(resolved_path).name}")
+            else:
+                logger.warning(f"Invalid line number {line_num} in {csv_file.name}")
+
+        logger.debug(f"  Group has {len(group)} unique files")
+        if len(group) > 1:
+            duplicate_groups.append(group)
+            logger.info(f"Found duplicate group with {len(group)} files from {csv_file.name}")
+        else:
+            logger.debug(f"  Skipping group with only {len(group)} file(s)")
+
+    logger.info(f"Found {len(duplicate_groups)} duplicate groups")
+
+    file_to_duplicates = defaultdict(set)
+
+    for group in duplicate_groups:
+        for file_path in group:
+            duplicates = group - {file_path}
+            file_to_duplicates[file_path].update(duplicates)
+
+    logger.info(f"Total files with duplicates: {len(file_to_duplicates)}")
+
+    return dict(file_to_duplicates)
+
+
+def create_duplicate_pairs_with_scores(
+    csv_dir: Path,
+    file_list_path: Path,
+) -> list[dict]:
+    """
+    Create list of duplicate pairs with similarity scores.
+
+    Parses onion phase 2 output which includes similarity scores.
+
+    Args:
+        csv_dir: Directory containing onion phase 2 CSVs
+        file_list_path: Path to file list
+
+    Returns:
+        List of duplicate pair dictionaries with format:
+        {
+            'doc1': file_path_1,
+            'doc2': file_path_2,
+            'similarity': score (0.0-1.0)
+        }
+    """
+    csv_dir = Path(csv_dir)
+
+    file_list = []
+    with open(file_list_path, encoding="utf-8") as f:
+        file_list = [line.strip() for line in f if line.strip()]
+
+    logger.info(f"Processing phase 2 outputs from {csv_dir}")
+
+    pairs = []
+    csv_files = list(csv_dir.glob("*.csv"))
+
+    for csv_file in tqdm(csv_files, desc="Processing similarity scores", unit="file"):
+        try:
+            with open(csv_file, encoding="utf-8") as f:
+                reader = csv.reader(f)
+                rows = list(reader)
+
+                if not rows:
+                    continue
+
+                for row in rows:
+                    if len(row) < 2:
+                        continue
+
+                    file_path = row[0].strip().strip('"')
+                    line_num = int(row[1]) if row[1].isdigit() else None
+
+                    score = 1.0
+                    if len(row) > 2:
+                        with contextlib.suppress(ValueError, IndexError):
+                            score = float(row[2])
+
+                    if not file_path and line_num and 1 <= line_num <= len(file_list):
+                        file_path = file_list[line_num - 1]
+
+                    if file_path:
+                        pairs.append(
+                            {
+                                "source_csv": csv_file.stem,
+                                "file_path": file_path,
+                                "similarity": score,
+                            }
+                        )
+
+        except Exception as e:
+            logger.warning(f"Error processing {csv_file}: {e}")
+
+    logger.info(f"Extracted {len(pairs)} duplicate entries from CSVs")
+
+    csv_groups = defaultdict(list)
+    for entry in pairs:
+        csv_groups[entry["source_csv"]].append(entry)
+
+    duplicate_pairs = []
+    for _, entries in csv_groups.items():
+        if len(entries) < 2:
+            continue
+
+        source = entries[0]["file_path"]
+
+        for entry in entries[1:]:
+            duplicate_pairs.append(
+                {
+                    "doc1": source,
+                    "doc2": entry["file_path"],
+                    "similarity": entry["similarity"],
+                }
+            )
+
+    logger.info(f"Created {len(duplicate_pairs)} duplicate pairs")
+
+    return duplicate_pairs
+
+
+__all__ = [
+    "parse_onion_csv",
+    "extract_duplicates_from_csvs",
+    "create_duplicate_pairs_with_scores",
+]

dalla/deduplication/preprocessing.py

@@ -0,0 +1,120 @@
+"""
+Preprocessing utilities for deduplication.
+
+Handles text normalization and conversion to vertical format for onion.
+"""
+
+import re
+from pathlib import Path
+
+from camel_tools.utils.dediac import dediac_ar
+from tqdm import tqdm
+
+from dalla.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def text_to_vertical(text: str, doc_id: str = "") -> str:
+    """
+    Convert text to vertical format (one word per line).
+
+    This format is required by the onion deduplication algorithm.
+    Uses CAMEL Tools tokenizer to properly split words and preserve spaces.
+
+    Args:
+        text: Input text (can be None, will be converted to empty string)
+        doc_id: Document identifier (optional)
+
+    Returns:
+        Text in vertical format with optional doc tags
+    """
+    if text is None or not isinstance(text, str):
+        text = ""
+
+    text = dediac_ar(text)
+
+    words = text.split()
+
+    vertical = "\n".join(words)
+
+    if doc_id:
+        vertical = f"<doc name='{doc_id}'>\n{vertical}\n</doc>"
+
+    return vertical
+
+
+def create_vert_files(
+    texts: list[str],
+    ids: list[str],
+    output_dir: Path,
+) -> tuple[list[Path], dict[str, str]]:
+    """
+    Create vertical format files from texts.
+
+    Args:
+        texts: List of text strings
+        ids: List of document IDs
+        output_dir: Directory to save vert files
+
+    Returns:
+        Tuple of (list of file paths, mapping of original IDs to vert file paths)
+    """
+    output_dir = Path(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    logger.info(f"Creating {len(texts)} vert files in {output_dir}")
+
+    file_paths = []
+    id_mapping = {}
+
+    for idx, (text, doc_id) in enumerate(
+        tqdm(zip(texts, ids, strict=False), total=len(texts), desc="Creating vert files")
+    ):
+        safe_filename = re.sub(r"[^\w\-_.]", "_", str(doc_id))
+        if len(safe_filename) > 200:
+            safe_filename = safe_filename[-200:]
+
+        filename = f"{idx:08d}_{safe_filename}.txt"
+        filepath = output_dir / filename
+        vert_text = text_to_vertical(text, doc_id)
+
+        with open(filepath, "w", encoding="utf-8") as f:
+            f.write(vert_text)
+
+        file_paths.append(filepath)
+        id_mapping[str(doc_id)] = str(filepath)
+
+    logger.info(f"Created {len(file_paths)} vert files")
+
+    return file_paths, id_mapping
+
+
+def create_file_list(file_paths: list[Path], output_file: Path) -> Path:
+    """
+    Create a text file listing all file paths (input for onion).
+
+    Args:
+        file_paths: List of file paths
+        output_file: Path to output file list
+
+    Returns:
+        Path to created file list
+    """
+    output_file = Path(output_file)
+    output_file.parent.mkdir(parents=True, exist_ok=True)
+
+    with open(output_file, "w", encoding="utf-8") as f:
+        for filepath in file_paths:
+            f.write(f"{filepath}\n")
+
+    logger.info(f"Created file list with {len(file_paths)} entries: {output_file}")
+
+    return output_file
+
+
+__all__ = [
+    "text_to_vertical",
+    "create_vert_files",
+    "create_file_list",
+]

dalla/quality/__init__.py

@@ -0,0 +1,5 @@
+"""Quality checking module for text quality assessment."""
+
+from dalla.quality.checker import QualityChecker, check_quality
+
+__all__ = ["check_quality", "QualityChecker"]