pseek 3.5.0__tar.gz → 3.6.0__tar.gz

{pseek-3.5.0 → pseek-3.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pseek
-Version: 3.5.0
+Version: 3.6.0
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
@@ -75,6 +75,7 @@ Fast and powerful command-line search tool for finding files, directories, and t
         * [Exclude](#exclude)
     * [Size](#size-filtering)
     * [Depth](#depth)
+* [Parallel Search](#parallel-search)
 * [Archive Search](#archive-search)
     * [Nested Archives](#nested-archives)
 * [Archive Filters](#archive-filters)
@@ -236,6 +237,7 @@ psk --exclude build
 | `--absolute-path` | Display full path of files and directories |
 | `--paths-only` | Only show matching file paths for content search |
 | `--stats` | Show search statistics including result counts and search time |
+| `--threads` | This flag sets the approximate number of threads to use. A value of 0 (which is the default) causes pseek to choose the thread count using heuristics |
 
 ## Specifying the Root Directory
 
@@ -720,6 +722,29 @@ project/                  ← base directory
 
 If `--depth` is not specified, the search has no depth limit.
 
+## Parallel Search
+
+Pseek can process multiple file paths concurrently to improve search performance, especially when searching large numbers of files.
+
+Use `--threads` to control the maximum number of worker threads:
+
+```bash
+# Use the default number of worker threads
+psk TODO
+
+# Use 4 worker threads
+psk TODO --threads 4
+
+# Use a single worker thread
+psk TODO --threads 1
+```
+
+If `--threads` is not specified, Pseek automatically selects the number of workers based on the available CPU count. Setting `--threads 1` disables parallel processing of candidate paths.
+
+More threads do not always mean faster searches. Performance depends on factors such as file sizes, storage speed, and the type of search being performed. You can experiment with different values to find the best setting for your workload.
+
+> **Note:** Parallel processing may change the order in which results are produced. Search filters and matching rules remain unchanged.
+
 ## Archive Search
 
 Enable archive support:

{pseek-3.5.0 → pseek-3.6.0}/README.md

@@ -47,6 +47,7 @@ Fast and powerful command-line search tool for finding files, directories, and t
         * [Exclude](#exclude)
     * [Size](#size-filtering)
     * [Depth](#depth)
+* [Parallel Search](#parallel-search)
 * [Archive Search](#archive-search)
     * [Nested Archives](#nested-archives)
 * [Archive Filters](#archive-filters)
@@ -208,6 +209,7 @@ psk --exclude build
 | `--absolute-path` | Display full path of files and directories |
 | `--paths-only` | Only show matching file paths for content search |
 | `--stats` | Show search statistics including result counts and search time |
+| `--threads` | This flag sets the approximate number of threads to use. A value of 0 (which is the default) causes pseek to choose the thread count using heuristics |
 
 ## Specifying the Root Directory
 
@@ -692,6 +694,29 @@ project/                  ← base directory
 
 If `--depth` is not specified, the search has no depth limit.
 
+## Parallel Search
+
+Pseek can process multiple file paths concurrently to improve search performance, especially when searching large numbers of files.
+
+Use `--threads` to control the maximum number of worker threads:
+
+```bash
+# Use the default number of worker threads
+psk TODO
+
+# Use 4 worker threads
+psk TODO --threads 4
+
+# Use a single worker thread
+psk TODO --threads 1
+```
+
+If `--threads` is not specified, Pseek automatically selects the number of workers based on the available CPU count. Setting `--threads 1` disables parallel processing of candidate paths.
+
+More threads do not always mean faster searches. Performance depends on factors such as file sizes, storage speed, and the type of search being performed. You can experiment with different values to find the best setting for your workload.
+
+> **Note:** Parallel processing may change the order in which results are produced. Search filters and matching rules remain unchanged.
+
 ## Archive Search
 
 Enable archive support:

{pseek-3.5.0 → pseek-3.6.0}/pseek/archive.py

@@ -1,8 +1,17 @@
+import bz2
+import gzip
 import io
+import lzma
+import tarfile
+
+# Archive modules
+import zipfile
 from pathlib import Path
+
+import py7zr
+import rarfile
+
 from .utils import get_path_suffix, is_binary
-# Archive modules
-import zipfile, py7zr, tarfile, gzip, bz2, lzma, rarfile
 
 # Archive extensions that are allowed
 ARCHIVE_EXTS = ('zip', 'rar', '7z', 'tar', 'tar.gz', 'tar.bz2', 'tar.xz', 'gz', 'bz2', 'xz')

{pseek-3.5.0 → pseek-3.6.0}/pseek/cli.py

@@ -1,10 +1,16 @@
-import click, time, shutil, rarfile, platform
-from .searcher import seek
-from .structs import SearchConfig
-from multiprocessing import Process, Queue
-from queue import Empty
+import platform
+import shutil
+import time
 from collections import defaultdict
+from multiprocessing import Process, Queue
 from pathlib import Path
+from queue import Empty
+
+import click
+import rarfile
+
+from .searcher import seek
+from .structs import SearchConfig
 
 BACKEND_PATH = Path(__file__).parent / "RARBackend"
 
@@ -424,6 +430,9 @@ def check_rar_backend(archive_enabled: bool, tool_path: str, backend: str):
 @click.option('--timeout', metavar='FLOAT',
               type=click.FloatRange(min=0, min_open=True),
               help='Stop the search after the specified number of seconds.')
+@click.option('--threads', metavar='NUM', type=click.IntRange(min=0), default=0,
+              help='This flag sets the approximate number of threads to use. '
+              'A value of 0 (which is the default) causes pseek to choose the thread count using heuristics.')
 def search(**kwargs):
     """Search for files, directories, and file content based on the query."""
 

{pseek-3.5.0 → pseek-3.6.0}/pseek/parser.py

@@ -1,8 +1,12 @@
-import re, click, json
+import json
+import re
+
+import click
 from lark import Lark, Transformer
-from .utils import compile_regex
 from rapidfuzz import fuzz
 
+from .utils import compile_regex
+
 
 class ExprNode:
     """Base class for expression tree nodes"""

{pseek-3.5.0 → pseek-3.6.0}/pseek/searcher.py

@@ -1,12 +1,18 @@
-import mmap, os
+import mmap
+import os
+from collections import defaultdict, deque
+from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
 from errno import ELOOP
-from pseek import _pignore
+from itertools import islice
 from pathlib import Path
-from collections import defaultdict, deque
-from .utils import get_path_suffix, is_binary
-from .parser import parse_query_expression, TermNode, find_matches
+from threading import RLock
+
+from pseek import _pignore
+
 from .archive import ARCHIVE_EXTS, extract_names_from_archive, extract_text_from_archive
-from .structs import FileDirResult, ContentResult, MatchGroup, Line
+from .parser import TermNode, find_matches, parse_query_expression
+from .structs import ContentResult, FileDirResult, Line, MatchGroup
+from .utils import get_path_suffix, is_binary
 
 
 def should_skip(config, p: Path, file_ext: str, p_size: int) -> bool:
@@ -14,7 +20,7 @@ def should_skip(config, p: Path, file_ext: str, p_size: int) -> bool:
     Check whether the file/directory should be skipped based on various filters.
     Returns True if the path should be skipped.
     """
-    if (config.ext and file_ext not in config.ext) \
+    return (config.ext and file_ext not in config.ext) \
         or (config.exclude_ext and file_ext in config.exclude_ext) \
         or (config.size and (p.is_dir() or not any(  # .stat().st_size doesn't give actual size of dir.
                                                      # To measure the actual size,
@@ -22,35 +28,37 @@ def should_skip(config, p: Path, file_ext: str, p_size: int) -> bool:
                                                      # which is time-consuming
             minimum <= p_size <= maximum
             for minimum, maximum in config.size
-        ))):
-        return True
-
-    # Filter by regex include and exclude
-    str_p = str(p)
-    if config.re_include:
-        return not config.re_include.search(str_p)
-    if config.re_exclude:
-        return config.re_exclude.search(str_p) is not None
+        )))
 
-    return False
 
-
-def add_result(matches, result_queue, match_type, value):
-    if result_queue:
+def add_result(matches, result_queue, match_type, value, lock):
+    if result_queue is not None:
+        # Queue.put() is thread-safe for standard Python queues
         result_queue.put((match_type, value))
-    else:
+        return
+
+    if lock is None:
         matches[match_type].append(value)
+    else:
+        with lock:
+            matches[match_type].append(value)
 
 
-def add_metric(metrics, result_queue, name, value):
-    if result_queue:
+def add_metric(metrics, result_queue, name, value, lock):
+    if result_queue is not None:
+        # Queue.put() is thread-safe for standard Python queues
         result_queue.put(('metric', (name, value)))
-    else:
+        return
+
+    if lock is None:
         metrics[name].add(value)
+    else:
+        with lock:
+            metrics[name].add(value)
 
 
 def search_file_and_dir(config, matches: dict, pattern, p: Path,
-                        p_ext: str, metrics, result_queue):
+                        p_ext: str, metrics, result_queue, result_lock):
     """Search files and folders on the system and within archive files"""
 
     path_str = str(p)
@@ -78,7 +86,8 @@ def search_file_and_dir(config, matches: dict, pattern, p: Path,
             FileDirResult(
                 path=path_str,
                 matches=name_matches
-            )
+            ),
+            result_lock
         )
 
     # Search for files and directories name inside archive files if archive is active
@@ -90,14 +99,32 @@ def search_file_and_dir(config, matches: dict, pattern, p: Path,
                 full_virtual_path = (path_str, *virtual_path, name_str)
                 
                 if not is_dir and config.file:
-                    add_metric(metrics, result_queue, 'files_scanned', full_virtual_path)
+                    add_metric(
+                        metrics,
+                        result_queue,
+                        'files_scanned',
+                        full_virtual_path,
+                        result_lock
+                    )
                 elif is_dir:
-                    add_metric(metrics, result_queue, 'directories_scanned', full_virtual_path)
+                    add_metric(
+                        metrics,
+                        result_queue,
+                        'directories_scanned',
+                        full_virtual_path,
+                        result_lock
+                    )
                 
                 # We cannot use "full_virtual_path[-1]" because it may be limited by --arc-depth
                 # and may not enter the archive at all to scan inside it
                 if len(full_virtual_path) >= 3 and get_path_suffix(full_virtual_path[-2]) in ARCHIVE_EXTS[:-3]:
-                    add_metric(metrics, result_queue, 'archives_scanned', full_virtual_path)
+                    add_metric(
+                        metrics,
+                        result_queue,
+                        'archives_scanned',
+                        full_virtual_path,
+                        result_lock
+                    )
 
             arc_match_type = (
                 'file' if config.file and not is_dir else
@@ -122,7 +149,8 @@ def search_file_and_dir(config, matches: dict, pattern, p: Path,
                         path=path_str,
                         matches=name_matches,
                         virtual_path=[*virtual_path, name_str]
-                    )
+                    ),
+                    result_lock
                 )
 
 
@@ -202,7 +230,7 @@ class ContextCollector:
 
 
 def search_content(config, matches: dict, pattern, binary_pattern,
-                   p: Path, p_ext: str, metrics, result_queue):
+                   p: Path, p_ext: str, metrics, result_queue, result_lock):
     """Search within the contents of system files and files inside archive files"""
 
     path_str = str(p)
@@ -217,12 +245,19 @@ def search_content(config, matches: dict, pattern, binary_pattern,
                     metrics,
                     result_queue,
                     'files_scanned',
-                    full_virtual_path if virtual_path else path_str
+                    full_virtual_path if virtual_path else path_str,
+                    result_lock
                 )
 
                 if (len(full_virtual_path) >= 3 and get_path_suffix(full_virtual_path[-2]) in ARCHIVE_EXTS[:-3] \
                     or get_path_suffix(full_virtual_path[-1]) in ARCHIVE_EXTS[-3:]):
-                    add_metric(metrics, result_queue, 'archives_scanned', full_virtual_path)
+                    add_metric(
+                        metrics,
+                        result_queue,
+                        'archives_scanned',
+                        full_virtual_path,
+                        result_lock
+                    )
 
             if binary_pattern and not content_contains(content, binary_pattern):
                 continue
@@ -245,7 +280,8 @@ def search_content(config, matches: dict, pattern, binary_pattern,
                         ContentResult(
                             path=path_str,
                             virtual_path=virtual_path
-                        )
+                        ),
+                        result_lock
                     )
                     break
 
@@ -266,7 +302,8 @@ def search_content(config, matches: dict, pattern, binary_pattern,
                         path=path_str,
                         virtual_path=virtual_path,
                         groups=groups
-                    )
+                    ),
+                    result_lock
                 )
 
         # Skip next block to avoid searching the contents of archive files
@@ -297,7 +334,8 @@ def search_content(config, matches: dict, pattern, binary_pattern,
                     matches,
                     result_queue,
                     'content',
-                    ContentResult(path=path_str)
+                    ContentResult(path=path_str),
+                    result_lock
                 )
                 return
 
@@ -317,7 +355,8 @@ def search_content(config, matches: dict, pattern, binary_pattern,
             ContentResult(
                 path=path_str,
                 groups=groups
-            )
+            ),
+            result_lock
         )
 
 
@@ -353,6 +392,10 @@ def walk_logic(path: Path, config, depth: int, matcher, ancestors: set):
                 # Prune explicitly excluded paths before further processing
                 if entry_path in config.exclude:
                     continue
+                # Filter by regex include and exclude
+                if (config.re_include and not config.re_include.search(entry_path.name)) \
+                    or (config.re_exclude and config.re_exclude.search(entry_path.name) is not None):
+                    continue
 
                 try:
                     is_dir = entry.is_dir(follow_symlinks=config.follow)
@@ -409,6 +452,7 @@ def walk_logic(path: Path, config, depth: int, matcher, ancestors: set):
         ancestors.remove(dir_key)
 
 
+# TODO: Make traversal parallelize just like seek func
 def walk(config):
     """Include paths define traversal roots, avoiding unnecessary traversal 
     of unrelated parts of the tree"""
@@ -440,62 +484,158 @@ def walk(config):
             yield from walk_logic(root, config, depth, matcher, set())
 
 
-def seek(config, result_queue=None) -> dict:
-    """Main search function"""
-    pattern = parse_query_expression(config)
-    # If expression is simple and is a single TermNode, we can use binary pattern
-    if config.content:
-        if isinstance(pattern, TermNode):
-            binary_pattern = pattern.get_binary_pattern()
-        else:
-            binary_pattern = None
+def process_candidate(item, config, pattern, binary_pattern,
+                      matches, metrics, result_queue, result_lock):
+    p, metric_only = item
 
-    matches = {'file': [], 'directory': [], 'content': []}
-    metrics = defaultdict(set)
+    # Dir metrics must be added before should_skip,
+    # as it can influence these metrics
+    if config.stats and p.is_dir():
+        add_metric(
+            metrics,
+            result_queue,
+            'directories_scanned',
+            str(p),
+            result_lock
+        )
 
-    for p, metric_only in walk(config):
-        # Dir metrics must be added before should_skip,
-        # as it can influence these metrics
-        if config.stats and p.is_dir():
-            add_metric(metrics, result_queue, 'directories_scanned', str(p))
+    if metric_only:
+        return
 
-        if metric_only:
-            continue
+    try:
+        p_ext = get_path_suffix(p)
+        # Path.state follows symlinks, so we use lstate here
+        p_size = p.lstat().st_size
+    except OSError:
+        return
 
-        try:
-            p_ext = get_path_suffix(p)
-            # Path.state follows symlinks, so we use lstate here
-            p_size = p.lstat().st_size
-        except OSError:
-            continue
+    if should_skip(config, p, p_ext, p_size):
+        return
 
-        if should_skip(config, p, p_ext, p_size):
-            continue
+    if config.stats:
+        path_str = str(p)
 
-        if config.stats:
-            path_str = str(p)
+        if (config.file or config.content) and p.is_file():
+            add_metric(
+                metrics,
+                result_queue,
+                'files_scanned',
+                path_str,
+                result_lock
+            )
 
-            if (config.file or config.content) and p.is_file():
-                add_metric(metrics, result_queue, 'files_scanned', path_str)
+        if config.archive and p_ext in ARCHIVE_EXTS:
+            add_metric(
+                metrics,
+                result_queue,
+                'archives_scanned',
+                (path_str,),
+                result_lock
+            )
 
-            if config.archive and p_ext in ARCHIVE_EXTS:
-                add_metric(
+    # Search for files and directories if requested
+    if config.file or config.directory:
+        search_file_and_dir(
+            config,
+            matches,
+            pattern,
+            p,
+            p_ext,
+            metrics,
+            result_queue,
+            result_lock
+        )
+
+    # Search for content inside files if requested
+    # Avoid empty files
+    if config.content and not p.is_symlink() \
+        and p.is_file() and p_size != 0:
+        search_content(
+            config,
+            matches,
+            pattern,
+            binary_pattern,
+            p,
+            p_ext,
+            metrics,
+            result_queue,
+            result_lock
+        )
+
+
+def batched(iterable, size):
+    # Convert iterable to iterator to prevent
+    # islice from creating identical batches.
+    # Example of using iterable:
+    # items = [1, 2, 3, 4, 5, 6]
+    # list(islice(items, 3)) -> [1, 2, 3]
+    # list(islice(items, 3)) -> Again [1, 2, 3]!
+    iterator = iter(iterable)
+
+    # If no elements remain, the batch becomes [],
+    # which causes while loop to terminate
+    while batch := list(islice(iterator, size)):
+        yield batch
+
+
+def process_batch(items, *args):
+    for item in items:
+        process_candidate(item, *args)
+
+
+def seek(config, result_queue=None) -> dict:
+    """Main search function"""
+    pattern = parse_query_expression(config)
+    # If expression is simple and is a single TermNode, we can use binary pattern
+    binary_pattern = None
+    if config.content and isinstance(pattern, TermNode):
+        binary_pattern = pattern.get_binary_pattern()
+
+    matches = {
+        'file': [],
+        'directory': [],
+        'content': []
+    }
+    metrics = defaultdict(set)
+    result_lock = RLock()
+
+    workers = config.threads or (os.cpu_count() or 1)
+
+    with ThreadPoolExecutor(
+        max_workers=workers,
+        thread_name_prefix='pseek'
+    ) as executor:
+        pending = set()
+
+        for batch in batched(walk(config), 16):
+            pending.add(
+                executor.submit(
+                    process_batch,
+                    batch,
+                    config,
+                    pattern,
+                    binary_pattern,
+                    matches,
                     metrics,
                     result_queue,
-                    'archives_scanned',
-                    (path_str,)
+                    result_lock
                 )
+            )
+
+            # Limit queued work to avoid excessive memory usage
+            # Using workers as max number of pending items
+            if len(pending) >= workers:
+                done, pending = wait(
+                    pending,
+                    return_when=FIRST_COMPLETED
+                )
+
+                # Don't ignore exceptions. If an exception occurs, re-raise it in main thread
+                for future in done:
+                    future.result()
 
-        # Search for files and directories if requested
-        if config.file or config.directory:
-            search_file_and_dir(config, matches, pattern, p,
-                                p_ext, metrics, result_queue)
-        
-        # Search for content inside files if requested
-        # Avoid empty files
-        if config.content and not p.is_symlink() \
-            and p.is_file() and p_size != 0:
-            search_content(config, matches, pattern, binary_pattern, p,
-                           p_ext, metrics, result_queue)
+        # Wait for workers that haven't finished their work yet
+        for future in pending:
+            future.result()
 
     return matches, metrics

{pseek-3.5.0 → pseek-3.6.0}/pseek/structs.py

@@ -1,8 +1,11 @@
-from click import BadParameter
-import re, os
-from pathlib import Path
+import os
+import re
 from dataclasses import dataclass, field
 from math import isfinite
+from pathlib import Path
+
+from click import BadParameter
+
 from .utils import compile_regex
 
 SUFFIXES = {
@@ -166,6 +169,7 @@ class SearchConfig:
     paths_only: bool
     stats: bool
     context: tuple[int, int]
+    threads: int
     
     def __post_init__(self):
         """Post-initialization processing to normalize and validate inputs"""

{pseek-3.5.0 → pseek-3.6.0}/pseek/utils.py

@@ -1,6 +1,8 @@
-import re, click
+import re
 from pathlib import Path
 
+import click
+
 EXTENSIONS_PATH = Path(__file__).parent / "compound_extensions"
 try:
     with open(EXTENSIONS_PATH, "r") as f:

{pseek-3.5.0 → pseek-3.6.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "pseek"
-version = "3.5.0"
+version = "3.6.0"
 description = "Pseek is a Python CLI toolkit to search files, folders, and text"
 readme = "README.md"
 license = "MIT"