pseek 3.4.2__tar.gz → 3.6.0__tar.gz

{pseek-3.4.2 → pseek-3.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pseek
-Version: 3.4.2
+Version: 3.6.0
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
@@ -50,6 +50,7 @@ Fast and powerful command-line search tool for finding files, directories, and t
     * [Regular Expression (Regex)](#regular-expression-search)
     * [Fuzzy](#fuzzy-search)
         * [Fuzzy Similarity Threshold](#fuzzy-similarity-threshold)
+    * [Full Path](#full-path)
 * [Expression Queries](#expression-queries)
     * [Operators](#supported-operators)
         * [AND](#and)
@@ -74,6 +75,7 @@ Fast and powerful command-line search tool for finding files, directories, and t
         * [Exclude](#exclude)
     * [Size](#size-filtering)
     * [Depth](#depth)
+* [Parallel Search](#parallel-search)
 * [Archive Search](#archive-search)
     * [Nested Archives](#nested-archives)
 * [Archive Filters](#archive-filters)
@@ -209,6 +211,7 @@ psk --exclude build
 | `--word` | Match the whole word only (except when `--expr` is enabled, in which case you can make it match whole word by putting `w` before term: `w"foo"`) |
 | `--expr` | Enable boolean query expressions. Example: `r"foo.*bar" and ("bar" or "baz") and not "qux"`. Prefixes: `r=regex`, `c=case-sensitive`, `w=whole-word`, `f=fuzzy` |
 | `--context` | Show N context lines before and after matches, or use N:M to specify before:after |
+| `--full-path` | By default, query is only matched against the filename or directory name (this applies only when searching for files or directories, not file contents). Using this flag, query is matched against the full path (and, if `--absolute-path` flag is enabled, against the absolute path) |
 | `--depth` | Limit directory traversal to given depth range. By default, there is no limit on search depth |
 | `--timeout` | Stop the search after the specified number of seconds |
 | `--fuzzy` | Enable fuzzy search (Highlighting and counting matches are disabled in this mode if `--word` is not enabled to prevent the program from slowing down). except when `--expr` is enabled, in which case you can make it fuzzy by putting `f` before term: `f"foo"` |
@@ -234,6 +237,7 @@ psk --exclude build
 | `--absolute-path` | Display full path of files and directories |
 | `--paths-only` | Only show matching file paths for content search |
 | `--stats` | Show search statistics including result counts and search time |
+| `--threads` | This flag sets the approximate number of threads to use. A value of 0 (which is the default) causes pseek to choose the thread count using heuristics |
 
 ## Specifying the Root Directory
 
@@ -351,6 +355,45 @@ Examples:
 
 Default: `80`
 
+### Full Path
+
+By default, Pseek matches file and directory queries against the entry's name only. Use `--full-path` to match against its full path within the search scope instead. This option affects file and directory searches, not content searches.
+
+For example, given the relative path `pseek/utils.py`, you can match part of the path rather than just the filename:
+
+```bash
+# Match against the full relative path
+psk "seek/utils" --full-path
+```
+
+**Relative paths vs. absolute paths**
+
+`--full-path` does not automatically include the absolute path. If the file's absolute path is:
+
+```text
+/home/python/cli/pseek/utils.py
+```
+
+a query such as `python/cli` will not match unless `--absolute-path` is also enabled:
+
+```bash
+psk "python/cli" --full-path --absolute-path
+```
+
+**Nested archives**
+
+When searching nested archives, `--full-path` applies only to the path within the current archive level, not the entire nested archive chain.
+
+For example:
+
+```text
+archive.zip::folder/arc.zip::fd/file.txt
+```
+
+With `--full-path`, you can match `fd/f` against the entry `fd/file.txt` inside the innermost archive. A query such as `zip::fd` does not match across archive boundaries. Without `--full-path`, matching is limited to the entry name, `file.txt`.
+
+> **Note:** For ordinary queries without `--regex`, backslashes are treated literally, so Windows paths can be searched without additional escaping. When using `--expr`, quoted terms use JSON-style escaping; escape backslashes accordingly. For example, `"\\"` represents a single literal backslash, while `"\\n"` represents the literal sequence `\n`.
+
 ## Expression Queries
 
 Expression mode enables logical search expressions.
@@ -584,6 +627,8 @@ psk TODO \
 
 Only search inside those paths.
 
+> **Note:** Paths manually included by the user take precedence over other filters; this means that even if a path conflicts with rules related to [--glob](#globs), [--exclude](#exclude-paths), etc., it will still be included in the results.
+
 #### Exclude Paths
 
 ```bash
@@ -677,6 +722,29 @@ project/                  ← base directory
 
 If `--depth` is not specified, the search has no depth limit.
 
+## Parallel Search
+
+Pseek can process multiple file paths concurrently to improve search performance, especially when searching large numbers of files.
+
+Use `--threads` to control the maximum number of worker threads:
+
+```bash
+# Use the default number of worker threads
+psk TODO
+
+# Use 4 worker threads
+psk TODO --threads 4
+
+# Use a single worker thread
+psk TODO --threads 1
+```
+
+If `--threads` is not specified, Pseek automatically selects the number of workers based on the available CPU count. Setting `--threads 1` disables parallel processing of candidate paths.
+
+More threads do not always mean faster searches. Performance depends on factors such as file sizes, storage speed, and the type of search being performed. You can experiment with different values to find the best setting for your workload.
+
+> **Note:** Parallel processing may change the order in which results are produced. Search filters and matching rules remain unchanged.
+
 ## Archive Search
 
 Enable archive support:

{pseek-3.4.2 → pseek-3.6.0}/README.md

@@ -22,6 +22,7 @@ Fast and powerful command-line search tool for finding files, directories, and t
     * [Regular Expression (Regex)](#regular-expression-search)
     * [Fuzzy](#fuzzy-search)
         * [Fuzzy Similarity Threshold](#fuzzy-similarity-threshold)
+    * [Full Path](#full-path)
 * [Expression Queries](#expression-queries)
     * [Operators](#supported-operators)
         * [AND](#and)
@@ -46,6 +47,7 @@ Fast and powerful command-line search tool for finding files, directories, and t
         * [Exclude](#exclude)
     * [Size](#size-filtering)
     * [Depth](#depth)
+* [Parallel Search](#parallel-search)
 * [Archive Search](#archive-search)
     * [Nested Archives](#nested-archives)
 * [Archive Filters](#archive-filters)
@@ -181,6 +183,7 @@ psk --exclude build
 | `--word` | Match the whole word only (except when `--expr` is enabled, in which case you can make it match whole word by putting `w` before term: `w"foo"`) |
 | `--expr` | Enable boolean query expressions. Example: `r"foo.*bar" and ("bar" or "baz") and not "qux"`. Prefixes: `r=regex`, `c=case-sensitive`, `w=whole-word`, `f=fuzzy` |
 | `--context` | Show N context lines before and after matches, or use N:M to specify before:after |
+| `--full-path` | By default, query is only matched against the filename or directory name (this applies only when searching for files or directories, not file contents). Using this flag, query is matched against the full path (and, if `--absolute-path` flag is enabled, against the absolute path) |
 | `--depth` | Limit directory traversal to given depth range. By default, there is no limit on search depth |
 | `--timeout` | Stop the search after the specified number of seconds |
 | `--fuzzy` | Enable fuzzy search (Highlighting and counting matches are disabled in this mode if `--word` is not enabled to prevent the program from slowing down). except when `--expr` is enabled, in which case you can make it fuzzy by putting `f` before term: `f"foo"` |
@@ -206,6 +209,7 @@ psk --exclude build
 | `--absolute-path` | Display full path of files and directories |
 | `--paths-only` | Only show matching file paths for content search |
 | `--stats` | Show search statistics including result counts and search time |
+| `--threads` | This flag sets the approximate number of threads to use. A value of 0 (which is the default) causes pseek to choose the thread count using heuristics |
 
 ## Specifying the Root Directory
 
@@ -323,6 +327,45 @@ Examples:
 
 Default: `80`
 
+### Full Path
+
+By default, Pseek matches file and directory queries against the entry's name only. Use `--full-path` to match against its full path within the search scope instead. This option affects file and directory searches, not content searches.
+
+For example, given the relative path `pseek/utils.py`, you can match part of the path rather than just the filename:
+
+```bash
+# Match against the full relative path
+psk "seek/utils" --full-path
+```
+
+**Relative paths vs. absolute paths**
+
+`--full-path` does not automatically include the absolute path. If the file's absolute path is:
+
+```text
+/home/python/cli/pseek/utils.py
+```
+
+a query such as `python/cli` will not match unless `--absolute-path` is also enabled:
+
+```bash
+psk "python/cli" --full-path --absolute-path
+```
+
+**Nested archives**
+
+When searching nested archives, `--full-path` applies only to the path within the current archive level, not the entire nested archive chain.
+
+For example:
+
+```text
+archive.zip::folder/arc.zip::fd/file.txt
+```
+
+With `--full-path`, you can match `fd/f` against the entry `fd/file.txt` inside the innermost archive. A query such as `zip::fd` does not match across archive boundaries. Without `--full-path`, matching is limited to the entry name, `file.txt`.
+
+> **Note:** For ordinary queries without `--regex`, backslashes are treated literally, so Windows paths can be searched without additional escaping. When using `--expr`, quoted terms use JSON-style escaping; escape backslashes accordingly. For example, `"\\"` represents a single literal backslash, while `"\\n"` represents the literal sequence `\n`.
+
 ## Expression Queries
 
 Expression mode enables logical search expressions.
@@ -556,6 +599,8 @@ psk TODO \
 
 Only search inside those paths.
 
+> **Note:** Paths manually included by the user take precedence over other filters; this means that even if a path conflicts with rules related to [--glob](#globs), [--exclude](#exclude-paths), etc., it will still be included in the results.
+
 #### Exclude Paths
 
 ```bash
@@ -649,6 +694,29 @@ project/                  ← base directory
 
 If `--depth` is not specified, the search has no depth limit.
 
+## Parallel Search
+
+Pseek can process multiple file paths concurrently to improve search performance, especially when searching large numbers of files.
+
+Use `--threads` to control the maximum number of worker threads:
+
+```bash
+# Use the default number of worker threads
+psk TODO
+
+# Use 4 worker threads
+psk TODO --threads 4
+
+# Use a single worker thread
+psk TODO --threads 1
+```
+
+If `--threads` is not specified, Pseek automatically selects the number of workers based on the available CPU count. Setting `--threads 1` disables parallel processing of candidate paths.
+
+More threads do not always mean faster searches. Performance depends on factors such as file sizes, storage speed, and the type of search being performed. You can experiment with different values to find the best setting for your workload.
+
+> **Note:** Parallel processing may change the order in which results are produced. Search filters and matching rules remain unchanged.
+
 ## Archive Search
 
 Enable archive support:

{pseek-3.4.2 → pseek-3.6.0}/pseek/archive.py

@@ -1,8 +1,17 @@
+import bz2
+import gzip
 import io
+import lzma
+import tarfile
+
+# Archive modules
+import zipfile
 from pathlib import Path
+
+import py7zr
+import rarfile
+
 from .utils import get_path_suffix, is_binary
-# Archive modules
-import zipfile, py7zr, tarfile, gzip, bz2, lzma, rarfile
 
 # Archive extensions that are allowed
 ARCHIVE_EXTS = ('zip', 'rar', '7z', 'tar', 'tar.gz', 'tar.bz2', 'tar.xz', 'gz', 'bz2', 'xz')

{pseek-3.4.2 → pseek-3.6.0}/pseek/cli.py

@@ -1,10 +1,16 @@
-import click, time, shutil, rarfile, platform
-from .searcher import seek
-from .structs import SearchConfig
-from multiprocessing import Process, Queue
-from queue import Empty
+import platform
+import shutil
+import time
 from collections import defaultdict
+from multiprocessing import Process, Queue
 from pathlib import Path
+from queue import Empty
+
+import click
+import rarfile
+
+from .searcher import seek
+from .structs import SearchConfig
 
 BACKEND_PATH = Path(__file__).parent / "RARBackend"
 
@@ -336,12 +342,14 @@ def check_rar_backend(archive_enabled: bool, tool_path: str, backend: str):
               help='Fuzzy matching threshold (1-99). Higher values require closer matches.')
 @click.option('-C', '--context', metavar='BEFORE:AFTER',
               help='Show N context lines before and after matches, or use N:M to specify before:after.')
-@click.option('--depth', multiple=True, metavar='MIN:MAX',
+@click.option('-p', '--full-path', is_flag=True,
+              help='By default, query is only matched against the filename or directory name '
+              '(this applies only when searching for files or directories, not file contents). '
+              'Using this flag, query is matched against the full path '
+              '(and, if --absolute-path flag is enabled, against the absolute path).')
+@click.option('-D', '--depth', multiple=True, metavar='MIN:MAX',
               help='Limit directory traversal to given depth range. '
               'By default, there is no limit on search depth.')
-@click.option('--timeout', metavar='FLOAT',
-              type=click.FloatRange(min=0, min_open=True),
-              help='Stop the search after the specified number of seconds.')
 # Automatic filtering
 @click.option('-u', '--unrestricted', is_flag=True,
               help='Disable all automatic filters except the binary file filter for content search.')
@@ -419,6 +427,12 @@ def check_rar_backend(archive_enabled: bool, tool_path: str, backend: str):
 @click.option('-l', '--paths-only', is_flag=True, help='Only show matching file paths for content search.')
 @click.option('-S', '--stats', is_flag=True,
               help='Show search statistics including result counts and search time.')
+@click.option('--timeout', metavar='FLOAT',
+              type=click.FloatRange(min=0, min_open=True),
+              help='Stop the search after the specified number of seconds.')
+@click.option('--threads', metavar='NUM', type=click.IntRange(min=0), default=0,
+              help='This flag sets the approximate number of threads to use. '
+              'A value of 0 (which is the default) causes pseek to choose the thread count using heuristics.')
 def search(**kwargs):
     """Search for files, directories, and file content based on the query."""
 

{pseek-3.4.2 → pseek-3.6.0}/pseek/parser.py

@@ -1,8 +1,12 @@
-import re, click, json
+import json
+import re
+
+import click
 from lark import Lark, Transformer
-from .utils import compile_regex
 from rapidfuzz import fuzz
 
+from .utils import compile_regex
+
 
 class ExprNode:
     """Base class for expression tree nodes"""

{pseek-3.4.2 → pseek-3.6.0}/pseek/searcher.py

@@ -1,12 +1,18 @@
-import mmap, os
+import mmap
+import os
+from collections import defaultdict, deque
+from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
 from errno import ELOOP
-from pseek import _pignore
+from itertools import islice
 from pathlib import Path
-from collections import defaultdict, deque
-from .utils import get_path_suffix, is_binary
-from .parser import parse_query_expression, TermNode, find_matches
+from threading import RLock
+
+from pseek import _pignore
+
 from .archive import ARCHIVE_EXTS, extract_names_from_archive, extract_text_from_archive
-from .structs import FileDirResult, ContentResult, MatchGroup, Line
+from .parser import TermNode, find_matches, parse_query_expression
+from .structs import ContentResult, FileDirResult, Line, MatchGroup
+from .utils import get_path_suffix, is_binary
 
 
 def should_skip(config, p: Path, file_ext: str, p_size: int) -> bool:
@@ -14,7 +20,7 @@ def should_skip(config, p: Path, file_ext: str, p_size: int) -> bool:
     Check whether the file/directory should be skipped based on various filters.
     Returns True if the path should be skipped.
     """
-    if (config.ext and file_ext not in config.ext) \
+    return (config.ext and file_ext not in config.ext) \
         or (config.exclude_ext and file_ext in config.exclude_ext) \
         or (config.size and (p.is_dir() or not any(  # .stat().st_size doesn't give actual size of dir.
                                                      # To measure the actual size,
@@ -22,35 +28,37 @@ def should_skip(config, p: Path, file_ext: str, p_size: int) -> bool:
                                                      # which is time-consuming
             minimum <= p_size <= maximum
             for minimum, maximum in config.size
-        ))):
-        return True
-
-    # Filter by regex include and exclude
-    str_p = str(p)
-    if config.re_include:
-        return not config.re_include.search(str_p)
-    if config.re_exclude:
-        return config.re_exclude.search(str_p) is not None
+        )))
 
-    return False
 
-
-def add_result(matches, result_queue, match_type, value):
-    if result_queue:
+def add_result(matches, result_queue, match_type, value, lock):
+    if result_queue is not None:
+        # Queue.put() is thread-safe for standard Python queues
         result_queue.put((match_type, value))
-    else:
+        return
+
+    if lock is None:
         matches[match_type].append(value)
+    else:
+        with lock:
+            matches[match_type].append(value)
 
 
-def add_metric(metrics, result_queue, name, value):
-    if result_queue:
+def add_metric(metrics, result_queue, name, value, lock):
+    if result_queue is not None:
+        # Queue.put() is thread-safe for standard Python queues
         result_queue.put(('metric', (name, value)))
-    else:
+        return
+
+    if lock is None:
         metrics[name].add(value)
+    else:
+        with lock:
+            metrics[name].add(value)
 
 
 def search_file_and_dir(config, matches: dict, pattern, p: Path,
-                        p_ext: str, metrics, result_queue):
+                        p_ext: str, metrics, result_queue, result_lock):
     """Search files and folders on the system and within archive files"""
 
     path_str = str(p)
@@ -60,14 +68,15 @@ def search_file_and_dir(config, matches: dict, pattern, p: Path,
         'directory' if config.directory and p.is_dir() else
         None
     )
+    p_name = p.name if not config.full_path else path_str
 
-    if match_type and (pattern is None or pattern.evaluate(p.name)):
+    if match_type and (pattern is None or pattern.evaluate(p_name)):
         # Find matched query in the name
         name_matches = find_matches(
             pattern,
-            p.name,
+            p_name,
             # Calculate number of chars that come before name of file or dir
-            len(path_str) - len(p.name)
+            len(path_str) - len(p_name)
         )
 
         add_result(
@@ -77,37 +86,59 @@ def search_file_and_dir(config, matches: dict, pattern, p: Path,
             FileDirResult(
                 path=path_str,
                 matches=name_matches
-            )
+            ),
+            result_lock
         )
 
     # Search for files and directories name inside archive files if archive is active
     if config.archive and p_ext in ARCHIVE_EXTS[:-3]:
         for virtual_path, name, is_dir in extract_names_from_archive(p, config):
+            name_str = str(name)
+
             if config.stats:
-                full_virtual_path = (path_str, *virtual_path, str(name))
+                full_virtual_path = (path_str, *virtual_path, name_str)
                 
                 if not is_dir and config.file:
-                    add_metric(metrics, result_queue, 'files_scanned', full_virtual_path)
+                    add_metric(
+                        metrics,
+                        result_queue,
+                        'files_scanned',
+                        full_virtual_path,
+                        result_lock
+                    )
                 elif is_dir:
-                    add_metric(metrics, result_queue, 'directories_scanned', full_virtual_path)
+                    add_metric(
+                        metrics,
+                        result_queue,
+                        'directories_scanned',
+                        full_virtual_path,
+                        result_lock
+                    )
                 
                 # We cannot use "full_virtual_path[-1]" because it may be limited by --arc-depth
                 # and may not enter the archive at all to scan inside it
                 if len(full_virtual_path) >= 3 and get_path_suffix(full_virtual_path[-2]) in ARCHIVE_EXTS[:-3]:
-                    add_metric(metrics, result_queue, 'archives_scanned', full_virtual_path)
+                    add_metric(
+                        metrics,
+                        result_queue,
+                        'archives_scanned',
+                        full_virtual_path,
+                        result_lock
+                    )
 
             arc_match_type = (
                 'file' if config.file and not is_dir else
                 'directory' if config.directory and is_dir else
                 None
             )
+            virtual_name = name.name if not config.full_path else name_str
             
-            if arc_match_type and (pattern is None or pattern.evaluate(name.name)):
+            if arc_match_type and (pattern is None or pattern.evaluate(virtual_name)):
                 name_matches = find_matches(
                     pattern,
-                    name.name,
+                    virtual_name,
                     # Calculate number of chars that come before name of file or dir
-                    len(str(name)) - len(name.name)
+                    len(name_str) - len(virtual_name)
                 )
 
                 add_result(
@@ -117,8 +148,9 @@ def search_file_and_dir(config, matches: dict, pattern, p: Path,
                     FileDirResult(
                         path=path_str,
                         matches=name_matches,
-                        virtual_path=[*virtual_path, str(name)]
-                    )
+                        virtual_path=[*virtual_path, name_str]
+                    ),
+                    result_lock
                 )
 
 
@@ -198,7 +230,7 @@ class ContextCollector:
 
 
 def search_content(config, matches: dict, pattern, binary_pattern,
-                   p: Path, p_ext: str, metrics, result_queue):
+                   p: Path, p_ext: str, metrics, result_queue, result_lock):
     """Search within the contents of system files and files inside archive files"""
 
     path_str = str(p)
@@ -213,12 +245,19 @@ def search_content(config, matches: dict, pattern, binary_pattern,
                     metrics,
                     result_queue,
                     'files_scanned',
-                    full_virtual_path if virtual_path else path_str
+                    full_virtual_path if virtual_path else path_str,
+                    result_lock
                 )
 
                 if (len(full_virtual_path) >= 3 and get_path_suffix(full_virtual_path[-2]) in ARCHIVE_EXTS[:-3] \
                     or get_path_suffix(full_virtual_path[-1]) in ARCHIVE_EXTS[-3:]):
-                    add_metric(metrics, result_queue, 'archives_scanned', full_virtual_path)
+                    add_metric(
+                        metrics,
+                        result_queue,
+                        'archives_scanned',
+                        full_virtual_path,
+                        result_lock
+                    )
 
             if binary_pattern and not content_contains(content, binary_pattern):
                 continue
@@ -241,7 +280,8 @@ def search_content(config, matches: dict, pattern, binary_pattern,
                         ContentResult(
                             path=path_str,
                             virtual_path=virtual_path
-                        )
+                        ),
+                        result_lock
                     )
                     break
 
@@ -262,7 +302,8 @@ def search_content(config, matches: dict, pattern, binary_pattern,
                         path=path_str,
                         virtual_path=virtual_path,
                         groups=groups
-                    )
+                    ),
+                    result_lock
                 )
 
         # Skip next block to avoid searching the contents of archive files
@@ -293,7 +334,8 @@ def search_content(config, matches: dict, pattern, binary_pattern,
                     matches,
                     result_queue,
                     'content',
-                    ContentResult(path=path_str)
+                    ContentResult(path=path_str),
+                    result_lock
                 )
                 return
 
@@ -313,7 +355,8 @@ def search_content(config, matches: dict, pattern, binary_pattern,
             ContentResult(
                 path=path_str,
                 groups=groups
-            )
+            ),
+            result_lock
         )
 
 
@@ -349,6 +392,10 @@ def walk_logic(path: Path, config, depth: int, matcher, ancestors: set):
                 # Prune explicitly excluded paths before further processing
                 if entry_path in config.exclude:
                     continue
+                # Filter by regex include and exclude
+                if (config.re_include and not config.re_include.search(entry_path.name)) \
+                    or (config.re_exclude and config.re_exclude.search(entry_path.name) is not None):
+                    continue
 
                 try:
                     is_dir = entry.is_dir(follow_symlinks=config.follow)
@@ -405,6 +452,7 @@ def walk_logic(path: Path, config, depth: int, matcher, ancestors: set):
         ancestors.remove(dir_key)
 
 
+# TODO: Make traversal parallelize just like seek func
 def walk(config):
     """Include paths define traversal roots, avoiding unnecessary traversal 
     of unrelated parts of the tree"""
@@ -436,62 +484,158 @@ def walk(config):
             yield from walk_logic(root, config, depth, matcher, set())
 
 
-def seek(config, result_queue=None) -> dict:
-    """Main search function"""
-    pattern = parse_query_expression(config)
-    # If expression is simple and is a single TermNode, we can use binary pattern
-    if config.content:
-        if isinstance(pattern, TermNode):
-            binary_pattern = pattern.get_binary_pattern()
-        else:
-            binary_pattern = None
+def process_candidate(item, config, pattern, binary_pattern,
+                      matches, metrics, result_queue, result_lock):
+    p, metric_only = item
 
-    matches = {'file': [], 'directory': [], 'content': []}
-    metrics = defaultdict(set)
+    # Dir metrics must be added before should_skip,
+    # as it can influence these metrics
+    if config.stats and p.is_dir():
+        add_metric(
+            metrics,
+            result_queue,
+            'directories_scanned',
+            str(p),
+            result_lock
+        )
 
-    for p, metric_only in walk(config):
-        # Dir metrics must be added before should_skip,
-        # as it can influence these metrics
-        if config.stats and p.is_dir():
-            add_metric(metrics, result_queue, 'directories_scanned', str(p))
+    if metric_only:
+        return
 
-        if metric_only:
-            continue
+    try:
+        p_ext = get_path_suffix(p)
+        # Path.state follows symlinks, so we use lstate here
+        p_size = p.lstat().st_size
+    except OSError:
+        return
 
-        try:
-            p_ext = get_path_suffix(p)
-            # Path.state follows symlinks, so we use lstate here
-            p_size = p.lstat().st_size
-        except OSError:
-            continue
+    if should_skip(config, p, p_ext, p_size):
+        return
 
-        if should_skip(config, p, p_ext, p_size):
-            continue
+    if config.stats:
+        path_str = str(p)
 
-        if config.stats:
-            path_str = str(p)
+        if (config.file or config.content) and p.is_file():
+            add_metric(
+                metrics,
+                result_queue,
+                'files_scanned',
+                path_str,
+                result_lock
+            )
 
-            if (config.file or config.content) and p.is_file():
-                add_metric(metrics, result_queue, 'files_scanned', path_str)
+        if config.archive and p_ext in ARCHIVE_EXTS:
+            add_metric(
+                metrics,
+                result_queue,
+                'archives_scanned',
+                (path_str,),
+                result_lock
+            )
 
-            if config.archive and p_ext in ARCHIVE_EXTS:
-                add_metric(
+    # Search for files and directories if requested
+    if config.file or config.directory:
+        search_file_and_dir(
+            config,
+            matches,
+            pattern,
+            p,
+            p_ext,
+            metrics,
+            result_queue,
+            result_lock
+        )
+
+    # Search for content inside files if requested
+    # Avoid empty files
+    if config.content and not p.is_symlink() \
+        and p.is_file() and p_size != 0:
+        search_content(
+            config,
+            matches,
+            pattern,
+            binary_pattern,
+            p,
+            p_ext,
+            metrics,
+            result_queue,
+            result_lock
+        )
+
+
+def batched(iterable, size):
+    # Convert iterable to iterator to prevent
+    # islice from creating identical batches.
+    # Example of using iterable:
+    # items = [1, 2, 3, 4, 5, 6]
+    # list(islice(items, 3)) -> [1, 2, 3]
+    # list(islice(items, 3)) -> Again [1, 2, 3]!
+    iterator = iter(iterable)
+
+    # If no elements remain, the batch becomes [],
+    # which causes while loop to terminate
+    while batch := list(islice(iterator, size)):
+        yield batch
+
+
+def process_batch(items, *args):
+    for item in items:
+        process_candidate(item, *args)
+
+
+def seek(config, result_queue=None) -> dict:
+    """Main search function"""
+    pattern = parse_query_expression(config)
+    # If expression is simple and is a single TermNode, we can use binary pattern
+    binary_pattern = None
+    if config.content and isinstance(pattern, TermNode):
+        binary_pattern = pattern.get_binary_pattern()
+
+    matches = {
+        'file': [],
+        'directory': [],
+        'content': []
+    }
+    metrics = defaultdict(set)
+    result_lock = RLock()
+
+    workers = config.threads or (os.cpu_count() or 1)
+
+    with ThreadPoolExecutor(
+        max_workers=workers,
+        thread_name_prefix='pseek'
+    ) as executor:
+        pending = set()
+
+        for batch in batched(walk(config), 16):
+            pending.add(
+                executor.submit(
+                    process_batch,
+                    batch,
+                    config,
+                    pattern,
+                    binary_pattern,
+                    matches,
                     metrics,
                     result_queue,
-                    'archives_scanned',
-                    (path_str,)
+                    result_lock
                 )
+            )
+
+            # Limit queued work to avoid excessive memory usage
+            # Using workers as max number of pending items
+            if len(pending) >= workers:
+                done, pending = wait(
+                    pending,
+                    return_when=FIRST_COMPLETED
+                )
+
+                # Don't ignore exceptions. If an exception occurs, re-raise it in main thread
+                for future in done:
+                    future.result()
 
-        # Search for files and directories if requested
-        if config.file or config.directory:
-            search_file_and_dir(config, matches, pattern, p,
-                                p_ext, metrics, result_queue)
-        
-        # Search for content inside files if requested
-        # Avoid empty files
-        if config.content and not p.is_symlink() \
-            and p.is_file() and p_size != 0:
-            search_content(config, matches, pattern, binary_pattern, p,
-                           p_ext, metrics, result_queue)
+        # Wait for workers that haven't finished their work yet
+        for future in pending:
+            future.result()
 
     return matches, metrics

{pseek-3.4.2 → pseek-3.6.0}/pseek/structs.py

@@ -1,8 +1,11 @@
-from click import BadParameter
-import re, os
-from pathlib import Path
+import os
+import re
 from dataclasses import dataclass, field
 from math import isfinite
+from pathlib import Path
+
+from click import BadParameter
+
 from .utils import compile_regex
 
 SUFFIXES = {
@@ -162,9 +165,11 @@ class SearchConfig:
     arc_size: list[tuple]
     rar_backend: str | None
     absolute_path: bool
+    full_path: bool
     paths_only: bool
     stats: bool
     context: tuple[int, int]
+    threads: int
     
     def __post_init__(self):
         """Post-initialization processing to normalize and validate inputs"""

{pseek-3.4.2 → pseek-3.6.0}/pseek/utils.py

@@ -1,6 +1,8 @@
-import re, click
+import re
 from pathlib import Path
 
+import click
+
 EXTENSIONS_PATH = Path(__file__).parent / "compound_extensions"
 try:
     with open(EXTENSIONS_PATH, "r") as f:

{pseek-3.4.2 → pseek-3.6.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "pseek"
-version = "3.4.2"
+version = "3.6.0"
 description = "Pseek is a Python CLI toolkit to search files, folders, and text"
 readme = "README.md"
 license = "MIT"