pseek 2.3.2__tar.gz → 2.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pseek-2.3.2 → pseek-2.4.0}/PKG-INFO +34 -20
- {pseek-2.3.2 → pseek-2.4.0}/README.md +31 -19
- {pseek-2.3.2 → pseek-2.4.0}/pseek/cli.py +32 -4
- pseek-2.4.0/pseek/searcher.py +479 -0
- pseek-2.4.0/pseek/utils.py +87 -0
- {pseek-2.3.2 → pseek-2.4.0}/pseek.egg-info/PKG-INFO +34 -20
- pseek-2.4.0/pseek.egg-info/requires.txt +4 -0
- {pseek-2.3.2 → pseek-2.4.0}/pyproject.toml +4 -2
- pseek-2.3.2/pseek/searcher.py +0 -208
- pseek-2.3.2/pseek/utils.py +0 -12
- pseek-2.3.2/pseek.egg-info/requires.txt +0 -2
- {pseek-2.3.2 → pseek-2.4.0}/LICENSE +0 -0
- {pseek-2.3.2 → pseek-2.4.0}/pseek/__init__.py +0 -0
- {pseek-2.3.2 → pseek-2.4.0}/pseek/parser.py +0 -0
- {pseek-2.3.2 → pseek-2.4.0}/pseek.egg-info/SOURCES.txt +0 -0
- {pseek-2.3.2 → pseek-2.4.0}/pseek.egg-info/dependency_links.txt +0 -0
- {pseek-2.3.2 → pseek-2.4.0}/pseek.egg-info/entry_points.txt +0 -0
- {pseek-2.3.2 → pseek-2.4.0}/pseek.egg-info/top_level.txt +0 -0
- {pseek-2.3.2 → pseek-2.4.0}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pseek
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.4.0
|
|
4
4
|
Summary: Pseek is a Python library to search files, folders, and text
|
|
5
5
|
Author-email: Arian <ariannasiri86@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -10,6 +10,8 @@ Description-Content-Type: text/markdown
|
|
|
10
10
|
License-File: LICENSE
|
|
11
11
|
Requires-Dist: click==8.1.8
|
|
12
12
|
Requires-Dist: lark==1.2.2
|
|
13
|
+
Requires-Dist: py7zr==1.0.0
|
|
14
|
+
Requires-Dist: rarfile==4.2
|
|
13
15
|
Dynamic: license-file
|
|
14
16
|
|
|
15
17
|
# Pseek
|
|
@@ -24,17 +26,19 @@ A powerful command-line tool for searching files, directories, and content insid
|
|
|
24
26
|
* **Search inside file contents**
|
|
25
27
|
* **Highlight matches** in terminal output
|
|
26
28
|
* **Optimized for speed** with ThreadPoolExecutor
|
|
29
|
+
* Support **logical expression** for search queries
|
|
30
|
+
* Search inside **archive files** (e.g. `zip`, `rar`, `7z`, `gz`, `bz2`, `xz`, `tar`, `tar.gz`, `tar.bz2`, `tar.xz`)
|
|
27
31
|
* **Cross-platform** (Linux, macOS, Windows)
|
|
28
32
|
|
|
29
33
|
## Installation
|
|
30
34
|
|
|
31
|
-
###
|
|
35
|
+
### **`1` Install via `pip` (Recommended)**
|
|
32
36
|
|
|
33
37
|
```sh
|
|
34
38
|
pip install pseek
|
|
35
39
|
```
|
|
36
40
|
|
|
37
|
-
###
|
|
41
|
+
### **`2` Install from source**
|
|
38
42
|
|
|
39
43
|
```sh
|
|
40
44
|
git clone https://github.com/ArianN8610/pysearch.git
|
|
@@ -77,20 +81,30 @@ pseek "error\d+" --regex
|
|
|
77
81
|
|
|
78
82
|
## Command Options
|
|
79
83
|
|
|
80
|
-
| Option | Description
|
|
81
|
-
|
|
82
|
-
| `--path` | Base directory to search in (default: current directory `.`)
|
|
83
|
-
| `--file` | Search only in file names
|
|
84
|
-
| `--directory` | Search only in directory names
|
|
85
|
-
| `--content` | Search inside file contents
|
|
86
|
-
| `--ext`, `--exclude-ext` | Filter by file extension (e.g.,
|
|
87
|
-
| `--case-sensitive` | Make the search case-sensitive (except when
|
|
88
|
-
| `--regex` | Use regular expressions to search (except when
|
|
89
|
-
| `--include`, `--exclude` | Limit search results to specific set of directories or files
|
|
90
|
-
| `--re-include`, `--re-exclude` | Limit search results to specific directories or files with regex
|
|
91
|
-
| `--word` | Match the whole word only (except when
|
|
92
|
-
| `--expr` | Enable to write conditions in the query. Example: r"foo.*bar" and ("bar" or "baz") and not "qux" (To use regex, word, and case-sensitive features, you can use the prefixes r
|
|
93
|
-
| `--timeout` | To stop the search after a specified period of time (Seconds)
|
|
94
|
-
| `--max-size`, `--min-size` | Specify maximum and minimum sizes for files and directories
|
|
95
|
-
| `--
|
|
96
|
-
| `--
|
|
84
|
+
| Option | Description |
|
|
85
|
+
|--------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|
86
|
+
| `--path` | Base directory to search in (default: current directory `.`) |
|
|
87
|
+
| `--file` | Search only in file names |
|
|
88
|
+
| `--directory` | Search only in directory names |
|
|
89
|
+
| `--content` | Search inside file contents |
|
|
90
|
+
| `--ext`, `--exclude-ext` | Filter by file extension (e.g., `txt`, `log`) |
|
|
91
|
+
| `--case-sensitive` | Make the search case-sensitive (except when `--expr` is enabled, in which case you can make it case sensitive by putting `c` before term: `c"foo"`) |
|
|
92
|
+
| `--regex` | Use regular expressions to search (except when `--expr` is enabled, in which case you can make it regex by putting `r` before term: `r"foo"`) |
|
|
93
|
+
| `--include`, `--exclude` | Limit search results to specific set of directories or files |
|
|
94
|
+
| `--re-include`, `--re-exclude` | Limit search results to specific directories or files with regex |
|
|
95
|
+
| `--word` | Match the whole word only (except when `--expr` is enabled, in which case you can make it match whole word by putting `w` before term: `w"foo"`) |
|
|
96
|
+
| `--expr` | Enable to write conditions in the query. Example: `r"foo.*bar" and ("bar" or "baz") and not "qux"` (To use regex, word, and case-sensitive features, you can use the prefixes `r`, `w`, and `c` before terms. Allowed modes: `r`, `w`, `c`, `wc`, `cw`, `rc`, `cr`. Examples: `r"foo.*bar"`, `wc"Foo"`, `cr".*Foo"`, ...) |
|
|
97
|
+
| `--timeout` | To stop the search after a specified period of time (Seconds) |
|
|
98
|
+
| `--max-size`, `--min-size` | Specify maximum and minimum sizes for files and directories |
|
|
99
|
+
| `--archive` | Enable search within archive files (e.g. `zip`, `rar`, `7z`, `gz`, `bz2`, `xz`, `tar`, `tar.gz`, `tar.bz2`, `tar.xz`) |
|
|
100
|
+
| `--arc-ext`, `--arc-ee` | Filter by file extension inside archive files |
|
|
101
|
+
| `--arc-inc`, `--arc-exc` | Limit search results to specific set of directories or files inside archive files |
|
|
102
|
+
| `--arc-max`, `--arc-min` | Specify maximum and minimum sizes for files inside archive files (It doesn't work for directories because their size is zero in archive files) |
|
|
103
|
+
| `--rarfb` | Path to RAR backend tool (e.g. UnRAR.exe, ...) |
|
|
104
|
+
| `--full-path` | Display full path of files and directories |
|
|
105
|
+
| `--no-content` | Only display files path for content search |
|
|
106
|
+
|
|
107
|
+
## Requirements
|
|
108
|
+
|
|
109
|
+
* Python 3.6 or higher
|
|
110
|
+
* `unrar`, `bsdtar`, `unar` or `7zip` for the rarfile library to support searching inside `.rar` files (optional)
|
|
@@ -10,17 +10,19 @@ A powerful command-line tool for searching files, directories, and content insid
|
|
|
10
10
|
* **Search inside file contents**
|
|
11
11
|
* **Highlight matches** in terminal output
|
|
12
12
|
* **Optimized for speed** with ThreadPoolExecutor
|
|
13
|
+
* Support **logical expression** for search queries
|
|
14
|
+
* Search inside **archive files** (e.g. `zip`, `rar`, `7z`, `gz`, `bz2`, `xz`, `tar`, `tar.gz`, `tar.bz2`, `tar.xz`)
|
|
13
15
|
* **Cross-platform** (Linux, macOS, Windows)
|
|
14
16
|
|
|
15
17
|
## Installation
|
|
16
18
|
|
|
17
|
-
###
|
|
19
|
+
### **`1` Install via `pip` (Recommended)**
|
|
18
20
|
|
|
19
21
|
```sh
|
|
20
22
|
pip install pseek
|
|
21
23
|
```
|
|
22
24
|
|
|
23
|
-
###
|
|
25
|
+
### **`2` Install from source**
|
|
24
26
|
|
|
25
27
|
```sh
|
|
26
28
|
git clone https://github.com/ArianN8610/pysearch.git
|
|
@@ -63,20 +65,30 @@ pseek "error\d+" --regex
|
|
|
63
65
|
|
|
64
66
|
## Command Options
|
|
65
67
|
|
|
66
|
-
| Option | Description
|
|
67
|
-
|
|
68
|
-
| `--path` | Base directory to search in (default: current directory `.`)
|
|
69
|
-
| `--file` | Search only in file names
|
|
70
|
-
| `--directory` | Search only in directory names
|
|
71
|
-
| `--content` | Search inside file contents
|
|
72
|
-
| `--ext`, `--exclude-ext` | Filter by file extension (e.g.,
|
|
73
|
-
| `--case-sensitive` | Make the search case-sensitive (except when
|
|
74
|
-
| `--regex` | Use regular expressions to search (except when
|
|
75
|
-
| `--include`, `--exclude` | Limit search results to specific set of directories or files
|
|
76
|
-
| `--re-include`, `--re-exclude` | Limit search results to specific directories or files with regex
|
|
77
|
-
| `--word` | Match the whole word only (except when
|
|
78
|
-
| `--expr` | Enable to write conditions in the query. Example: r"foo.*bar" and ("bar" or "baz") and not "qux" (To use regex, word, and case-sensitive features, you can use the prefixes r
|
|
79
|
-
| `--timeout` | To stop the search after a specified period of time (Seconds)
|
|
80
|
-
| `--max-size`, `--min-size` | Specify maximum and minimum sizes for files and directories
|
|
81
|
-
| `--
|
|
82
|
-
| `--
|
|
68
|
+
| Option | Description |
|
|
69
|
+
|--------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|
70
|
+
| `--path` | Base directory to search in (default: current directory `.`) |
|
|
71
|
+
| `--file` | Search only in file names |
|
|
72
|
+
| `--directory` | Search only in directory names |
|
|
73
|
+
| `--content` | Search inside file contents |
|
|
74
|
+
| `--ext`, `--exclude-ext` | Filter by file extension (e.g., `txt`, `log`) |
|
|
75
|
+
| `--case-sensitive` | Make the search case-sensitive (except when `--expr` is enabled, in which case you can make it case sensitive by putting `c` before term: `c"foo"`) |
|
|
76
|
+
| `--regex` | Use regular expressions to search (except when `--expr` is enabled, in which case you can make it regex by putting `r` before term: `r"foo"`) |
|
|
77
|
+
| `--include`, `--exclude` | Limit search results to specific set of directories or files |
|
|
78
|
+
| `--re-include`, `--re-exclude` | Limit search results to specific directories or files with regex |
|
|
79
|
+
| `--word` | Match the whole word only (except when `--expr` is enabled, in which case you can make it match whole word by putting `w` before term: `w"foo"`) |
|
|
80
|
+
| `--expr` | Enable to write conditions in the query. Example: `r"foo.*bar" and ("bar" or "baz") and not "qux"` (To use regex, word, and case-sensitive features, you can use the prefixes `r`, `w`, and `c` before terms. Allowed modes: `r`, `w`, `c`, `wc`, `cw`, `rc`, `cr`. Examples: `r"foo.*bar"`, `wc"Foo"`, `cr".*Foo"`, ...) |
|
|
81
|
+
| `--timeout` | To stop the search after a specified period of time (Seconds) |
|
|
82
|
+
| `--max-size`, `--min-size` | Specify maximum and minimum sizes for files and directories |
|
|
83
|
+
| `--archive` | Enable search within archive files (e.g. `zip`, `rar`, `7z`, `gz`, `bz2`, `xz`, `tar`, `tar.gz`, `tar.bz2`, `tar.xz`) |
|
|
84
|
+
| `--arc-ext`, `--arc-ee` | Filter by file extension inside archive files |
|
|
85
|
+
| `--arc-inc`, `--arc-exc` | Limit search results to specific set of directories or files inside archive files |
|
|
86
|
+
| `--arc-max`, `--arc-min` | Specify maximum and minimum sizes for files inside archive files (It doesn't work for directories because their size is zero in archive files) |
|
|
87
|
+
| `--rarfb` | Path to RAR backend tool (e.g. UnRAR.exe, ...) |
|
|
88
|
+
| `--full-path` | Display full path of files and directories |
|
|
89
|
+
| `--no-content` | Only display files path for content search |
|
|
90
|
+
|
|
91
|
+
## Requirements
|
|
92
|
+
|
|
93
|
+
* Python 3.6 or higher
|
|
94
|
+
* `unrar`, `bsdtar`, `unar` or `7zip` for the rarfile library to support searching inside `.rar` files (optional)
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import click
|
|
2
2
|
from .searcher import Search
|
|
3
3
|
from multiprocessing import Process
|
|
4
|
+
from .utils import check_rar_backend
|
|
4
5
|
|
|
5
6
|
|
|
6
|
-
def run_search_process(file, directory, content,
|
|
7
|
+
def run_search_process(file, directory, content, search_instance):
|
|
7
8
|
"""Performs the basic search operation"""
|
|
8
9
|
total_results = 0
|
|
9
10
|
|
|
@@ -11,7 +12,7 @@ def run_search_process(file, directory, content, ext, exclude_ext, search_instan
|
|
|
11
12
|
if file:
|
|
12
13
|
total_results += search_instance.search('file').echo('Files', 'file')
|
|
13
14
|
# Search for directories if requested and extension filters are not active.
|
|
14
|
-
if directory
|
|
15
|
+
if directory:
|
|
15
16
|
total_results += search_instance.search('directory').echo('Directories', 'directory')
|
|
16
17
|
# Search for content inside files if requested.
|
|
17
18
|
if content:
|
|
@@ -68,12 +69,32 @@ def run_search_process(file, directory, content, ext, exclude_ext, search_instan
|
|
|
68
69
|
# Size filters
|
|
69
70
|
@click.option('--max-size', type=click.FLOAT, help='Maximum file/directory size (in MB).')
|
|
70
71
|
@click.option('--min-size', type=click.FLOAT, help='Minimum file/directory size (in MB).')
|
|
72
|
+
# Archive options
|
|
73
|
+
@click.option('--archive', is_flag=True,
|
|
74
|
+
help='Enable search within archive files (e.g. zip, rar, 7z, gz, bz2, xz, tar, tar.gz, tar.bz2, tar.xz)')
|
|
75
|
+
@click.option('--arc-ext', multiple=True, type=click.STRING,
|
|
76
|
+
help='Include files with these extensions inside archive files. Example: --arc-ext py --arc-ext js')
|
|
77
|
+
@click.option('--arc-ee', multiple=True, type=click.STRING,
|
|
78
|
+
help='Exclude files with these extensions inside archive files. Example: --arc-ee jpg --arc-ee exe')
|
|
79
|
+
@click.option('--arc-inc', type=click.Path(file_okay=True, dir_okay=True),
|
|
80
|
+
multiple=True, help='Directories or files to include in search for inside archive files.')
|
|
81
|
+
@click.option('--arc-exc', type=click.Path(file_okay=True, dir_okay=True),
|
|
82
|
+
multiple=True, help='Directories or files to exclude from search for inside archive files.')
|
|
83
|
+
@click.option('--arc-max', type=click.FLOAT, help='Maximum size of files in the archive (in MB).')
|
|
84
|
+
@click.option('--arc-min', type=click.FLOAT, help='Minimum size of files in the archive (in MB).')
|
|
85
|
+
@click.option('--rarfb', type=click.Path(exists=True, file_okay=True, dir_okay=False),
|
|
86
|
+
help='Path to RAR backend tool (e.g. UnRAR.exe, ...). '
|
|
87
|
+
'Enter the file type in the query (e.g. unrar, bsdtar, unar, 7z).')
|
|
71
88
|
# Output option
|
|
72
89
|
@click.option('--full-path', is_flag=True, help='Display full paths for results.')
|
|
73
90
|
@click.option('--no-content', is_flag=True, help='Only display files path for content search.')
|
|
74
91
|
def search(query, path, file, directory, content, case_sensitive, ext, exclude_ext, regex, include, exclude,
|
|
75
|
-
re_include, re_exclude, word, expr, timeout, max_size, min_size,
|
|
92
|
+
re_include, re_exclude, word, expr, timeout, max_size, min_size, archive, arc_ext, arc_ee, arc_inc,
|
|
93
|
+
arc_exc, arc_max, arc_min, rarfb, full_path, no_content):
|
|
76
94
|
"""Search for files, directories, and file content based on the query."""
|
|
95
|
+
|
|
96
|
+
check_rar_backend(archive, rarfb, query)
|
|
97
|
+
|
|
77
98
|
# If no search type is specified, search in all types.
|
|
78
99
|
if not any((file, directory, content)):
|
|
79
100
|
file = directory = content = True
|
|
@@ -94,6 +115,13 @@ def search(query, path, file, directory, content, case_sensitive, ext, exclude_e
|
|
|
94
115
|
expr=expr,
|
|
95
116
|
max_size=max_size,
|
|
96
117
|
min_size=min_size,
|
|
118
|
+
archive=archive,
|
|
119
|
+
arc_ext=arc_ext,
|
|
120
|
+
arc_ee=arc_ee,
|
|
121
|
+
arc_inc=arc_inc,
|
|
122
|
+
arc_exc=arc_exc,
|
|
123
|
+
arc_max=arc_max,
|
|
124
|
+
arc_min=arc_min,
|
|
97
125
|
full_path=full_path,
|
|
98
126
|
no_content=no_content
|
|
99
127
|
)
|
|
@@ -112,7 +140,7 @@ def search(query, path, file, directory, content, case_sensitive, ext, exclude_e
|
|
|
112
140
|
p.join()
|
|
113
141
|
click.echo(click.style(f"\nTimeout! Search exceeded {timeout} seconds and was stopped.", fg="red"))
|
|
114
142
|
else:
|
|
115
|
-
run_search_process(file, directory, content,
|
|
143
|
+
run_search_process(file, directory, content, search_instance)
|
|
116
144
|
|
|
117
145
|
|
|
118
146
|
if __name__ == "__main__":
|
|
@@ -0,0 +1,479 @@
|
|
|
1
|
+
import mmap
|
|
2
|
+
import click
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from .utils import compile_regex, get_archive_path_size, try_decode
|
|
5
|
+
from .parser import parse_query_expression, TermNode, highlight_text_safe
|
|
6
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
7
|
+
import zipfile
|
|
8
|
+
import py7zr
|
|
9
|
+
import tarfile
|
|
10
|
+
import gzip
|
|
11
|
+
import bz2
|
|
12
|
+
import lzma
|
|
13
|
+
import rarfile
|
|
14
|
+
|
|
15
|
+
# Archive extensions that are allowed
|
|
16
|
+
ARCHIVE_EXTS = ('zip', 'rar', '7z', 'tar', 'gz', 'bz2', 'xz', 'tar.gz', 'tar.bz2', 'tar.xz')
|
|
17
|
+
|
|
18
|
+
# Extensions that are not suitable for content search (binary, media, etc.)
|
|
19
|
+
EXCLUDED_EXTENSIONS = (
|
|
20
|
+
'jpg', 'jpeg', 'png', 'gif', 'bmp', 'tiff', 'webp', 'svg',
|
|
21
|
+
'mp4', 'mov', 'avi', 'mkv', 'webm', 'flv', 'm4v', 'mpg', 'wmv',
|
|
22
|
+
'mp3', 'wav', 'ogg', 'flac', 'aac', 'wma', 'opus',
|
|
23
|
+
'exe', 'dll', 'bin', 'iso', 'img', 'dat', 'dmg', 'class', 'so', 'o', 'obj',
|
|
24
|
+
'ttf', 'otf', 'woff', 'woff2', 'eot',
|
|
25
|
+
'db', 'sqlite', 'mdf', 'bak', 'log', 'jsonl', 'dat',
|
|
26
|
+
'apk', 'ipa', 'deb', 'rpm', 'pkg', 'appimage', 'jar', 'war',
|
|
27
|
+
'pyc', 'ps1', 'pem', 'pyd', 'whl'
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class Search:
|
|
32
|
+
def __init__(self, base_path, query, case_sensitive, ext, exclude_ext, regex, include, exclude, re_include,
|
|
33
|
+
re_exclude, whole_word, expr, max_size, min_size, archive, arc_ext, arc_ee, arc_inc, arc_exc,
|
|
34
|
+
arc_max, arc_min, full_path, no_content):
|
|
35
|
+
"""Initialize search parameters"""
|
|
36
|
+
self.base_path = Path(base_path)
|
|
37
|
+
self.query = query
|
|
38
|
+
self.case_sensitive = case_sensitive
|
|
39
|
+
self.ext = set(ext)
|
|
40
|
+
self.exclude_ext = set(exclude_ext) | {''} if exclude_ext else set() # To exclude directories from search result
|
|
41
|
+
self.regex = regex
|
|
42
|
+
self.include = {Path(p).resolve() for p in include}
|
|
43
|
+
self.exclude = {Path(p).resolve() for p in exclude}
|
|
44
|
+
self.re_include = compile_regex(re_include)
|
|
45
|
+
self.re_exclude = compile_regex(re_exclude)
|
|
46
|
+
self.whole_word = whole_word
|
|
47
|
+
self.expr = expr
|
|
48
|
+
self.max_size = max_size
|
|
49
|
+
self.min_size = min_size
|
|
50
|
+
self.archive = archive
|
|
51
|
+
self.arc_ext = set(arc_ext)
|
|
52
|
+
self.arc_ee = set(arc_ee) | {''} if arc_ee else set()
|
|
53
|
+
self.arc_inc = {Path(p) for p in arc_inc}
|
|
54
|
+
self.arc_exc = {Path(p) for p in arc_exc}
|
|
55
|
+
self.arc_max = arc_max
|
|
56
|
+
self.arc_min = arc_min
|
|
57
|
+
self.full_path = full_path
|
|
58
|
+
self.no_content = no_content
|
|
59
|
+
self.result = None
|
|
60
|
+
|
|
61
|
+
def should_skip(self, p_resolved: Path, search_type: str) -> bool:
|
|
62
|
+
"""
|
|
63
|
+
Check whether the file/directory should be skipped based on various filters.
|
|
64
|
+
Returns True if the path should be skipped.
|
|
65
|
+
"""
|
|
66
|
+
try:
|
|
67
|
+
p_size_mb = p_resolved.stat().st_size / 1_048_576 # Convert size to MB
|
|
68
|
+
except OSError:
|
|
69
|
+
# If path is inaccessible, skip it.
|
|
70
|
+
return True
|
|
71
|
+
|
|
72
|
+
file_ext = ''.join(p_resolved.suffixes)[1:].lower()
|
|
73
|
+
|
|
74
|
+
# Ignore some filters for archive files when archive is enabled
|
|
75
|
+
if (not self.archive or not file_ext in ('zip', 'rar', '7z', 'tar', 'tar.gz', 'tar.bz2', 'tar.xz')) and \
|
|
76
|
+
((search_type in ('file', 'content') and not p_resolved.is_file())
|
|
77
|
+
or (search_type == 'directory' and not p_resolved.is_dir())):
|
|
78
|
+
return True
|
|
79
|
+
|
|
80
|
+
if (self.include and not any(p_resolved.is_relative_to(inc) for inc in self.include)) \
|
|
81
|
+
or (self.exclude and any(p_resolved.is_relative_to(exc) for exc in self.exclude)) \
|
|
82
|
+
or (self.ext and file_ext not in self.ext) \
|
|
83
|
+
or (self.exclude_ext and file_ext in self.exclude_ext) \
|
|
84
|
+
or (search_type == 'content' and file_ext in EXCLUDED_EXTENSIONS) \
|
|
85
|
+
or (self.max_size and p_size_mb > self.max_size) \
|
|
86
|
+
or (self.min_size and p_size_mb < self.min_size):
|
|
87
|
+
return True
|
|
88
|
+
|
|
89
|
+
# Filter by regex include and exclude
|
|
90
|
+
if self.re_include:
|
|
91
|
+
return not self.re_include.search(str(p_resolved))
|
|
92
|
+
if self.re_exclude:
|
|
93
|
+
return self.re_exclude.search(str(p_resolved)) is not None
|
|
94
|
+
|
|
95
|
+
return False
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def archive_should_skip(self, path_info: Path, search_type: str, is_file: bool, is_dir: bool, p_size: float):
|
|
99
|
+
"""Check whether the file/directory inside archive files should be skipped based on various filters"""
|
|
100
|
+
|
|
101
|
+
file_ext = ''.join(path_info.suffixes)[1:].lower()
|
|
102
|
+
|
|
103
|
+
if (search_type in ('file', 'content') and not is_file) \
|
|
104
|
+
or (search_type == 'directory' and not is_dir):
|
|
105
|
+
return True
|
|
106
|
+
|
|
107
|
+
if (self.arc_inc and not any(path_info.is_relative_to(inc) for inc in self.arc_inc)) \
|
|
108
|
+
or (self.arc_exc and any(path_info.is_relative_to(exc) for exc in self.arc_exc)) \
|
|
109
|
+
or (self.arc_ext and file_ext not in self.arc_ext) \
|
|
110
|
+
or (self.arc_ee and file_ext in self.arc_ee) \
|
|
111
|
+
or (search_type == 'content' and file_ext in EXCLUDED_EXTENSIONS) \
|
|
112
|
+
or (self.arc_max and p_size > self.arc_max) \
|
|
113
|
+
or (self.arc_min and p_size < self.arc_min):
|
|
114
|
+
return True
|
|
115
|
+
|
|
116
|
+
if self.re_include:
|
|
117
|
+
return not self.re_include.search(str(path_info))
|
|
118
|
+
if self.re_exclude:
|
|
119
|
+
return self.re_exclude.search(str(path_info)) is not None
|
|
120
|
+
|
|
121
|
+
return False
|
|
122
|
+
|
|
123
|
+
def extract_names_from_archive(self, file_path: Path, search_type: str):
|
|
124
|
+
"""Extract files and directories name from archive files to search"""
|
|
125
|
+
|
|
126
|
+
file_ext = ''.join(file_path.suffixes)[1:].lower()
|
|
127
|
+
|
|
128
|
+
try:
|
|
129
|
+
if file_ext == 'zip':
|
|
130
|
+
with zipfile.ZipFile(file_path) as zf:
|
|
131
|
+
for info in zf.infolist():
|
|
132
|
+
name = Path(info.filename)
|
|
133
|
+
if not self.archive_should_skip(
|
|
134
|
+
name,
|
|
135
|
+
search_type,
|
|
136
|
+
not info.is_dir(),
|
|
137
|
+
info.is_dir(),
|
|
138
|
+
get_archive_path_size(info, 'zip')
|
|
139
|
+
):
|
|
140
|
+
yield name
|
|
141
|
+
elif file_ext == 'rar':
|
|
142
|
+
with rarfile.RarFile(file_path) as rf:
|
|
143
|
+
for info in rf.infolist():
|
|
144
|
+
name = Path(info.filename)
|
|
145
|
+
if not self.archive_should_skip(
|
|
146
|
+
name,
|
|
147
|
+
search_type,
|
|
148
|
+
not info.is_dir(),
|
|
149
|
+
info.is_dir(),
|
|
150
|
+
get_archive_path_size(info, 'rar')
|
|
151
|
+
):
|
|
152
|
+
yield name
|
|
153
|
+
elif file_ext == '7z':
|
|
154
|
+
with py7zr.SevenZipFile(file_path, mode='r') as z:
|
|
155
|
+
for info in z.list():
|
|
156
|
+
name = Path(info.filename)
|
|
157
|
+
if not self.archive_should_skip(
|
|
158
|
+
name,
|
|
159
|
+
search_type,
|
|
160
|
+
not info.is_directory,
|
|
161
|
+
info.is_directory,
|
|
162
|
+
get_archive_path_size(info, '7z')
|
|
163
|
+
):
|
|
164
|
+
yield name
|
|
165
|
+
elif file_ext in ('tar', 'tar.gz', 'tar.bz2', 'tar.xz'):
|
|
166
|
+
# Specify the mode based on the file ext to open it
|
|
167
|
+
mode = 'r'
|
|
168
|
+
if file_ext == 'tar.gz':
|
|
169
|
+
mode = 'r:gz'
|
|
170
|
+
elif file_ext == 'tar.bz2':
|
|
171
|
+
mode = 'r:bz2'
|
|
172
|
+
elif file_ext == 'tar.xz':
|
|
173
|
+
mode = 'r:xz'
|
|
174
|
+
|
|
175
|
+
with tarfile.open(file_path, mode) as tf:
|
|
176
|
+
for member in tf.getmembers():
|
|
177
|
+
name = Path(member.name)
|
|
178
|
+
if not self.archive_should_skip(
|
|
179
|
+
name,
|
|
180
|
+
search_type,
|
|
181
|
+
member.isfile(),
|
|
182
|
+
member.isdir(),
|
|
183
|
+
get_archive_path_size(member, file_ext)
|
|
184
|
+
):
|
|
185
|
+
yield name
|
|
186
|
+
except Exception:
|
|
187
|
+
return # silently skip invalid archives
|
|
188
|
+
|
|
189
|
+
def extract_text_from_archive(self, file_path: Path):
|
|
190
|
+
"""
|
|
191
|
+
Generator yielding (filename, content_text) from archive files.
|
|
192
|
+
Supports zip, rar, tar, gz, bz2, xz, 7z.
|
|
193
|
+
"""
|
|
194
|
+
|
|
195
|
+
file_ext = ''.join(file_path.suffixes)[1:].lower()
|
|
196
|
+
|
|
197
|
+
try:
|
|
198
|
+
if file_ext == 'zip':
|
|
199
|
+
with zipfile.ZipFile(file_path) as zf:
|
|
200
|
+
for info in zf.infolist():
|
|
201
|
+
file_name = Path(info.filename)
|
|
202
|
+
if self.archive_should_skip(
|
|
203
|
+
file_name,
|
|
204
|
+
'content',
|
|
205
|
+
not info.is_dir(),
|
|
206
|
+
info.is_dir(),
|
|
207
|
+
get_archive_path_size(info, 'zip')
|
|
208
|
+
):
|
|
209
|
+
continue
|
|
210
|
+
|
|
211
|
+
with zf.open(info) as f:
|
|
212
|
+
data = f.read()
|
|
213
|
+
text = try_decode(data)
|
|
214
|
+
if text is not None:
|
|
215
|
+
yield info.filename, text
|
|
216
|
+
elif file_ext == 'rar':
|
|
217
|
+
with rarfile.RarFile(file_path) as rf:
|
|
218
|
+
for info in rf.infolist():
|
|
219
|
+
file_name = Path(info.filename)
|
|
220
|
+
if self.archive_should_skip(
|
|
221
|
+
file_name,
|
|
222
|
+
'content',
|
|
223
|
+
not info.is_dir(),
|
|
224
|
+
info.is_dir(),
|
|
225
|
+
get_archive_path_size(info, 'rar')
|
|
226
|
+
):
|
|
227
|
+
continue
|
|
228
|
+
|
|
229
|
+
with rf.open(info) as f:
|
|
230
|
+
data = f.read()
|
|
231
|
+
text = try_decode(data)
|
|
232
|
+
if text is not None:
|
|
233
|
+
yield info.filename, text
|
|
234
|
+
elif file_ext == '7z':
|
|
235
|
+
with py7zr.SevenZipFile(file_path, mode='r') as archive:
|
|
236
|
+
for info in archive.list():
|
|
237
|
+
file_name = Path(info.filename)
|
|
238
|
+
if self.archive_should_skip(
|
|
239
|
+
file_name,
|
|
240
|
+
'content',
|
|
241
|
+
not info.is_directory,
|
|
242
|
+
info.is_directory,
|
|
243
|
+
get_archive_path_size(info, '7z')
|
|
244
|
+
):
|
|
245
|
+
continue
|
|
246
|
+
|
|
247
|
+
data = archive.read([info.filename])
|
|
248
|
+
filedata = data.get(info.filename)
|
|
249
|
+
if filedata is None:
|
|
250
|
+
continue
|
|
251
|
+
text = try_decode(filedata.read())
|
|
252
|
+
if text is not None:
|
|
253
|
+
yield info.filename, text
|
|
254
|
+
elif file_ext in ('tar', 'tar.gz', 'tar.bz2', 'tar.xz'):
|
|
255
|
+
mode = 'r'
|
|
256
|
+
if file_ext == 'tar.gz':
|
|
257
|
+
mode = 'r:gz'
|
|
258
|
+
elif file_ext == 'tar.bz2':
|
|
259
|
+
mode = 'r:bz2'
|
|
260
|
+
elif file_ext == 'tar.xz':
|
|
261
|
+
mode = 'r:xz'
|
|
262
|
+
with tarfile.open(file_path, mode) as tf:
|
|
263
|
+
for member in tf.getmembers():
|
|
264
|
+
file_name = Path(member.name)
|
|
265
|
+
if self.archive_should_skip(
|
|
266
|
+
file_name,
|
|
267
|
+
'content',
|
|
268
|
+
member.isfile(),
|
|
269
|
+
member.isdir(),
|
|
270
|
+
get_archive_path_size(member, file_ext)
|
|
271
|
+
):
|
|
272
|
+
continue
|
|
273
|
+
|
|
274
|
+
f = tf.extractfile(member)
|
|
275
|
+
if f is None:
|
|
276
|
+
continue
|
|
277
|
+
data = f.read()
|
|
278
|
+
text = try_decode(data)
|
|
279
|
+
if text is not None:
|
|
280
|
+
yield member.name, text
|
|
281
|
+
elif file_ext == 'gz':
|
|
282
|
+
with gzip.open(file_path, 'rb') as f:
|
|
283
|
+
data = f.read()
|
|
284
|
+
text = try_decode(data)
|
|
285
|
+
if text is not None:
|
|
286
|
+
yield file_path.name, text
|
|
287
|
+
elif file_ext == 'bz2':
|
|
288
|
+
with bz2.open(file_path, 'rb') as f:
|
|
289
|
+
data = f.read()
|
|
290
|
+
text = try_decode(data)
|
|
291
|
+
if text is not None:
|
|
292
|
+
yield file_path.name, text
|
|
293
|
+
elif file_ext == 'xz':
|
|
294
|
+
with lzma.open(file_path, 'rb') as f:
|
|
295
|
+
data = f.read()
|
|
296
|
+
text = try_decode(data)
|
|
297
|
+
if text is not None:
|
|
298
|
+
yield file_path.name, text
|
|
299
|
+
except Exception:
|
|
300
|
+
return
|
|
301
|
+
|
|
302
|
+
def search(self, search_type: str):
|
|
303
|
+
"""Main search function. search_type can be 'file', 'directory' or 'content'"""
|
|
304
|
+
pattern = parse_query_expression(self.query, self.expr, self.regex, self.whole_word, self.case_sensitive)
|
|
305
|
+
|
|
306
|
+
if search_type in ('file', 'directory'):
|
|
307
|
+
matches = []
|
|
308
|
+
for p in self.base_path.rglob('*'):
|
|
309
|
+
try:
|
|
310
|
+
p_resolved = p.resolve()
|
|
311
|
+
except Exception:
|
|
312
|
+
continue
|
|
313
|
+
# Skip if conditions fail
|
|
314
|
+
if self.should_skip(p_resolved, search_type):
|
|
315
|
+
continue
|
|
316
|
+
|
|
317
|
+
# Choose parent path based on full_path flag
|
|
318
|
+
p_parent = p_resolved.parent if self.full_path else p.parent
|
|
319
|
+
p_ext = ''.join(p_resolved.suffixes)[1:].lower()
|
|
320
|
+
|
|
321
|
+
if pattern.evaluate(p.name) and not (search_type == 'directory' and p_resolved.is_file()):
|
|
322
|
+
# Highlight matched query in the name
|
|
323
|
+
highlighted_name = highlight_text_safe(pattern, p.name)
|
|
324
|
+
matches.append(f'{p_parent}\\{highlighted_name}')
|
|
325
|
+
|
|
326
|
+
# Search for files and directories name inside archive files if archive is active
|
|
327
|
+
if self.archive and p_ext in ('zip', 'rar', '7z', 'tar', 'tar.gz', 'tar.bz2', 'tar.xz'):
|
|
328
|
+
for name in self.extract_names_from_archive(p_resolved, search_type):
|
|
329
|
+
if pattern.evaluate(name.name):
|
|
330
|
+
highlighted_name = highlight_text_safe(pattern, name.name)
|
|
331
|
+
matches.append(f'{p_parent}\\{p.name}::{name.parent}\\{highlighted_name}')
|
|
332
|
+
else: # content search
|
|
333
|
+
# Use dictionary: key: file path (colored), value: list of line matches
|
|
334
|
+
matches = {} if not self.no_content else set()
|
|
335
|
+
|
|
336
|
+
# If expression is simple and is a single TermNode, we can use binary pattern
|
|
337
|
+
binary_pattern = None
|
|
338
|
+
if isinstance(pattern, TermNode):
|
|
339
|
+
try:
|
|
340
|
+
binary_pattern = pattern.get_binary_pattern()
|
|
341
|
+
except Exception:
|
|
342
|
+
binary_pattern = None
|
|
343
|
+
|
|
344
|
+
def process_file(file_path: Path):
|
|
345
|
+
"""Process a single file for content search"""
|
|
346
|
+
try:
|
|
347
|
+
# Avoid empty files for mmap
|
|
348
|
+
if file_path.stat().st_size == 0:
|
|
349
|
+
return
|
|
350
|
+
|
|
351
|
+
# Choose the file path format based on the full_path setting
|
|
352
|
+
file_label = str(file_path.resolve()) if self.full_path else str(file_path)
|
|
353
|
+
|
|
354
|
+
# First, check if the file is an archive, extract it from the archive and perform a search
|
|
355
|
+
if self.archive and any(str(file_path).endswith(ext) for ext in ARCHIVE_EXTS):
|
|
356
|
+
for fname, content in self.extract_text_from_archive(file_path):
|
|
357
|
+
if not pattern.evaluate(content) and not self.expr:
|
|
358
|
+
continue
|
|
359
|
+
|
|
360
|
+
# Change file_label for archive files
|
|
361
|
+
file_label += '::' + fname.replace('/', '\\')
|
|
362
|
+
|
|
363
|
+
if self.no_content and not self.expr:
|
|
364
|
+
matches.add(click.style(file_label, fg='cyan'))
|
|
365
|
+
continue
|
|
366
|
+
|
|
367
|
+
lines = []
|
|
368
|
+
for num, line in enumerate(content.splitlines(), 1):
|
|
369
|
+
if not pattern.evaluate(line):
|
|
370
|
+
continue
|
|
371
|
+
|
|
372
|
+
if self.no_content and self.expr:
|
|
373
|
+
matches.add(click.style(file_label, fg='cyan'))
|
|
374
|
+
continue
|
|
375
|
+
|
|
376
|
+
count = pattern.count_matches(line) if isinstance(pattern, TermNode) else 0
|
|
377
|
+
# Highlight the matching parts in green
|
|
378
|
+
highlighted = highlight_text_safe(pattern, line)
|
|
379
|
+
# Show a note if the pattern repeats 3 or more times
|
|
380
|
+
count_query = f' - Repeated {count} times' if count >= 3 else ''
|
|
381
|
+
# Format the output line with line number and highlighted matches
|
|
382
|
+
lines.append(
|
|
383
|
+
click.style(f'Line {num}{count_query}: ', fg='magenta') + highlighted
|
|
384
|
+
)
|
|
385
|
+
|
|
386
|
+
if lines:
|
|
387
|
+
matches[click.style(file_label, fg='cyan')] = lines
|
|
388
|
+
|
|
389
|
+
# Open the file in binary read mode
|
|
390
|
+
with open(file_path, 'rb') as f:
|
|
391
|
+
# Memory-map the file for efficient access
|
|
392
|
+
with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
|
|
393
|
+
if binary_pattern is not None:
|
|
394
|
+
if not binary_pattern.search(mm):
|
|
395
|
+
return
|
|
396
|
+
else:
|
|
397
|
+
# fallback: decode whole file for complex expressions
|
|
398
|
+
try:
|
|
399
|
+
content = mm.read().decode('utf-8')
|
|
400
|
+
except UnicodeDecodeError:
|
|
401
|
+
return
|
|
402
|
+
|
|
403
|
+
if not pattern.evaluate(content) and not self.expr:
|
|
404
|
+
return
|
|
405
|
+
|
|
406
|
+
# Avoid searching through the entire file content if the fast-content flag is True
|
|
407
|
+
if self.no_content and not self.expr:
|
|
408
|
+
matches.add(click.style(file_label, fg='cyan'))
|
|
409
|
+
return
|
|
410
|
+
|
|
411
|
+
lines = []
|
|
412
|
+
mm.seek(0) # Move the cursor to the beginning of the file
|
|
413
|
+
|
|
414
|
+
# Iterate over each line in the file
|
|
415
|
+
for num, line in enumerate(iter(mm.readline, b''), 1):
|
|
416
|
+
try:
|
|
417
|
+
# Decode the binary line as UTF-8 and strip whitespace
|
|
418
|
+
line_decoded = line.decode('utf-8').strip()
|
|
419
|
+
except UnicodeDecodeError:
|
|
420
|
+
# Skip lines that can't be decoded
|
|
421
|
+
continue
|
|
422
|
+
|
|
423
|
+
# If the pattern matches in the decoded line
|
|
424
|
+
if pattern.evaluate(line_decoded):
|
|
425
|
+
if self.no_content and self.expr:
|
|
426
|
+
matches.add(click.style(file_label, fg='cyan'))
|
|
427
|
+
return
|
|
428
|
+
count = pattern.count_matches(line_decoded) if isinstance(pattern, TermNode) else 0
|
|
429
|
+
# Highlight the matching parts in green
|
|
430
|
+
highlighted = highlight_text_safe(pattern, line_decoded)
|
|
431
|
+
# Show a note if the pattern repeats 3 or more times
|
|
432
|
+
count_query = f' - Repeated {count} times' if count >= 3 else ''
|
|
433
|
+
# Format the output line with line number and highlighted matches
|
|
434
|
+
lines.append(
|
|
435
|
+
click.style(f'Line {num}{count_query}: ', fg='magenta') + highlighted
|
|
436
|
+
)
|
|
437
|
+
|
|
438
|
+
# If any matching lines were found
|
|
439
|
+
if lines:
|
|
440
|
+
# Add the file and its matching lines to the results
|
|
441
|
+
matches[click.style(file_label, fg='cyan')] = lines
|
|
442
|
+
except Exception:
|
|
443
|
+
return
|
|
444
|
+
|
|
445
|
+
# Filter files before processing
|
|
446
|
+
files_to_process = {
|
|
447
|
+
p for p in self.base_path.rglob('*') if not self.should_skip(p.resolve(), 'content')
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
with ThreadPoolExecutor(max_workers=8) as executor:
|
|
451
|
+
executor.map(process_file, files_to_process)
|
|
452
|
+
|
|
453
|
+
self.result = matches
|
|
454
|
+
return self
|
|
455
|
+
|
|
456
|
+
def echo(self, title: str, result_name: str) -> int:
|
|
457
|
+
"""
|
|
458
|
+
Display the search results with a title.
|
|
459
|
+
Returns the count of results.
|
|
460
|
+
"""
|
|
461
|
+
count_result = 0
|
|
462
|
+
|
|
463
|
+
if self.result:
|
|
464
|
+
click.echo(click.style(f'\n{title}:\n', fg='yellow'))
|
|
465
|
+
if isinstance(self.result, dict):
|
|
466
|
+
# For content search results
|
|
467
|
+
for key, value in self.result.items():
|
|
468
|
+
click.echo(key)
|
|
469
|
+
click.echo('\n'.join(value) + '\n')
|
|
470
|
+
count_result += len(value)
|
|
471
|
+
else:
|
|
472
|
+
# For file/directory search results
|
|
473
|
+
count_result = len(self.result)
|
|
474
|
+
click.echo('\n'.join(self.result))
|
|
475
|
+
|
|
476
|
+
if count_result >= 3:
|
|
477
|
+
click.echo(click.style(f'\n{count_result} results found for {result_name}', fg='blue'))
|
|
478
|
+
|
|
479
|
+
return count_result
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import re
|
|
2
|
+
import sys
|
|
3
|
+
import click
|
|
4
|
+
import shutil
|
|
5
|
+
import rarfile
|
|
6
|
+
import platform
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def compile_regex(txt, flags=0):
|
|
11
|
+
if txt is not None:
|
|
12
|
+
try:
|
|
13
|
+
return re.compile(txt, flags)
|
|
14
|
+
except re.error as e:
|
|
15
|
+
click.echo(click.style(f"Regex compile error: {e}", fg='red'))
|
|
16
|
+
sys.exit(1)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def get_archive_path_size(info, file_type: str) -> float:
|
|
20
|
+
"""Get and return the size of the files inside the archive files in MB"""
|
|
21
|
+
if file_type in ('zip', 'rar'):
|
|
22
|
+
return info.file_size / 1_048_576
|
|
23
|
+
elif file_type == '7z':
|
|
24
|
+
return info.uncompressed / 1_048_576
|
|
25
|
+
elif file_type in ('tar', 'tar.gz', 'tar.bz2', 'tar.xz'):
|
|
26
|
+
return info.size / 1_048_576
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def try_decode(data: bytes):
|
|
30
|
+
"""Try decoding byte data to UTF-8 text. Return None if decoding fails."""
|
|
31
|
+
try:
|
|
32
|
+
return data.decode('utf-8')
|
|
33
|
+
except UnicodeDecodeError:
|
|
34
|
+
return None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def check_rar_backend(archive_enabled: bool, tool_path: str, backend: str):
|
|
38
|
+
"""Check for the existence of rar backend or save and set it for rarfile"""
|
|
39
|
+
|
|
40
|
+
backend_path = Path(__file__).parent / "RARBackend"
|
|
41
|
+
# Save backend path for later executions
|
|
42
|
+
if tool_path:
|
|
43
|
+
if backend in ('unrar', 'bsdtar', 'unar', '7z'):
|
|
44
|
+
with open(backend_path, 'w') as f:
|
|
45
|
+
f.write(f'{backend}:{tool_path}')
|
|
46
|
+
click.secho(f"RAR backend set to: {backend}", fg="green")
|
|
47
|
+
else:
|
|
48
|
+
click.secho("Unknown RAR backend tool. Please provide unrar, bsdtar, unar, or 7z.", fg="red")
|
|
49
|
+
sys.exit(1)
|
|
50
|
+
|
|
51
|
+
if archive_enabled:
|
|
52
|
+
# Try to detect presence of RAR backends in PATH
|
|
53
|
+
unrar_path = shutil.which('unrar')
|
|
54
|
+
bsdtar_path = shutil.which('bsdtar')
|
|
55
|
+
sevenzip_path = shutil.which('7z') or shutil.which('7za') # Some versions are in 7za format
|
|
56
|
+
unar_path = shutil.which('unar')
|
|
57
|
+
|
|
58
|
+
if not any((unrar_path, bsdtar_path, sevenzip_path, unar_path)) and not backend_path.exists():
|
|
59
|
+
system = platform.system()
|
|
60
|
+
if system == 'Linux':
|
|
61
|
+
install_tip = "sudo apt install unrar"
|
|
62
|
+
elif system == 'Darwin':
|
|
63
|
+
install_tip = "brew install unrar"
|
|
64
|
+
else:
|
|
65
|
+
install_tip = "Download from https://www.rarlab.com/download.htm"
|
|
66
|
+
|
|
67
|
+
click.secho(
|
|
68
|
+
"Warning: unrar, bsdtar, 7zip or unar is not installed on system or "
|
|
69
|
+
"it is not in the system PATH.\nRAR archive support is disabled.\n"
|
|
70
|
+
"To enable RAR support, please install one of them. For example:\n"
|
|
71
|
+
f" - {install_tip}\n"
|
|
72
|
+
"If it is installed or in the system PATH and you still have problems, use this option: '--rarfb'\n",
|
|
73
|
+
fg='yellow'
|
|
74
|
+
)
|
|
75
|
+
elif backend_path.exists():
|
|
76
|
+
with open(backend_path, 'r') as f:
|
|
77
|
+
b, tool = f.read().split(':', 1)
|
|
78
|
+
|
|
79
|
+
# Set up the backend for rarfile
|
|
80
|
+
if b == 'unrar':
|
|
81
|
+
rarfile.UNRAR_TOOL = tool
|
|
82
|
+
elif b == 'bsdtar':
|
|
83
|
+
rarfile.BSDTAR_TOOL = tool
|
|
84
|
+
elif b == 'unar':
|
|
85
|
+
rarfile.UNAR_TOOL = tool
|
|
86
|
+
elif b == '7z':
|
|
87
|
+
rarfile.SEVENZIP_TOOL = tool
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pseek
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.4.0
|
|
4
4
|
Summary: Pseek is a Python library to search files, folders, and text
|
|
5
5
|
Author-email: Arian <ariannasiri86@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -10,6 +10,8 @@ Description-Content-Type: text/markdown
|
|
|
10
10
|
License-File: LICENSE
|
|
11
11
|
Requires-Dist: click==8.1.8
|
|
12
12
|
Requires-Dist: lark==1.2.2
|
|
13
|
+
Requires-Dist: py7zr==1.0.0
|
|
14
|
+
Requires-Dist: rarfile==4.2
|
|
13
15
|
Dynamic: license-file
|
|
14
16
|
|
|
15
17
|
# Pseek
|
|
@@ -24,17 +26,19 @@ A powerful command-line tool for searching files, directories, and content insid
|
|
|
24
26
|
* **Search inside file contents**
|
|
25
27
|
* **Highlight matches** in terminal output
|
|
26
28
|
* **Optimized for speed** with ThreadPoolExecutor
|
|
29
|
+
* Support **logical expression** for search queries
|
|
30
|
+
* Search inside **archive files** (e.g. `zip`, `rar`, `7z`, `gz`, `bz2`, `xz`, `tar`, `tar.gz`, `tar.bz2`, `tar.xz`)
|
|
27
31
|
* **Cross-platform** (Linux, macOS, Windows)
|
|
28
32
|
|
|
29
33
|
## Installation
|
|
30
34
|
|
|
31
|
-
###
|
|
35
|
+
### **`1` Install via `pip` (Recommended)**
|
|
32
36
|
|
|
33
37
|
```sh
|
|
34
38
|
pip install pseek
|
|
35
39
|
```
|
|
36
40
|
|
|
37
|
-
###
|
|
41
|
+
### **`2` Install from source**
|
|
38
42
|
|
|
39
43
|
```sh
|
|
40
44
|
git clone https://github.com/ArianN8610/pysearch.git
|
|
@@ -77,20 +81,30 @@ pseek "error\d+" --regex
|
|
|
77
81
|
|
|
78
82
|
## Command Options
|
|
79
83
|
|
|
80
|
-
| Option | Description
|
|
81
|
-
|
|
82
|
-
| `--path` | Base directory to search in (default: current directory `.`)
|
|
83
|
-
| `--file` | Search only in file names
|
|
84
|
-
| `--directory` | Search only in directory names
|
|
85
|
-
| `--content` | Search inside file contents
|
|
86
|
-
| `--ext`, `--exclude-ext` | Filter by file extension (e.g.,
|
|
87
|
-
| `--case-sensitive` | Make the search case-sensitive (except when
|
|
88
|
-
| `--regex` | Use regular expressions to search (except when
|
|
89
|
-
| `--include`, `--exclude` | Limit search results to specific set of directories or files
|
|
90
|
-
| `--re-include`, `--re-exclude` | Limit search results to specific directories or files with regex
|
|
91
|
-
| `--word` | Match the whole word only (except when
|
|
92
|
-
| `--expr` | Enable to write conditions in the query. Example: r"foo.*bar" and ("bar" or "baz") and not "qux" (To use regex, word, and case-sensitive features, you can use the prefixes r
|
|
93
|
-
| `--timeout` | To stop the search after a specified period of time (Seconds)
|
|
94
|
-
| `--max-size`, `--min-size` | Specify maximum and minimum sizes for files and directories
|
|
95
|
-
| `--
|
|
96
|
-
| `--
|
|
84
|
+
| Option | Description |
|
|
85
|
+
|--------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|
86
|
+
| `--path` | Base directory to search in (default: current directory `.`) |
|
|
87
|
+
| `--file` | Search only in file names |
|
|
88
|
+
| `--directory` | Search only in directory names |
|
|
89
|
+
| `--content` | Search inside file contents |
|
|
90
|
+
| `--ext`, `--exclude-ext` | Filter by file extension (e.g., `txt`, `log`) |
|
|
91
|
+
| `--case-sensitive` | Make the search case-sensitive (except when `--expr` is enabled, in which case you can make it case sensitive by putting `c` before term: `c"foo"`) |
|
|
92
|
+
| `--regex` | Use regular expressions to search (except when `--expr` is enabled, in which case you can make it regex by putting `r` before term: `r"foo"`) |
|
|
93
|
+
| `--include`, `--exclude` | Limit search results to specific set of directories or files |
|
|
94
|
+
| `--re-include`, `--re-exclude` | Limit search results to specific directories or files with regex |
|
|
95
|
+
| `--word` | Match the whole word only (except when `--expr` is enabled, in which case you can make it match whole word by putting `w` before term: `w"foo"`) |
|
|
96
|
+
| `--expr` | Enable to write conditions in the query. Example: `r"foo.*bar" and ("bar" or "baz") and not "qux"` (To use regex, word, and case-sensitive features, you can use the prefixes `r`, `w`, and `c` before terms. Allowed modes: `r`, `w`, `c`, `wc`, `cw`, `rc`, `cr`. Examples: `r"foo.*bar"`, `wc"Foo"`, `cr".*Foo"`, ...) |
|
|
97
|
+
| `--timeout` | To stop the search after a specified period of time (Seconds) |
|
|
98
|
+
| `--max-size`, `--min-size` | Specify maximum and minimum sizes for files and directories |
|
|
99
|
+
| `--archive` | Enable search within archive files (e.g. `zip`, `rar`, `7z`, `gz`, `bz2`, `xz`, `tar`, `tar.gz`, `tar.bz2`, `tar.xz`) |
|
|
100
|
+
| `--arc-ext`, `--arc-ee` | Filter by file extension inside archive files |
|
|
101
|
+
| `--arc-inc`, `--arc-exc` | Limit search results to specific set of directories or files inside archive files |
|
|
102
|
+
| `--arc-max`, `--arc-min` | Specify maximum and minimum sizes for files inside archive files (It doesn't work for directories because their size is zero in archive files) |
|
|
103
|
+
| `--rarfb` | Path to RAR backend tool (e.g. UnRAR.exe, ...) |
|
|
104
|
+
| `--full-path` | Display full path of files and directories |
|
|
105
|
+
| `--no-content` | Only display files path for content search |
|
|
106
|
+
|
|
107
|
+
## Requirements
|
|
108
|
+
|
|
109
|
+
* Python 3.6 or higher
|
|
110
|
+
* `unrar`, `bsdtar`, `unar` or `7zip` for the rarfile library to support searching inside `.rar` files (optional)
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "pseek"
|
|
7
|
-
version = "2.
|
|
7
|
+
version = "2.4.0"
|
|
8
8
|
description = "Pseek is a Python library to search files, folders, and text"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -15,7 +15,9 @@ authors = [
|
|
|
15
15
|
requires-python = ">=3.6"
|
|
16
16
|
dependencies = [
|
|
17
17
|
"click==8.1.8",
|
|
18
|
-
"lark==1.2.2"
|
|
18
|
+
"lark==1.2.2",
|
|
19
|
+
"py7zr==1.0.0",
|
|
20
|
+
"rarfile==4.2"
|
|
19
21
|
]
|
|
20
22
|
urls = {Homepage = "https://github.com/ArianN8610/pysearch"}
|
|
21
23
|
|
pseek-2.3.2/pseek/searcher.py
DELETED
|
@@ -1,208 +0,0 @@
|
|
|
1
|
-
import mmap
|
|
2
|
-
import click
|
|
3
|
-
from pathlib import Path
|
|
4
|
-
from .utils import compile_regex
|
|
5
|
-
from .parser import parse_query_expression, TermNode, highlight_text_safe
|
|
6
|
-
from concurrent.futures import ThreadPoolExecutor
|
|
7
|
-
|
|
8
|
-
# Extensions that are not suitable for content search (binary, media, etc.)
|
|
9
|
-
EXCLUDED_EXTENSIONS = {
|
|
10
|
-
'jpg', 'jpeg', 'png', 'gif', 'bmp', 'tiff', 'webp', 'svg',
|
|
11
|
-
'mp4', 'mov', 'avi', 'mkv', 'webm', 'flv', 'm4v', 'mpg', 'wmv',
|
|
12
|
-
'mp3', 'wav', 'ogg', 'flac', 'aac', 'wma', 'opus',
|
|
13
|
-
'exe', 'dll', 'bin', 'iso', 'img', 'dat', 'dmg', 'class', 'so', 'o', 'obj',
|
|
14
|
-
'zip', 'rar', '7z', 'tar', 'gz', 'bz2', 'xz',
|
|
15
|
-
'ttf', 'otf', 'woff', 'woff2', 'eot',
|
|
16
|
-
'db', 'sqlite', 'mdf', 'bak', 'log', 'jsonl', 'dat',
|
|
17
|
-
'apk', 'ipa', 'deb', 'rpm', 'pkg', 'appimage', 'jar', 'war',
|
|
18
|
-
'pyc', 'ps1', 'pem', 'pyd', 'whl'
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
class Search:
|
|
23
|
-
def __init__(self, base_path, query, case_sensitive, ext, exclude_ext, regex, include, exclude, re_include,
|
|
24
|
-
re_exclude, whole_word, expr, max_size, min_size, full_path, no_content):
|
|
25
|
-
"""Initialize search parameters"""
|
|
26
|
-
self.base_path = Path(base_path)
|
|
27
|
-
self.query = query
|
|
28
|
-
self.case_sensitive = case_sensitive
|
|
29
|
-
self.ext = set(ext)
|
|
30
|
-
self.exclude_ext = set(exclude_ext)
|
|
31
|
-
self.regex = regex
|
|
32
|
-
self.include = {Path(p).resolve() for p in include}
|
|
33
|
-
self.exclude = {Path(p).resolve() for p in exclude}
|
|
34
|
-
self.re_include = re_include
|
|
35
|
-
self.re_exclude = re_exclude
|
|
36
|
-
self.whole_word = whole_word
|
|
37
|
-
self.expr = expr
|
|
38
|
-
self.max_size = max_size
|
|
39
|
-
self.min_size = min_size
|
|
40
|
-
self.full_path = full_path
|
|
41
|
-
self.no_content = no_content
|
|
42
|
-
self.result = None
|
|
43
|
-
|
|
44
|
-
def should_skip(self, p_resolved: Path, search_type: str) -> bool:
|
|
45
|
-
"""
|
|
46
|
-
Check whether the file/directory should be skipped based on various filters.
|
|
47
|
-
Returns True if the path should be skipped.
|
|
48
|
-
"""
|
|
49
|
-
try:
|
|
50
|
-
p_size_mb = p_resolved.stat().st_size / 1_048_576 # Convert size to MB
|
|
51
|
-
except OSError:
|
|
52
|
-
# If path is inaccessible, skip it.
|
|
53
|
-
return True
|
|
54
|
-
|
|
55
|
-
file_ext = p_resolved.suffix[1:].lower()
|
|
56
|
-
|
|
57
|
-
if (self.include and not any(p_resolved.is_relative_to(inc) for inc in self.include)) \
|
|
58
|
-
or (self.exclude and any(p_resolved.is_relative_to(exc) for exc in self.exclude)) \
|
|
59
|
-
or (self.ext and file_ext not in self.ext) \
|
|
60
|
-
or (self.exclude_ext and file_ext in self.exclude_ext) \
|
|
61
|
-
or (search_type == 'content' and file_ext in EXCLUDED_EXTENSIONS) \
|
|
62
|
-
or (self.max_size and p_size_mb > self.max_size) \
|
|
63
|
-
or (self.min_size and p_size_mb < self.min_size) \
|
|
64
|
-
or ((search_type in ('file', 'content')) and not p_resolved.is_file()) \
|
|
65
|
-
or (search_type == 'directory' and not p_resolved.is_dir()):
|
|
66
|
-
return True
|
|
67
|
-
|
|
68
|
-
# Filter by regex include and exclude
|
|
69
|
-
if compiled_inc:=compile_regex(self.re_include):
|
|
70
|
-
return not compiled_inc.search(str(p_resolved))
|
|
71
|
-
if compiled_exc:=compile_regex(self.re_exclude):
|
|
72
|
-
return compiled_exc.search(str(p_resolved)) is not None
|
|
73
|
-
|
|
74
|
-
return False
|
|
75
|
-
|
|
76
|
-
def search(self, search_type: str):
|
|
77
|
-
"""Main search function. search_type can be 'file', 'directory' or 'content'"""
|
|
78
|
-
pattern = parse_query_expression(self.query, self.expr, self.regex, self.whole_word, self.case_sensitive)
|
|
79
|
-
|
|
80
|
-
if search_type in ('file', 'directory'):
|
|
81
|
-
matches = []
|
|
82
|
-
for p in self.base_path.rglob('*'):
|
|
83
|
-
try:
|
|
84
|
-
p_resolved = p.resolve()
|
|
85
|
-
except Exception:
|
|
86
|
-
continue
|
|
87
|
-
# Skip if conditions fail or if name doesn't match the query
|
|
88
|
-
if self.should_skip(p_resolved, search_type) or not pattern.evaluate(p.name):
|
|
89
|
-
continue
|
|
90
|
-
|
|
91
|
-
# Highlight matched query in the name
|
|
92
|
-
highlighted_name = highlight_text_safe(pattern, p.name)
|
|
93
|
-
# Choose parent path based on full_path flag
|
|
94
|
-
p_parent = p_resolved.parent if self.full_path else p.parent
|
|
95
|
-
matches.append(f'{p_parent}\\{highlighted_name}')
|
|
96
|
-
else: # content search
|
|
97
|
-
# Use dictionary: key: file path (colored), value: list of line matches
|
|
98
|
-
matches = {} if not self.no_content else set()
|
|
99
|
-
|
|
100
|
-
# If expression is simple and is a single TermNode, we can use binary pattern
|
|
101
|
-
binary_pattern = None
|
|
102
|
-
if isinstance(pattern, TermNode):
|
|
103
|
-
try:
|
|
104
|
-
binary_pattern = pattern.get_binary_pattern()
|
|
105
|
-
except Exception:
|
|
106
|
-
binary_pattern = None
|
|
107
|
-
|
|
108
|
-
def process_file(file_path: Path):
|
|
109
|
-
"""Process a single file for content search"""
|
|
110
|
-
try:
|
|
111
|
-
# Avoid empty files for mmap
|
|
112
|
-
if file_path.stat().st_size == 0:
|
|
113
|
-
return
|
|
114
|
-
|
|
115
|
-
# Open the file in binary read mode
|
|
116
|
-
with open(file_path, 'rb') as f:
|
|
117
|
-
# Memory-map the file for efficient access
|
|
118
|
-
with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
|
|
119
|
-
if binary_pattern is not None:
|
|
120
|
-
if not binary_pattern.search(mm):
|
|
121
|
-
return
|
|
122
|
-
else:
|
|
123
|
-
# fallback: decode whole file for complex expressions
|
|
124
|
-
try:
|
|
125
|
-
content = mm.read().decode('utf-8')
|
|
126
|
-
except UnicodeDecodeError:
|
|
127
|
-
return
|
|
128
|
-
|
|
129
|
-
if not pattern.evaluate(content) and not self.expr:
|
|
130
|
-
return
|
|
131
|
-
|
|
132
|
-
# Choose the file path format based on the full_path setting
|
|
133
|
-
file_label = str(file_path.resolve()) if self.full_path else str(file_path)
|
|
134
|
-
|
|
135
|
-
# Avoid searching through the entire file content if the fast-content flag is True
|
|
136
|
-
if self.no_content and not self.expr:
|
|
137
|
-
matches.add(click.style(file_label, fg='cyan'))
|
|
138
|
-
return
|
|
139
|
-
|
|
140
|
-
lines = []
|
|
141
|
-
mm.seek(0) # Move the cursor to the beginning of the file
|
|
142
|
-
|
|
143
|
-
# Iterate over each line in the file
|
|
144
|
-
for num, line in enumerate(iter(mm.readline, b''), 1):
|
|
145
|
-
try:
|
|
146
|
-
# Decode the binary line as UTF-8 and strip whitespace
|
|
147
|
-
line_decoded = line.decode('utf-8').strip()
|
|
148
|
-
except UnicodeDecodeError:
|
|
149
|
-
# Skip lines that can't be decoded
|
|
150
|
-
continue
|
|
151
|
-
|
|
152
|
-
# If the pattern matches in the decoded line
|
|
153
|
-
if pattern.evaluate(line_decoded):
|
|
154
|
-
if self.no_content and self.expr:
|
|
155
|
-
matches.add(click.style(file_label, fg='cyan'))
|
|
156
|
-
return
|
|
157
|
-
count = pattern.count_matches(line_decoded) if isinstance(pattern, TermNode) else 0
|
|
158
|
-
# Highlight the matching parts in green
|
|
159
|
-
highlighted = highlight_text_safe(pattern, line_decoded)
|
|
160
|
-
# Show a note if the pattern repeats 3 or more times
|
|
161
|
-
count_query = f' - Repeated {count} times' if count >= 3 else ''
|
|
162
|
-
# Format the output line with line number and highlighted matches
|
|
163
|
-
lines.append(
|
|
164
|
-
click.style(f'Line {num}{count_query}: ', fg='magenta') + highlighted
|
|
165
|
-
)
|
|
166
|
-
|
|
167
|
-
# If any matching lines were found
|
|
168
|
-
if lines:
|
|
169
|
-
# Add the file and its matching lines to the results
|
|
170
|
-
matches[click.style(file_label, fg='cyan')] = lines
|
|
171
|
-
except Exception:
|
|
172
|
-
return
|
|
173
|
-
|
|
174
|
-
# Filter files before processing
|
|
175
|
-
files_to_process = {
|
|
176
|
-
p for p in self.base_path.rglob('*') if not self.should_skip(p.resolve(), 'content')
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
with ThreadPoolExecutor(max_workers=8) as executor:
|
|
180
|
-
executor.map(process_file, files_to_process)
|
|
181
|
-
|
|
182
|
-
self.result = matches
|
|
183
|
-
return self
|
|
184
|
-
|
|
185
|
-
def echo(self, title: str, result_name: str) -> int:
|
|
186
|
-
"""
|
|
187
|
-
Display the search results with a title.
|
|
188
|
-
Returns the count of results.
|
|
189
|
-
"""
|
|
190
|
-
count_result = 0
|
|
191
|
-
|
|
192
|
-
if self.result:
|
|
193
|
-
click.echo(click.style(f'\n{title}:\n', fg='yellow'))
|
|
194
|
-
if isinstance(self.result, dict):
|
|
195
|
-
# For content search results
|
|
196
|
-
for key, value in self.result.items():
|
|
197
|
-
click.echo(key)
|
|
198
|
-
click.echo('\n'.join(value) + '\n')
|
|
199
|
-
count_result += len(value)
|
|
200
|
-
else:
|
|
201
|
-
# For file/directory search results
|
|
202
|
-
count_result = len(self.result)
|
|
203
|
-
click.echo('\n'.join(self.result))
|
|
204
|
-
|
|
205
|
-
if count_result >= 3:
|
|
206
|
-
click.echo(click.style(f'\n{count_result} results found for {result_name}', fg='blue'))
|
|
207
|
-
|
|
208
|
-
return count_result
|
pseek-2.3.2/pseek/utils.py
DELETED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|