smith-utils 0.3.1__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {smith_utils-0.3.1 → smith_utils-0.4.0}/PKG-INFO +11 -2
- {smith_utils-0.3.1 → smith_utils-0.4.0}/README.md +10 -1
- {smith_utils-0.3.1 → smith_utils-0.4.0}/pyproject.toml +1 -1
- {smith_utils-0.3.1 → smith_utils-0.4.0}/src/smith_utils/__init__.py +3 -0
- smith_utils-0.4.0/src/smith_utils/file/__init__.py +6 -0
- smith_utils-0.4.0/src/smith_utils/file/classification.py +177 -0
- smith_utils-0.4.0/tests/test_file_classification.py +79 -0
- {smith_utils-0.3.1 → smith_utils-0.4.0}/tests/test_package_exports.py +2 -0
- {smith_utils-0.3.1 → smith_utils-0.4.0}/.gitignore +0 -0
- {smith_utils-0.3.1 → smith_utils-0.4.0}/LICENSE +0 -0
- {smith_utils-0.3.1 → smith_utils-0.4.0}/src/smith_utils/crypto/__init__.py +0 -0
- {smith_utils-0.3.1 → smith_utils-0.4.0}/src/smith_utils/crypto/hashes.py +0 -0
- {smith_utils-0.3.1 → smith_utils-0.4.0}/src/smith_utils/datetime/__init__.py +0 -0
- {smith_utils-0.3.1 → smith_utils-0.4.0}/src/smith_utils/datetime/date_utils.py +0 -0
- {smith_utils-0.3.1 → smith_utils-0.4.0}/src/smith_utils/numeric/__init__.py +0 -0
- {smith_utils-0.3.1 → smith_utils-0.4.0}/src/smith_utils/numeric/refinement.py +0 -0
- {smith_utils-0.3.1 → smith_utils-0.4.0}/src/smith_utils/text/__init__.py +0 -0
- {smith_utils-0.3.1 → smith_utils-0.4.0}/src/smith_utils/text/metrics.py +0 -0
- {smith_utils-0.3.1 → smith_utils-0.4.0}/src/smith_utils/text/newlines.py +0 -0
- {smith_utils-0.3.1 → smith_utils-0.4.0}/src/smith_utils/text/normalization.py +0 -0
- {smith_utils-0.3.1 → smith_utils-0.4.0}/src/smith_utils/text/unicode_char_names.py +0 -0
- {smith_utils-0.3.1 → smith_utils-0.4.0}/tests/test_date_utils.py +0 -0
- {smith_utils-0.3.1 → smith_utils-0.4.0}/tests/test_hashes.py +0 -0
- {smith_utils-0.3.1 → smith_utils-0.4.0}/tests/test_metrics.py +0 -0
- {smith_utils-0.3.1 → smith_utils-0.4.0}/tests/test_newlines.py +0 -0
- {smith_utils-0.3.1 → smith_utils-0.4.0}/tests/test_normalization.py +0 -0
- {smith_utils-0.3.1 → smith_utils-0.4.0}/tests/test_refinement.py +0 -0
- {smith_utils-0.3.1 → smith_utils-0.4.0}/tests/test_unicode_char_names.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: smith-utils
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: A utility library for data cleaning and parsing.
|
|
5
5
|
Project-URL: Homepage, https://github.com/yeiichi/smith-utils
|
|
6
6
|
Project-URL: Repository, https://github.com/yeiichi/smith-utils
|
|
@@ -86,6 +86,11 @@ Calculate SHA-256 digests for text and files.
|
|
|
86
86
|
- `get_text_digest`: Returns the SHA-256 hexadecimal digest for a text string.
|
|
87
87
|
- `get_file_digest`: Returns the SHA-256 hexadecimal digest for a file using streaming reads.
|
|
88
88
|
|
|
89
|
+
### File Classification Utilities (`smith_utils.file`)
|
|
90
|
+
Classify files from multiple evidence sources.
|
|
91
|
+
- `classify_file`: Returns extension, MIME, magic-number, and `file(1)` classification evidence.
|
|
92
|
+
- `FileClassification`: Dataclass result containing raw signals, `file_class`, and derived categories.
|
|
93
|
+
|
|
89
94
|
## Installation
|
|
90
95
|
|
|
91
96
|
Install via pip:
|
|
@@ -99,7 +104,7 @@ pip install smith-utils
|
|
|
99
104
|
```python
|
|
100
105
|
from smith_utils import ensure_date, get_text_digest, parse_numeric_value, normalize_text
|
|
101
106
|
from smith_utils import make_unicode_char_name_records
|
|
102
|
-
from smith_utils import get_file_digest, normalize_file_to_lf
|
|
107
|
+
from smith_utils import classify_file, get_file_digest, normalize_file_to_lf
|
|
103
108
|
|
|
104
109
|
# Datetime
|
|
105
110
|
date = ensure_date("20231225") # datetime.date(2023, 12, 25)
|
|
@@ -121,6 +126,10 @@ summary = normalize_file_to_lf("input.txt", "output.txt")
|
|
|
121
126
|
# SHA-256 digests
|
|
122
127
|
text_digest = get_text_digest("smith-utils")
|
|
123
128
|
file_digest = get_file_digest("input.txt")
|
|
129
|
+
|
|
130
|
+
# File classification
|
|
131
|
+
classification = classify_file("input.pdf")
|
|
132
|
+
# FileClassification(extension='.pdf', file_class='document', categories=('document', 'pdf'), ...)
|
|
124
133
|
```
|
|
125
134
|
|
|
126
135
|
## Directory Structure
|
|
@@ -38,6 +38,11 @@ Calculate SHA-256 digests for text and files.
|
|
|
38
38
|
- `get_text_digest`: Returns the SHA-256 hexadecimal digest for a text string.
|
|
39
39
|
- `get_file_digest`: Returns the SHA-256 hexadecimal digest for a file using streaming reads.
|
|
40
40
|
|
|
41
|
+
### File Classification Utilities (`smith_utils.file`)
|
|
42
|
+
Classify files from multiple evidence sources.
|
|
43
|
+
- `classify_file`: Returns extension, MIME, magic-number, and `file(1)` classification evidence.
|
|
44
|
+
- `FileClassification`: Dataclass result containing raw signals, `file_class`, and derived categories.
|
|
45
|
+
|
|
41
46
|
## Installation
|
|
42
47
|
|
|
43
48
|
Install via pip:
|
|
@@ -51,7 +56,7 @@ pip install smith-utils
|
|
|
51
56
|
```python
|
|
52
57
|
from smith_utils import ensure_date, get_text_digest, parse_numeric_value, normalize_text
|
|
53
58
|
from smith_utils import make_unicode_char_name_records
|
|
54
|
-
from smith_utils import get_file_digest, normalize_file_to_lf
|
|
59
|
+
from smith_utils import classify_file, get_file_digest, normalize_file_to_lf
|
|
55
60
|
|
|
56
61
|
# Datetime
|
|
57
62
|
date = ensure_date("20231225") # datetime.date(2023, 12, 25)
|
|
@@ -73,6 +78,10 @@ summary = normalize_file_to_lf("input.txt", "output.txt")
|
|
|
73
78
|
# SHA-256 digests
|
|
74
79
|
text_digest = get_text_digest("smith-utils")
|
|
75
80
|
file_digest = get_file_digest("input.txt")
|
|
81
|
+
|
|
82
|
+
# File classification
|
|
83
|
+
classification = classify_file("input.pdf")
|
|
84
|
+
# FileClassification(extension='.pdf', file_class='document', categories=('document', 'pdf'), ...)
|
|
76
85
|
```
|
|
77
86
|
|
|
78
87
|
## Directory Structure
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from .crypto import get_file_digest, get_text_digest
|
|
2
2
|
from .datetime import ensure_date, format_ordinal, parse_strict_date
|
|
3
|
+
from .file import FileClassification, classify_file
|
|
3
4
|
from .numeric import parse_currency_value, parse_numeric_value
|
|
4
5
|
from .text import (
|
|
5
6
|
NewlineType,
|
|
@@ -21,6 +22,8 @@ __all__ = [
|
|
|
21
22
|
"StringDistance",
|
|
22
23
|
"UnicodeCharNameRecord",
|
|
23
24
|
"analyze_pair",
|
|
25
|
+
"FileClassification",
|
|
26
|
+
"classify_file",
|
|
24
27
|
"ensure_date",
|
|
25
28
|
"format_ordinal",
|
|
26
29
|
"get_file_digest",
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"""Utilities for classifying file types from multiple signals."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
import mimetypes
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
import shutil
|
|
9
|
+
import subprocess
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
_MAGIC_SIGNATURES = [
|
|
13
|
+
(b"%PDF-", "PDF document", "application/pdf", {"document", "pdf"}),
|
|
14
|
+
(b"\x89PNG\r\n\x1a\n", "PNG image", "image/png", {"image", "png"}),
|
|
15
|
+
(b"\xff\xd8\xff", "JPEG image", "image/jpeg", {"image", "jpeg"}),
|
|
16
|
+
(b"GIF87a", "GIF image", "image/gif", {"image", "gif"}),
|
|
17
|
+
(b"GIF89a", "GIF image", "image/gif", {"image", "gif"}),
|
|
18
|
+
(b"PK\x03\x04", "ZIP archive", "application/zip", {"archive", "zip"}),
|
|
19
|
+
(b"PK\x05\x06", "ZIP archive", "application/zip", {"archive", "zip"}),
|
|
20
|
+
(b"PK\x07\x08", "ZIP archive", "application/zip", {"archive", "zip"}),
|
|
21
|
+
(b"\x1f\x8b", "gzip compressed data", "application/gzip", {"archive", "gzip"}),
|
|
22
|
+
(b"BZh", "bzip2 compressed data", "application/x-bzip2", {"archive", "bzip2"}),
|
|
23
|
+
(b"7z\xbc\xaf\x27\x1c", "7-zip archive", "application/x-7z-compressed", {"archive", "7z"}),
|
|
24
|
+
(b"SQLite format 3\x00", "SQLite database", "application/vnd.sqlite3", {"database", "sqlite"}),
|
|
25
|
+
(b"\x7fELF", "ELF executable", "application/x-elf", {"binary", "executable", "elf"}),
|
|
26
|
+
(b"\xca\xfe\xba\xbe", "Java class or Mach-O universal binary", "application/octet-stream", {"binary"}),
|
|
27
|
+
(b"\xfe\xed\xfa\xce", "Mach-O executable", "application/x-mach-binary", {"binary", "executable", "mach-o"}),
|
|
28
|
+
(b"\xfe\xed\xfa\xcf", "Mach-O executable", "application/x-mach-binary", {"binary", "executable", "mach-o"}),
|
|
29
|
+
(b"\xce\xfa\xed\xfe", "Mach-O executable", "application/x-mach-binary", {"binary", "executable", "mach-o"}),
|
|
30
|
+
(b"\xcf\xfa\xed\xfe", "Mach-O executable", "application/x-mach-binary", {"binary", "executable", "mach-o"}),
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
_MIME_CATEGORIES = {
|
|
35
|
+
"application/gzip": {"archive", "gzip"},
|
|
36
|
+
"application/pdf": {"document", "pdf"},
|
|
37
|
+
"application/vnd.sqlite3": {"database", "sqlite"},
|
|
38
|
+
"application/x-7z-compressed": {"archive", "7z"},
|
|
39
|
+
"application/x-bzip2": {"archive", "bzip2"},
|
|
40
|
+
"application/x-elf": {"binary", "executable", "elf"},
|
|
41
|
+
"application/x-mach-binary": {"binary", "executable", "mach-o"},
|
|
42
|
+
"application/zip": {"archive", "zip"},
|
|
43
|
+
"text/csv": {"text", "csv"},
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
_EXTENSION_CATEGORIES = {
|
|
48
|
+
".7z": {"archive", "7z"},
|
|
49
|
+
".bz2": {"archive", "bzip2"},
|
|
50
|
+
".csv": {"text", "csv"},
|
|
51
|
+
".db": {"database"},
|
|
52
|
+
".gif": {"image", "gif"},
|
|
53
|
+
".gz": {"archive", "gzip"},
|
|
54
|
+
".jpeg": {"image", "jpeg"},
|
|
55
|
+
".jpg": {"image", "jpeg"},
|
|
56
|
+
".json": {"text", "json"},
|
|
57
|
+
".md": {"text", "markdown"},
|
|
58
|
+
".pdf": {"document", "pdf"},
|
|
59
|
+
".png": {"image", "png"},
|
|
60
|
+
".sqlite": {"database", "sqlite"},
|
|
61
|
+
".sqlite3": {"database", "sqlite"},
|
|
62
|
+
".txt": {"text"},
|
|
63
|
+
".zip": {"archive", "zip"},
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
_CLASS_PRIORITY = (
|
|
68
|
+
"document",
|
|
69
|
+
"image",
|
|
70
|
+
"video",
|
|
71
|
+
"audio",
|
|
72
|
+
"archive",
|
|
73
|
+
"database",
|
|
74
|
+
"text",
|
|
75
|
+
"binary",
|
|
76
|
+
"font",
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@dataclass(frozen=True)
|
|
81
|
+
class FileClassification:
|
|
82
|
+
"""Classification evidence for a file path."""
|
|
83
|
+
|
|
84
|
+
path: Path
|
|
85
|
+
extension: str
|
|
86
|
+
file_description: str | None
|
|
87
|
+
file_mime_type: str | None
|
|
88
|
+
extension_mime_type: str | None
|
|
89
|
+
magic_type: str | None
|
|
90
|
+
magic_mime_type: str | None
|
|
91
|
+
file_class: str | None
|
|
92
|
+
categories: tuple[str, ...]
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def classify_file(path: str | Path, *, sample_size: int = 4096) -> FileClassification:
|
|
96
|
+
"""Classify a file using extension, magic bytes, MIME, and ``file(1)`` signals."""
|
|
97
|
+
|
|
98
|
+
file_path = Path(path)
|
|
99
|
+
extension = file_path.suffix.lower()
|
|
100
|
+
header = _read_header(file_path, sample_size)
|
|
101
|
+
magic_type, magic_mime_type, magic_categories = _classify_magic(header)
|
|
102
|
+
file_description = _run_file_command(file_path, "--brief")
|
|
103
|
+
file_mime_type = _run_file_command(file_path, "--brief", "--mime-type")
|
|
104
|
+
extension_mime_type = mimetypes.guess_type(file_path.name)[0]
|
|
105
|
+
|
|
106
|
+
categories = set()
|
|
107
|
+
categories.update(_EXTENSION_CATEGORIES.get(extension, set()))
|
|
108
|
+
categories.update(magic_categories)
|
|
109
|
+
categories.update(_categories_for_mime(extension_mime_type))
|
|
110
|
+
categories.update(_categories_for_mime(file_mime_type))
|
|
111
|
+
categories.update(_categories_for_mime(magic_mime_type))
|
|
112
|
+
|
|
113
|
+
sorted_categories = tuple(sorted(categories))
|
|
114
|
+
|
|
115
|
+
return FileClassification(
|
|
116
|
+
path=file_path,
|
|
117
|
+
extension=extension,
|
|
118
|
+
file_description=file_description,
|
|
119
|
+
file_mime_type=file_mime_type,
|
|
120
|
+
extension_mime_type=extension_mime_type,
|
|
121
|
+
magic_type=magic_type,
|
|
122
|
+
magic_mime_type=magic_mime_type,
|
|
123
|
+
file_class=_select_file_class(sorted_categories),
|
|
124
|
+
categories=sorted_categories,
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _read_header(path: Path, sample_size: int) -> bytes:
|
|
129
|
+
if sample_size < 1:
|
|
130
|
+
raise ValueError("sample_size must be greater than zero")
|
|
131
|
+
|
|
132
|
+
with path.open("rb") as file:
|
|
133
|
+
return file.read(sample_size)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _classify_magic(header: bytes) -> tuple[str | None, str | None, set[str]]:
|
|
137
|
+
for signature, file_type, mime_type, categories in _MAGIC_SIGNATURES:
|
|
138
|
+
if header.startswith(signature):
|
|
139
|
+
return file_type, mime_type, set(categories)
|
|
140
|
+
return None, None, set()
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _categories_for_mime(mime_type: str | None) -> set[str]:
|
|
144
|
+
if not mime_type:
|
|
145
|
+
return set()
|
|
146
|
+
|
|
147
|
+
categories = set(_MIME_CATEGORIES.get(mime_type, set()))
|
|
148
|
+
root_type = mime_type.split("/", 1)[0]
|
|
149
|
+
if root_type in {"audio", "font", "image", "text", "video"}:
|
|
150
|
+
categories.add(root_type)
|
|
151
|
+
return categories
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _select_file_class(categories: tuple[str, ...]) -> str | None:
|
|
155
|
+
category_set = set(categories)
|
|
156
|
+
for file_class in _CLASS_PRIORITY:
|
|
157
|
+
if file_class in category_set:
|
|
158
|
+
return file_class
|
|
159
|
+
return categories[0] if categories else None
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _run_file_command(path: Path, *args: str) -> str | None:
|
|
163
|
+
if shutil.which("file") is None:
|
|
164
|
+
return None
|
|
165
|
+
|
|
166
|
+
try:
|
|
167
|
+
result = subprocess.run(
|
|
168
|
+
["file", *args, str(path)],
|
|
169
|
+
capture_output=True,
|
|
170
|
+
check=True,
|
|
171
|
+
text=True,
|
|
172
|
+
)
|
|
173
|
+
except (OSError, subprocess.CalledProcessError):
|
|
174
|
+
return None
|
|
175
|
+
|
|
176
|
+
output = result.stdout.strip()
|
|
177
|
+
return output or None
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
from src.smith_utils import FileClassification, classify_file
|
|
4
|
+
from src.smith_utils.file import classification
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def test_classify_file_combines_extension_magic_and_mime(tmp_path):
|
|
8
|
+
sample_file = tmp_path / "sample.pdf"
|
|
9
|
+
sample_file.write_bytes(b"%PDF-1.7\nsmith-utils\n")
|
|
10
|
+
|
|
11
|
+
result = classify_file(sample_file)
|
|
12
|
+
|
|
13
|
+
assert isinstance(result, FileClassification)
|
|
14
|
+
assert result.path == sample_file
|
|
15
|
+
assert result.extension == ".pdf"
|
|
16
|
+
assert result.extension_mime_type == "application/pdf"
|
|
17
|
+
assert result.magic_type == "PDF document"
|
|
18
|
+
assert result.magic_mime_type == "application/pdf"
|
|
19
|
+
assert result.file_class == "document"
|
|
20
|
+
assert "document" in result.categories
|
|
21
|
+
assert "pdf" in result.categories
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def test_classify_file_detects_magic_even_when_extension_disagrees(tmp_path):
|
|
25
|
+
sample_file = tmp_path / "wrong.txt"
|
|
26
|
+
sample_file.write_bytes(b"\x89PNG\r\n\x1a\nimage bytes")
|
|
27
|
+
|
|
28
|
+
result = classify_file(sample_file)
|
|
29
|
+
|
|
30
|
+
assert result.extension == ".txt"
|
|
31
|
+
assert result.extension_mime_type == "text/plain"
|
|
32
|
+
assert result.magic_type == "PNG image"
|
|
33
|
+
assert result.magic_mime_type == "image/png"
|
|
34
|
+
assert result.file_class == "image"
|
|
35
|
+
assert "text" in result.categories
|
|
36
|
+
assert "image" in result.categories
|
|
37
|
+
assert "png" in result.categories
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def test_classify_file_returns_file_command_output_when_available(tmp_path, monkeypatch):
|
|
41
|
+
sample_file = tmp_path / "sample.bin"
|
|
42
|
+
sample_file.write_bytes(b"data")
|
|
43
|
+
|
|
44
|
+
monkeypatch.setattr(classification.shutil, "which", lambda command: "/usr/bin/file")
|
|
45
|
+
|
|
46
|
+
def fake_run(command, capture_output, check, text):
|
|
47
|
+
assert command[-1] == str(sample_file)
|
|
48
|
+
|
|
49
|
+
class Result:
|
|
50
|
+
stdout = "ASCII text\n" if "--mime-type" not in command else "text/plain\n"
|
|
51
|
+
|
|
52
|
+
return Result()
|
|
53
|
+
|
|
54
|
+
monkeypatch.setattr(classification.subprocess, "run", fake_run)
|
|
55
|
+
|
|
56
|
+
result = classify_file(sample_file)
|
|
57
|
+
|
|
58
|
+
assert result.file_description == "ASCII text"
|
|
59
|
+
assert result.file_mime_type == "text/plain"
|
|
60
|
+
assert "text" in result.categories
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def test_classify_file_tolerates_missing_file_command(tmp_path, monkeypatch):
|
|
64
|
+
sample_file = tmp_path / "sample.unknown"
|
|
65
|
+
sample_file.write_bytes(b"data")
|
|
66
|
+
monkeypatch.setattr(classification.shutil, "which", lambda command: None)
|
|
67
|
+
|
|
68
|
+
result = classify_file(sample_file)
|
|
69
|
+
|
|
70
|
+
assert result.file_description is None
|
|
71
|
+
assert result.file_mime_type is None
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def test_classify_file_rejects_invalid_sample_size(tmp_path):
|
|
75
|
+
sample_file = tmp_path / "sample.txt"
|
|
76
|
+
sample_file.write_text("text")
|
|
77
|
+
|
|
78
|
+
with pytest.raises(ValueError, match="sample_size"):
|
|
79
|
+
classify_file(sample_file, sample_size=0)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|