smith-utils 0.3.1__tar.gz → 0.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. {smith_utils-0.3.1 → smith_utils-0.4.1}/PKG-INFO +29 -3
  2. {smith_utils-0.3.1 → smith_utils-0.4.1}/README.md +28 -2
  3. {smith_utils-0.3.1 → smith_utils-0.4.1}/pyproject.toml +4 -1
  4. {smith_utils-0.3.1 → smith_utils-0.4.1}/src/smith_utils/__init__.py +9 -0
  5. smith_utils-0.4.1/src/smith_utils/cli.py +126 -0
  6. smith_utils-0.4.1/src/smith_utils/file/__init__.py +6 -0
  7. smith_utils-0.4.1/src/smith_utils/file/classification.py +177 -0
  8. {smith_utils-0.3.1 → smith_utils-0.4.1}/src/smith_utils/text/__init__.py +4 -0
  9. smith_utils-0.4.1/src/smith_utils/text/random_string.py +54 -0
  10. smith_utils-0.4.1/tests/test_cli.py +79 -0
  11. smith_utils-0.4.1/tests/test_file_classification.py +79 -0
  12. {smith_utils-0.3.1 → smith_utils-0.4.1}/tests/test_package_exports.py +5 -0
  13. smith_utils-0.4.1/tests/test_random_string.py +64 -0
  14. {smith_utils-0.3.1 → smith_utils-0.4.1}/.gitignore +0 -0
  15. {smith_utils-0.3.1 → smith_utils-0.4.1}/LICENSE +0 -0
  16. {smith_utils-0.3.1 → smith_utils-0.4.1}/src/smith_utils/crypto/__init__.py +0 -0
  17. {smith_utils-0.3.1 → smith_utils-0.4.1}/src/smith_utils/crypto/hashes.py +0 -0
  18. {smith_utils-0.3.1 → smith_utils-0.4.1}/src/smith_utils/datetime/__init__.py +0 -0
  19. {smith_utils-0.3.1 → smith_utils-0.4.1}/src/smith_utils/datetime/date_utils.py +0 -0
  20. {smith_utils-0.3.1 → smith_utils-0.4.1}/src/smith_utils/numeric/__init__.py +0 -0
  21. {smith_utils-0.3.1 → smith_utils-0.4.1}/src/smith_utils/numeric/refinement.py +0 -0
  22. {smith_utils-0.3.1 → smith_utils-0.4.1}/src/smith_utils/text/metrics.py +0 -0
  23. {smith_utils-0.3.1 → smith_utils-0.4.1}/src/smith_utils/text/newlines.py +0 -0
  24. {smith_utils-0.3.1 → smith_utils-0.4.1}/src/smith_utils/text/normalization.py +0 -0
  25. {smith_utils-0.3.1 → smith_utils-0.4.1}/src/smith_utils/text/unicode_char_names.py +0 -0
  26. {smith_utils-0.3.1 → smith_utils-0.4.1}/tests/test_date_utils.py +0 -0
  27. {smith_utils-0.3.1 → smith_utils-0.4.1}/tests/test_hashes.py +0 -0
  28. {smith_utils-0.3.1 → smith_utils-0.4.1}/tests/test_metrics.py +0 -0
  29. {smith_utils-0.3.1 → smith_utils-0.4.1}/tests/test_newlines.py +0 -0
  30. {smith_utils-0.3.1 → smith_utils-0.4.1}/tests/test_normalization.py +0 -0
  31. {smith_utils-0.3.1 → smith_utils-0.4.1}/tests/test_refinement.py +0 -0
  32. {smith_utils-0.3.1 → smith_utils-0.4.1}/tests/test_unicode_char_names.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: smith-utils
3
- Version: 0.3.1
3
+ Version: 0.4.1
4
4
  Summary: A utility library for data cleaning and parsing.
5
5
  Project-URL: Homepage, https://github.com/yeiichi/smith-utils
6
6
  Project-URL: Repository, https://github.com/yeiichi/smith-utils
@@ -74,6 +74,7 @@ Clean and parse messy numeric data.
74
74
  ### Text Normalization & Metrics (`smith_utils.text`)
75
75
  Standardize text and compare string similarity.
76
76
  - `normalize_text`: Unicode NFKC normalization, case folding, and whitespace handling.
77
+ - `random_string`: Generate random ASCII, kanji, or mixed strings using secure randomness.
77
78
  - `StringDistance`: Implementation of Damerau-Levenshtein and Jaro-Winkler algorithms for fuzzy matching.
78
79
  - `analyze_pair`: Convenience function for string comparison returning a `Result`.
79
80
  - `Relation` / `Result`: Relation enum and typed result for text comparisons.
@@ -86,6 +87,11 @@ Calculate SHA-256 digests for text and files.
86
87
  - `get_text_digest`: Returns the SHA-256 hexadecimal digest for a text string.
87
88
  - `get_file_digest`: Returns the SHA-256 hexadecimal digest for a file using streaming reads.
88
89
 
90
+ ### File Classification Utilities (`smith_utils.file`)
91
+ Classify files from multiple evidence sources.
92
+ - `classify_file`: Returns extension, MIME, magic-number, and `file(1)` classification evidence.
93
+ - `FileClassification`: Dataclass result containing raw signals, `file_class`, and derived categories.
94
+
89
95
  ## Installation
90
96
 
91
97
  Install via pip:
@@ -94,12 +100,14 @@ Install via pip:
94
100
  pip install smith-utils
95
101
  ```
96
102
 
103
+ This also installs the `smith` command.
104
+
97
105
  ## Quick Start
98
106
 
99
107
  ```python
100
- from smith_utils import ensure_date, get_text_digest, parse_numeric_value, normalize_text
108
+ from smith_utils import ensure_date, get_text_digest, parse_numeric_value, normalize_text, random_string
101
109
  from smith_utils import make_unicode_char_name_records
102
- from smith_utils import get_file_digest, normalize_file_to_lf
110
+ from smith_utils import classify_file, get_file_digest, normalize_file_to_lf
103
111
 
104
112
  # Datetime
105
113
  date = ensure_date("20231225") # datetime.date(2023, 12, 25)
@@ -110,6 +118,9 @@ value = parse_numeric_value("(1,250.50)") # -1250.5
110
118
  # Text
111
119
  clean_text = normalize_text(" Smith Utils ") # "smith utils"
112
120
 
121
+ # Random strings
122
+ token = random_string(16)
123
+
113
124
  # Unicode metadata
114
125
  records = make_unicode_char_name_records("Aあ")
115
126
  # [UnicodeCharNameRecord(index=0, codepoint='U+0041', ...), ...]
@@ -121,6 +132,21 @@ summary = normalize_file_to_lf("input.txt", "output.txt")
121
132
  # SHA-256 digests
122
133
  text_digest = get_text_digest("smith-utils")
123
134
  file_digest = get_file_digest("input.txt")
135
+
136
+ # File classification
137
+ classification = classify_file("input.pdf")
138
+ # FileClassification(extension='.pdf', file_class='document', categories=('document', 'pdf'), ...)
139
+ ```
140
+
141
+ ## CLI
142
+
143
+ ```bash
144
+ smith random-string 16
145
+ smith digest --text "smith-utils"
146
+ smith digest input.txt
147
+ smith normalize-text " Smith Utils "
148
+ smith unicode-names "Aあ"
149
+ smith classify-file input.pdf
124
150
  ```
125
151
 
126
152
  ## Directory Structure
@@ -26,6 +26,7 @@ Clean and parse messy numeric data.
26
26
  ### Text Normalization & Metrics (`smith_utils.text`)
27
27
  Standardize text and compare string similarity.
28
28
  - `normalize_text`: Unicode NFKC normalization, case folding, and whitespace handling.
29
+ - `random_string`: Generate random ASCII, kanji, or mixed strings using secure randomness.
29
30
  - `StringDistance`: Implementation of Damerau-Levenshtein and Jaro-Winkler algorithms for fuzzy matching.
30
31
  - `analyze_pair`: Convenience function for string comparison returning a `Result`.
31
32
  - `Relation` / `Result`: Relation enum and typed result for text comparisons.
@@ -38,6 +39,11 @@ Calculate SHA-256 digests for text and files.
38
39
  - `get_text_digest`: Returns the SHA-256 hexadecimal digest for a text string.
39
40
  - `get_file_digest`: Returns the SHA-256 hexadecimal digest for a file using streaming reads.
40
41
 
42
+ ### File Classification Utilities (`smith_utils.file`)
43
+ Classify files from multiple evidence sources.
44
+ - `classify_file`: Returns extension, MIME, magic-number, and `file(1)` classification evidence.
45
+ - `FileClassification`: Dataclass result containing raw signals, `file_class`, and derived categories.
46
+
41
47
  ## Installation
42
48
 
43
49
  Install via pip:
@@ -46,12 +52,14 @@ Install via pip:
46
52
  pip install smith-utils
47
53
  ```
48
54
 
55
+ This also installs the `smith` command.
56
+
49
57
  ## Quick Start
50
58
 
51
59
  ```python
52
- from smith_utils import ensure_date, get_text_digest, parse_numeric_value, normalize_text
60
+ from smith_utils import ensure_date, get_text_digest, parse_numeric_value, normalize_text, random_string
53
61
  from smith_utils import make_unicode_char_name_records
54
- from smith_utils import get_file_digest, normalize_file_to_lf
62
+ from smith_utils import classify_file, get_file_digest, normalize_file_to_lf
55
63
 
56
64
  # Datetime
57
65
  date = ensure_date("20231225") # datetime.date(2023, 12, 25)
@@ -62,6 +70,9 @@ value = parse_numeric_value("(1,250.50)") # -1250.5
62
70
  # Text
63
71
  clean_text = normalize_text(" Smith Utils ") # "smith utils"
64
72
 
73
+ # Random strings
74
+ token = random_string(16)
75
+
65
76
  # Unicode metadata
66
77
  records = make_unicode_char_name_records("Aあ")
67
78
  # [UnicodeCharNameRecord(index=0, codepoint='U+0041', ...), ...]
@@ -73,6 +84,21 @@ summary = normalize_file_to_lf("input.txt", "output.txt")
73
84
  # SHA-256 digests
74
85
  text_digest = get_text_digest("smith-utils")
75
86
  file_digest = get_file_digest("input.txt")
87
+
88
+ # File classification
89
+ classification = classify_file("input.pdf")
90
+ # FileClassification(extension='.pdf', file_class='document', categories=('document', 'pdf'), ...)
91
+ ```
92
+
93
+ ## CLI
94
+
95
+ ```bash
96
+ smith random-string 16
97
+ smith digest --text "smith-utils"
98
+ smith digest input.txt
99
+ smith normalize-text " Smith Utils "
100
+ smith unicode-names "Aあ"
101
+ smith classify-file input.pdf
76
102
  ```
77
103
 
78
104
  ## Directory Structure
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "smith-utils"
7
- version = "0.3.1"
7
+ version = "0.4.1"
8
8
  description = "A utility library for data cleaning and parsing."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -33,6 +33,9 @@ classifiers = [
33
33
 
34
34
  dependencies = []
35
35
 
36
+ [project.scripts]
37
+ smith = "smith_utils.cli:main"
38
+
36
39
  [project.optional-dependencies]
37
40
  dev = [
38
41
  "pytest>=8,<9",
@@ -1,7 +1,9 @@
1
1
  from .crypto import get_file_digest, get_text_digest
2
2
  from .datetime import ensure_date, format_ordinal, parse_strict_date
3
+ from .file import FileClassification, classify_file
3
4
  from .numeric import parse_currency_value, parse_numeric_value
4
5
  from .text import (
6
+ CharCategory,
5
7
  NewlineType,
6
8
  Relation,
7
9
  Result,
@@ -12,15 +14,20 @@ from .text import (
12
14
  normalize_file_to_lf,
13
15
  normalize_newlines_stream,
14
16
  normalize_text,
17
+ random_mixed_char,
18
+ random_string,
15
19
  )
16
20
 
17
21
  __all__ = [
22
+ "CharCategory",
18
23
  "NewlineType",
19
24
  "Relation",
20
25
  "Result",
21
26
  "StringDistance",
22
27
  "UnicodeCharNameRecord",
23
28
  "analyze_pair",
29
+ "FileClassification",
30
+ "classify_file",
24
31
  "ensure_date",
25
32
  "format_ordinal",
26
33
  "get_file_digest",
@@ -32,4 +39,6 @@ __all__ = [
32
39
  "parse_currency_value",
33
40
  "parse_numeric_value",
34
41
  "parse_strict_date",
42
+ "random_mixed_char",
43
+ "random_string",
35
44
  ]
@@ -0,0 +1,126 @@
1
+ """Command line interface for smith-utils."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import sys
7
+
8
+ from . import (
9
+ classify_file,
10
+ get_file_digest,
11
+ get_text_digest,
12
+ make_unicode_char_name_records,
13
+ normalize_text,
14
+ random_string,
15
+ )
16
+
17
+
18
+ def main(argv: list[str] | None = None) -> int:
19
+ parser = _make_parser()
20
+ args = parser.parse_args(argv)
21
+ args.func(args)
22
+ return 0
23
+
24
+
25
+ def _make_parser() -> argparse.ArgumentParser:
26
+ parser = argparse.ArgumentParser(prog="smith")
27
+ subparsers = parser.add_subparsers(dest="command", required=True)
28
+
29
+ random_parser = subparsers.add_parser("random-string", help="generate a random string")
30
+ random_parser.add_argument("length", type=int)
31
+ random_parser.add_argument(
32
+ "--category",
33
+ choices=("ascii", "kanji", "mix"),
34
+ default="ascii",
35
+ help="character category to use",
36
+ )
37
+ random_parser.add_argument(
38
+ "--ascii-ratio",
39
+ type=float,
40
+ default=0.5,
41
+ help="ASCII probability for --category mix",
42
+ )
43
+ random_parser.set_defaults(func=_random_string_command)
44
+
45
+ digest_parser = subparsers.add_parser("digest", help="calculate a SHA-256 digest")
46
+ digest_input = digest_parser.add_mutually_exclusive_group(required=True)
47
+ digest_input.add_argument("--text", help="text to digest")
48
+ digest_input.add_argument("path", nargs="?", help="file path to digest")
49
+ digest_parser.set_defaults(func=_digest_command)
50
+
51
+ normalize_parser = subparsers.add_parser("normalize-text", help="normalize text")
52
+ normalize_parser.add_argument("text", nargs="?", help="text to normalize; reads stdin when omitted")
53
+ normalize_parser.add_argument("--keep-case", action="store_true", help="preserve text case")
54
+ normalize_parser.add_argument(
55
+ "--remove-all-whitespace",
56
+ action="store_true",
57
+ help="remove all whitespace instead of collapsing it",
58
+ )
59
+ normalize_parser.add_argument("--no-nfkc", action="store_true", help="skip NFKC normalization")
60
+ normalize_parser.set_defaults(func=_normalize_text_command)
61
+
62
+ unicode_parser = subparsers.add_parser("unicode-names", help="show Unicode character names")
63
+ unicode_parser.add_argument("text", help="text to inspect")
64
+ unicode_parser.set_defaults(func=_unicode_names_command)
65
+
66
+ classify_parser = subparsers.add_parser("classify-file", help="classify a file")
67
+ classify_parser.add_argument("path", help="file path to classify")
68
+ classify_parser.set_defaults(func=_classify_file_command)
69
+
70
+ return parser
71
+
72
+
73
+ def _random_string_command(args: argparse.Namespace) -> None:
74
+ print(
75
+ random_string(
76
+ args.length,
77
+ char_category=args.category,
78
+ ascii_ratio=args.ascii_ratio,
79
+ )
80
+ )
81
+
82
+
83
+ def _digest_command(args: argparse.Namespace) -> None:
84
+ if args.text is not None:
85
+ print(get_text_digest(args.text))
86
+ return
87
+
88
+ print(get_file_digest(args.path))
89
+
90
+
91
+ def _normalize_text_command(args: argparse.Namespace) -> None:
92
+ text = sys.stdin.read() if args.text is None else args.text
93
+ print(
94
+ normalize_text(
95
+ text,
96
+ ignore_case=not args.keep_case,
97
+ remove_all_whitespace=args.remove_all_whitespace,
98
+ nfkc=not args.no_nfkc,
99
+ )
100
+ )
101
+
102
+
103
+ def _unicode_names_command(args: argparse.Namespace) -> None:
104
+ for record in make_unicode_char_name_records(args.text):
105
+ print(f"{record.index}\t{record.codepoint}\t{record.char}\t{record.name}")
106
+
107
+
108
+ def _classify_file_command(args: argparse.Namespace) -> None:
109
+ result = classify_file(args.path)
110
+ fields = (
111
+ ("path", str(result.path)),
112
+ ("extension", result.extension),
113
+ ("file_class", result.file_class),
114
+ ("categories", ",".join(result.categories)),
115
+ ("extension_mime_type", result.extension_mime_type),
116
+ ("magic_type", result.magic_type),
117
+ ("magic_mime_type", result.magic_mime_type),
118
+ ("file_description", result.file_description),
119
+ ("file_mime_type", result.file_mime_type),
120
+ )
121
+ for key, value in fields:
122
+ print(f"{key}={'' if value is None else value}")
123
+
124
+
125
+ if __name__ == "__main__":
126
+ raise SystemExit(main())
@@ -0,0 +1,6 @@
1
+ from .classification import FileClassification, classify_file
2
+
3
+ __all__ = [
4
+ "FileClassification",
5
+ "classify_file",
6
+ ]
@@ -0,0 +1,177 @@
1
+ """Utilities for classifying file types from multiple signals."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ import mimetypes
7
+ from pathlib import Path
8
+ import shutil
9
+ import subprocess
10
+
11
+
12
+ _MAGIC_SIGNATURES = [
13
+ (b"%PDF-", "PDF document", "application/pdf", {"document", "pdf"}),
14
+ (b"\x89PNG\r\n\x1a\n", "PNG image", "image/png", {"image", "png"}),
15
+ (b"\xff\xd8\xff", "JPEG image", "image/jpeg", {"image", "jpeg"}),
16
+ (b"GIF87a", "GIF image", "image/gif", {"image", "gif"}),
17
+ (b"GIF89a", "GIF image", "image/gif", {"image", "gif"}),
18
+ (b"PK\x03\x04", "ZIP archive", "application/zip", {"archive", "zip"}),
19
+ (b"PK\x05\x06", "ZIP archive", "application/zip", {"archive", "zip"}),
20
+ (b"PK\x07\x08", "ZIP archive", "application/zip", {"archive", "zip"}),
21
+ (b"\x1f\x8b", "gzip compressed data", "application/gzip", {"archive", "gzip"}),
22
+ (b"BZh", "bzip2 compressed data", "application/x-bzip2", {"archive", "bzip2"}),
23
+ (b"7z\xbc\xaf\x27\x1c", "7-zip archive", "application/x-7z-compressed", {"archive", "7z"}),
24
+ (b"SQLite format 3\x00", "SQLite database", "application/vnd.sqlite3", {"database", "sqlite"}),
25
+ (b"\x7fELF", "ELF executable", "application/x-elf", {"binary", "executable", "elf"}),
26
+ (b"\xca\xfe\xba\xbe", "Java class or Mach-O universal binary", "application/octet-stream", {"binary"}),
27
+ (b"\xfe\xed\xfa\xce", "Mach-O executable", "application/x-mach-binary", {"binary", "executable", "mach-o"}),
28
+ (b"\xfe\xed\xfa\xcf", "Mach-O executable", "application/x-mach-binary", {"binary", "executable", "mach-o"}),
29
+ (b"\xce\xfa\xed\xfe", "Mach-O executable", "application/x-mach-binary", {"binary", "executable", "mach-o"}),
30
+ (b"\xcf\xfa\xed\xfe", "Mach-O executable", "application/x-mach-binary", {"binary", "executable", "mach-o"}),
31
+ ]
32
+
33
+
34
+ _MIME_CATEGORIES = {
35
+ "application/gzip": {"archive", "gzip"},
36
+ "application/pdf": {"document", "pdf"},
37
+ "application/vnd.sqlite3": {"database", "sqlite"},
38
+ "application/x-7z-compressed": {"archive", "7z"},
39
+ "application/x-bzip2": {"archive", "bzip2"},
40
+ "application/x-elf": {"binary", "executable", "elf"},
41
+ "application/x-mach-binary": {"binary", "executable", "mach-o"},
42
+ "application/zip": {"archive", "zip"},
43
+ "text/csv": {"text", "csv"},
44
+ }
45
+
46
+
47
+ _EXTENSION_CATEGORIES = {
48
+ ".7z": {"archive", "7z"},
49
+ ".bz2": {"archive", "bzip2"},
50
+ ".csv": {"text", "csv"},
51
+ ".db": {"database"},
52
+ ".gif": {"image", "gif"},
53
+ ".gz": {"archive", "gzip"},
54
+ ".jpeg": {"image", "jpeg"},
55
+ ".jpg": {"image", "jpeg"},
56
+ ".json": {"text", "json"},
57
+ ".md": {"text", "markdown"},
58
+ ".pdf": {"document", "pdf"},
59
+ ".png": {"image", "png"},
60
+ ".sqlite": {"database", "sqlite"},
61
+ ".sqlite3": {"database", "sqlite"},
62
+ ".txt": {"text"},
63
+ ".zip": {"archive", "zip"},
64
+ }
65
+
66
+
67
+ _CLASS_PRIORITY = (
68
+ "document",
69
+ "image",
70
+ "video",
71
+ "audio",
72
+ "archive",
73
+ "database",
74
+ "text",
75
+ "binary",
76
+ "font",
77
+ )
78
+
79
+
80
+ @dataclass(frozen=True)
81
+ class FileClassification:
82
+ """Classification evidence for a file path."""
83
+
84
+ path: Path
85
+ extension: str
86
+ file_description: str | None
87
+ file_mime_type: str | None
88
+ extension_mime_type: str | None
89
+ magic_type: str | None
90
+ magic_mime_type: str | None
91
+ file_class: str | None
92
+ categories: tuple[str, ...]
93
+
94
+
95
+ def classify_file(path: str | Path, *, sample_size: int = 4096) -> FileClassification:
96
+ """Classify a file using extension, magic bytes, MIME, and ``file(1)`` signals."""
97
+
98
+ file_path = Path(path)
99
+ extension = file_path.suffix.lower()
100
+ header = _read_header(file_path, sample_size)
101
+ magic_type, magic_mime_type, magic_categories = _classify_magic(header)
102
+ file_description = _run_file_command(file_path, "--brief")
103
+ file_mime_type = _run_file_command(file_path, "--brief", "--mime-type")
104
+ extension_mime_type = mimetypes.guess_type(file_path.name)[0]
105
+
106
+ categories = set()
107
+ categories.update(_EXTENSION_CATEGORIES.get(extension, set()))
108
+ categories.update(magic_categories)
109
+ categories.update(_categories_for_mime(extension_mime_type))
110
+ categories.update(_categories_for_mime(file_mime_type))
111
+ categories.update(_categories_for_mime(magic_mime_type))
112
+
113
+ sorted_categories = tuple(sorted(categories))
114
+
115
+ return FileClassification(
116
+ path=file_path,
117
+ extension=extension,
118
+ file_description=file_description,
119
+ file_mime_type=file_mime_type,
120
+ extension_mime_type=extension_mime_type,
121
+ magic_type=magic_type,
122
+ magic_mime_type=magic_mime_type,
123
+ file_class=_select_file_class(sorted_categories),
124
+ categories=sorted_categories,
125
+ )
126
+
127
+
128
+ def _read_header(path: Path, sample_size: int) -> bytes:
129
+ if sample_size < 1:
130
+ raise ValueError("sample_size must be greater than zero")
131
+
132
+ with path.open("rb") as file:
133
+ return file.read(sample_size)
134
+
135
+
136
+ def _classify_magic(header: bytes) -> tuple[str | None, str | None, set[str]]:
137
+ for signature, file_type, mime_type, categories in _MAGIC_SIGNATURES:
138
+ if header.startswith(signature):
139
+ return file_type, mime_type, set(categories)
140
+ return None, None, set()
141
+
142
+
143
+ def _categories_for_mime(mime_type: str | None) -> set[str]:
144
+ if not mime_type:
145
+ return set()
146
+
147
+ categories = set(_MIME_CATEGORIES.get(mime_type, set()))
148
+ root_type = mime_type.split("/", 1)[0]
149
+ if root_type in {"audio", "font", "image", "text", "video"}:
150
+ categories.add(root_type)
151
+ return categories
152
+
153
+
154
+ def _select_file_class(categories: tuple[str, ...]) -> str | None:
155
+ category_set = set(categories)
156
+ for file_class in _CLASS_PRIORITY:
157
+ if file_class in category_set:
158
+ return file_class
159
+ return categories[0] if categories else None
160
+
161
+
162
+ def _run_file_command(path: Path, *args: str) -> str | None:
163
+ if shutil.which("file") is None:
164
+ return None
165
+
166
+ try:
167
+ result = subprocess.run(
168
+ ["file", *args, str(path)],
169
+ capture_output=True,
170
+ check=True,
171
+ text=True,
172
+ )
173
+ except (OSError, subprocess.CalledProcessError):
174
+ return None
175
+
176
+ output = result.stdout.strip()
177
+ return output or None
@@ -1,12 +1,16 @@
1
1
  from .metrics import StringDistance, analyze_pair, Result, Relation
2
2
  from .newlines import NewlineType, normalize_file_to_lf, normalize_newlines_stream
3
3
  from .normalization import normalize_text
4
+ from .random_string import CharCategory, random_mixed_char, random_string
4
5
  from .unicode_char_names import UnicodeCharNameRecord, make_unicode_char_name_records
5
6
 
6
7
  __all__ = [
8
+ "CharCategory",
7
9
  "NewlineType",
8
10
  "normalize_file_to_lf",
9
11
  "normalize_newlines_stream",
12
+ "random_mixed_char",
13
+ "random_string",
10
14
  "StringDistance",
11
15
  "analyze_pair",
12
16
  "Result",
@@ -0,0 +1,54 @@
1
+ """Utilities for generating random strings."""
2
+
3
+ import secrets
4
+ import string
5
+ from typing import Literal
6
+
7
+ CharCategory = Literal["ascii", "kanji", "mix"]
8
+
9
+ ASCII_CHARS = string.ascii_letters + string.digits
10
+ KANJI_CHARS = "".join(chr(i) for i in range(0x4E00, 0xA000))
11
+
12
+ CHARSETS = {
13
+ "ascii": ASCII_CHARS,
14
+ "kanji": KANJI_CHARS,
15
+ }
16
+
17
+
18
+ def _validate_length(length: int) -> None:
19
+ if length < 0:
20
+ raise ValueError("length must be greater than or equal to 0")
21
+
22
+
23
+ def _validate_ascii_ratio(ascii_ratio: float) -> None:
24
+ if not 0 <= ascii_ratio <= 1:
25
+ raise ValueError("ascii_ratio must be between 0 and 1")
26
+
27
+
28
+ def random_mixed_char(*, ascii_ratio: float = 0.5) -> str:
29
+ """Return one random ASCII or kanji character based on ``ascii_ratio``."""
30
+
31
+ _validate_ascii_ratio(ascii_ratio)
32
+ charset = ASCII_CHARS if secrets.randbelow(10_000) < ascii_ratio * 10_000 else KANJI_CHARS
33
+ return secrets.choice(charset)
34
+
35
+
36
+ def random_string(
37
+ length: int,
38
+ *,
39
+ char_category: CharCategory = "ascii",
40
+ ascii_ratio: float = 0.5,
41
+ ) -> str:
42
+ """Generate a random string of the given length and category."""
43
+
44
+ _validate_length(length)
45
+ if char_category == "mix":
46
+ return "".join(random_mixed_char(ascii_ratio=ascii_ratio) for _ in range(length))
47
+
48
+ try:
49
+ population = CHARSETS[char_category]
50
+ except KeyError:
51
+ allowed = ", ".join(repr(key) for key in (*CHARSETS, "mix"))
52
+ raise ValueError(f"char_category must be one of: {allowed}") from None
53
+
54
+ return "".join(secrets.choice(population) for _ in range(length))
@@ -0,0 +1,79 @@
1
+ import hashlib
2
+ import string
3
+
4
+ from src.smith_utils.cli import main
5
+
6
+
7
+ def test_random_string_command_prints_requested_length(capsys):
8
+ assert main(["random-string", "12"]) == 0
9
+
10
+ output = capsys.readouterr().out.strip()
11
+ assert len(output) == 12
12
+ assert set(output) <= set(string.ascii_letters + string.digits)
13
+
14
+
15
+ def test_random_string_command_accepts_category(capsys):
16
+ assert main(["random-string", "4", "--category", "kanji"]) == 0
17
+
18
+ assert len(capsys.readouterr().out.strip()) == 4
19
+
20
+
21
+ def test_digest_command_hashes_text(capsys):
22
+ assert main(["digest", "--text", "smith-utils"]) == 0
23
+
24
+ assert capsys.readouterr().out.strip() == hashlib.sha256(b"smith-utils").hexdigest()
25
+
26
+
27
+ def test_digest_command_hashes_file(tmp_path, capsys):
28
+ sample_file = tmp_path / "sample.txt"
29
+ sample_file.write_bytes(b"file content")
30
+
31
+ assert main(["digest", str(sample_file)]) == 0
32
+
33
+ assert capsys.readouterr().out.strip() == hashlib.sha256(b"file content").hexdigest()
34
+
35
+
36
+ def test_normalize_text_command_normalizes_argument(capsys):
37
+ assert main(["normalize-text", " Smith Utils "]) == 0
38
+
39
+ assert capsys.readouterr().out.strip() == "smith utils"
40
+
41
+
42
+ def test_normalize_text_command_can_read_stdin(monkeypatch, capsys):
43
+ class FakeStdin:
44
+ def read(self):
45
+ return " Hello World "
46
+
47
+ monkeypatch.setattr("src.smith_utils.cli.sys.stdin", FakeStdin())
48
+
49
+ assert main(["normalize-text"]) == 0
50
+
51
+ assert capsys.readouterr().out.strip() == "hello world"
52
+
53
+
54
+ def test_normalize_text_command_can_keep_case(capsys):
55
+ assert main(["normalize-text", "--keep-case", " Hello World "]) == 0
56
+
57
+ assert capsys.readouterr().out.strip() == "Hello World"
58
+
59
+
60
+ def test_unicode_names_command_prints_tab_separated_records(capsys):
61
+ assert main(["unicode-names", "Aあ"]) == 0
62
+
63
+ assert capsys.readouterr().out.splitlines() == [
64
+ "0\tU+0041\tA\tLATIN CAPITAL LETTER A",
65
+ "1\tU+3042\tあ\tHIRAGANA LETTER A",
66
+ ]
67
+
68
+
69
+ def test_classify_file_command_prints_key_value_fields(tmp_path, capsys):
70
+ sample_file = tmp_path / "sample.pdf"
71
+ sample_file.write_bytes(b"%PDF-1.7\nsmith-utils\n")
72
+
73
+ assert main(["classify-file", str(sample_file)]) == 0
74
+
75
+ output = capsys.readouterr().out.splitlines()
76
+ assert f"path={sample_file}" in output
77
+ assert "extension=.pdf" in output
78
+ assert "file_class=document" in output
79
+ assert "magic_type=PDF document" in output
@@ -0,0 +1,79 @@
1
+ import pytest
2
+
3
+ from src.smith_utils import FileClassification, classify_file
4
+ from src.smith_utils.file import classification
5
+
6
+
7
+ def test_classify_file_combines_extension_magic_and_mime(tmp_path):
8
+ sample_file = tmp_path / "sample.pdf"
9
+ sample_file.write_bytes(b"%PDF-1.7\nsmith-utils\n")
10
+
11
+ result = classify_file(sample_file)
12
+
13
+ assert isinstance(result, FileClassification)
14
+ assert result.path == sample_file
15
+ assert result.extension == ".pdf"
16
+ assert result.extension_mime_type == "application/pdf"
17
+ assert result.magic_type == "PDF document"
18
+ assert result.magic_mime_type == "application/pdf"
19
+ assert result.file_class == "document"
20
+ assert "document" in result.categories
21
+ assert "pdf" in result.categories
22
+
23
+
24
+ def test_classify_file_detects_magic_even_when_extension_disagrees(tmp_path):
25
+ sample_file = tmp_path / "wrong.txt"
26
+ sample_file.write_bytes(b"\x89PNG\r\n\x1a\nimage bytes")
27
+
28
+ result = classify_file(sample_file)
29
+
30
+ assert result.extension == ".txt"
31
+ assert result.extension_mime_type == "text/plain"
32
+ assert result.magic_type == "PNG image"
33
+ assert result.magic_mime_type == "image/png"
34
+ assert result.file_class == "image"
35
+ assert "text" in result.categories
36
+ assert "image" in result.categories
37
+ assert "png" in result.categories
38
+
39
+
40
+ def test_classify_file_returns_file_command_output_when_available(tmp_path, monkeypatch):
41
+ sample_file = tmp_path / "sample.bin"
42
+ sample_file.write_bytes(b"data")
43
+
44
+ monkeypatch.setattr(classification.shutil, "which", lambda command: "/usr/bin/file")
45
+
46
+ def fake_run(command, capture_output, check, text):
47
+ assert command[-1] == str(sample_file)
48
+
49
+ class Result:
50
+ stdout = "ASCII text\n" if "--mime-type" not in command else "text/plain\n"
51
+
52
+ return Result()
53
+
54
+ monkeypatch.setattr(classification.subprocess, "run", fake_run)
55
+
56
+ result = classify_file(sample_file)
57
+
58
+ assert result.file_description == "ASCII text"
59
+ assert result.file_mime_type == "text/plain"
60
+ assert "text" in result.categories
61
+
62
+
63
+ def test_classify_file_tolerates_missing_file_command(tmp_path, monkeypatch):
64
+ sample_file = tmp_path / "sample.unknown"
65
+ sample_file.write_bytes(b"data")
66
+ monkeypatch.setattr(classification.shutil, "which", lambda command: None)
67
+
68
+ result = classify_file(sample_file)
69
+
70
+ assert result.file_description is None
71
+ assert result.file_mime_type is None
72
+
73
+
74
+ def test_classify_file_rejects_invalid_sample_size(tmp_path):
75
+ sample_file = tmp_path / "sample.txt"
76
+ sample_file.write_text("text")
77
+
78
+ with pytest.raises(ValueError, match="sample_size"):
79
+ classify_file(sample_file, sample_size=0)
@@ -5,9 +5,12 @@ def test_exports_are_available_globally():
5
5
  expected = {
6
6
  "get_file_digest",
7
7
  "get_text_digest",
8
+ "CharCategory",
8
9
  "NewlineType",
9
10
  "normalize_file_to_lf",
10
11
  "normalize_newlines_stream",
12
+ "random_mixed_char",
13
+ "random_string",
11
14
  "StringDistance",
12
15
  "analyze_pair",
13
16
  "Result",
@@ -15,6 +18,8 @@ def test_exports_are_available_globally():
15
18
  "normalize_text",
16
19
  "UnicodeCharNameRecord",
17
20
  "make_unicode_char_name_records",
21
+ "FileClassification",
22
+ "classify_file",
18
23
  }
19
24
 
20
25
  for name in expected:
@@ -0,0 +1,64 @@
1
+ import string
2
+
3
+ import pytest
4
+
5
+ from src.smith_utils import random_mixed_char, random_string
6
+ from src.smith_utils.text import CharCategory
7
+ from src.smith_utils.text.random_string import ASCII_CHARS, KANJI_CHARS
8
+
9
+
10
+ def test_random_string_generates_ascii_string_of_requested_length():
11
+ result = random_string(32)
12
+
13
+ assert len(result) == 32
14
+ assert set(result) <= set(string.ascii_letters + string.digits)
15
+
16
+
17
+ def test_random_string_generates_kanji_string_of_requested_length():
18
+ result = random_string(16, char_category="kanji")
19
+
20
+ assert len(result) == 16
21
+ assert set(result) <= set(KANJI_CHARS)
22
+
23
+
24
+ def test_random_string_generates_mixed_string_of_requested_length():
25
+ result = random_string(20, char_category="mix")
26
+
27
+ assert len(result) == 20
28
+ assert set(result) <= set(ASCII_CHARS + KANJI_CHARS)
29
+
30
+
31
+ def test_random_string_accepts_zero_length():
32
+ assert random_string(0) == ""
33
+
34
+
35
+ def test_random_string_rejects_negative_length():
36
+ with pytest.raises(ValueError, match="length"):
37
+ random_string(-1)
38
+
39
+
40
+ def test_random_string_rejects_unknown_char_category():
41
+ with pytest.raises(ValueError, match="char_category"):
42
+ random_string(8, char_category="emoji")
43
+
44
+
45
+ @pytest.mark.parametrize("ascii_ratio", [-0.1, 1.1])
46
+ def test_random_mixed_char_rejects_ascii_ratio_outside_unit_interval(ascii_ratio):
47
+ with pytest.raises(ValueError, match="ascii_ratio"):
48
+ random_mixed_char(ascii_ratio=ascii_ratio)
49
+
50
+
51
+ def test_random_mixed_char_can_force_ascii():
52
+ result = random_mixed_char(ascii_ratio=1)
53
+
54
+ assert result in ASCII_CHARS
55
+
56
+
57
+ def test_random_mixed_char_can_force_kanji():
58
+ result = random_mixed_char(ascii_ratio=0)
59
+
60
+ assert result in KANJI_CHARS
61
+
62
+
63
+ def test_char_category_type_alias_is_exported():
64
+ assert CharCategory is not None
File without changes
File without changes