oj-toolkit 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/PKG-INFO +84 -16
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/README.md +83 -15
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/console/terminal.py +17 -13
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/parsing/consts.py +4 -1
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/parsing/types.py +33 -8
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit.egg-info/PKG-INFO +84 -16
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/pyproject.toml +1 -1
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/LICENSE +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/__init__.py +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/asynchronous/__init__.py +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/asynchronous/async_chunks.py +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/asynchronous/decorators.py +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/console/__init__.py +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/console/box.py +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/console/colored_text.py +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/console/colors.py +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/console/status.py +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/console/streams.py +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/console/table.py +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/console/text.py +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/data/__init__.py +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/data/flex.py +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/diagnostics.py +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/logging/__init__.py +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/logging/config.py +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/logging/consts.py +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/logging/decorators.py +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/logging/formatters.py +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/logging/handlers.py +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit/parsing/__init__.py +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit.egg-info/SOURCES.txt +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit.egg-info/dependency_links.txt +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit.egg-info/requires.txt +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/oj_toolkit.egg-info/top_level.txt +0 -0
- {oj_toolkit-0.2.0 → oj_toolkit-0.2.2}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: oj-toolkit
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Centralized shared utilities for ownjoo-org projects: standardized logging config, type validation, data parsing, console output, and async utilities
|
|
5
5
|
Author-email: Speedy <speedy@ownjoo.org>
|
|
6
6
|
License: MIT
|
|
@@ -33,7 +33,7 @@ Requires-Dist: mypy>=1.0; extra == "dev"
|
|
|
33
33
|
Requires-Dist: pylint>=2.0; extra == "dev"
|
|
34
34
|
Dynamic: license-file
|
|
35
35
|
|
|
36
|
-
#
|
|
36
|
+
# oj-toolkit
|
|
37
37
|
|
|
38
38
|
Centralized shared utilities library for all ownjoo-org projects. Provides battle-tested functions for type validation, data parsing, and progress logging.
|
|
39
39
|
|
|
@@ -187,11 +187,19 @@ fields = dig_many(
|
|
|
187
187
|
# Bind a path once, reuse it against many objects
|
|
188
188
|
get_name = Digger(path='name', exp=str, default='')
|
|
189
189
|
[get_name(user) for user in data['users']] # Returns: ['Alice', 'Bob']
|
|
190
|
+
|
|
191
|
+
# Constrain a string result with a regex (checked via re.fullmatch after conversion)
|
|
192
|
+
mac = dig({'device': {'mac': 'AA:BB:CC:DD:EE:FF'}}, path='device.mac', exp=str,
|
|
193
|
+
pattern=r'([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}')
|
|
194
|
+
# Returns: 'AA:BB:CC:DD:EE:FF'
|
|
190
195
|
```
|
|
191
196
|
|
|
192
|
-
> **Note:**
|
|
193
|
-
> `
|
|
194
|
-
>
|
|
197
|
+
> **Note:** if the path resolves to nothing (missing key/index, or `src` doesn't contain it),
|
|
198
|
+
> `dig()`'s default `post_processor` (`validate()`) still runs and returns `default` (`None`
|
|
199
|
+
> unless set) -- a missing path never silently drops `default` in favor of a bare `None`.
|
|
200
|
+
> Passing no `exp=` means "accept any type, no isinstance check" -- not "always fail". Pass
|
|
201
|
+
> `exp=<type>` to add a type check, or `post_processor=None` to bypass validation entirely and
|
|
202
|
+
> get the raw match untouched.
|
|
195
203
|
|
|
196
204
|
### Logging Setup
|
|
197
205
|
|
|
@@ -711,18 +719,25 @@ else:
|
|
|
711
719
|
|
|
712
720
|
Returns `True` if the terminal likely supports Unicode characters. Used internally to choose between ASCII and Unicode box/table borders.
|
|
713
721
|
|
|
722
|
+
Checks `NO_COLOR`/`CI`/`TERM` env vars first, then verifies `sys.stdout.encoding` can actually
|
|
723
|
+
encode a box-drawing character -- this catches Windows consoles that default to a codepage
|
|
724
|
+
(`cp1252`, `cp437`, etc.) that can't, even though the platform/terminal is otherwise
|
|
725
|
+
Unicode-capable. Set `PYTHONIOENCODING=utf-8` (or use Windows Terminal with a UTF-8 codepage)
|
|
726
|
+
to get Unicode borders on Windows instead of the ASCII fallback.
|
|
727
|
+
|
|
714
728
|
### `parsing` Module
|
|
715
729
|
|
|
716
|
-
#### `validate(v, exp=None, default=None, converter=None, validator=None, **kwargs)`
|
|
730
|
+
#### `validate(v, exp=None, default=None, converter=None, validator=None, pattern=None, **kwargs)`
|
|
717
731
|
|
|
718
732
|
Generic validation utility that converts and validates a value.
|
|
719
733
|
|
|
720
734
|
- **Parameters:**
|
|
721
735
|
- `v` (Any): The value to validate
|
|
722
|
-
- `exp` (Type): Expected type. Enables automatic converter selection (list → str_to_list, datetime → get_datetime)
|
|
723
|
-
- `default` (Any): Return this if validation fails
|
|
724
|
-
- `converter` (Callable): Custom converter function. Default: auto-selected based on exp
|
|
725
|
-
- `validator` (Callable): Custom validator function(result, exp, **kwargs) → bool. Default: isinstance check
|
|
736
|
+
- `exp` (Type): Expected type. Enables automatic converter selection (list → str_to_list, datetime → get_datetime). If `None`, no isinstance check is performed -- any type passes (see `validator` below)
|
|
737
|
+
- `default` (Any): Return this if validation fails, or `v` is missing/`None`. Default: None
|
|
738
|
+
- `converter` (Callable): Custom converter function. Default: auto-selected based on exp. Auto-selection is limited to `list`/`datetime` -- `bool`/`int`/`float`/etc. are pass-through unless you supply your own `converter=`
|
|
739
|
+
- `validator` (Callable): Custom validator function(result, exp, **kwargs) → bool. Default: isinstance check (`exp=None` always passes; a non-type `exp` is rejected rather than raising)
|
|
740
|
+
- `pattern` (str | re.Pattern): Optional regex checked via `re.fullmatch` against the result, but only when the result is a `str` -- ignored for other types. Applied after conversion/validation succeed; a mismatch falls back to `default`
|
|
726
741
|
- `**kwargs`: Passed to converter and validator
|
|
727
742
|
|
|
728
743
|
- **Returns:** Converted and validated value, or default if validation fails
|
|
@@ -734,6 +749,16 @@ Generic validation utility that converts and validates a value.
|
|
|
734
749
|
numbers = validate('1,2,3', exp=list) # Returns: ['1', '2', '3']
|
|
735
750
|
dt = validate('2024-01-15T10:30:00', exp=datetime) # Returns: datetime object
|
|
736
751
|
|
|
752
|
+
# exp=bool/int/float do NOT auto-coerce strings -- pass converter= explicitly for that
|
|
753
|
+
validate('42', exp=int) # Returns: None ('42' is a str, not an int)
|
|
754
|
+
validate('42', exp=int, converter=int) # Returns: 42
|
|
755
|
+
|
|
756
|
+
# Regex-constrain a string result
|
|
757
|
+
validate('AA:BB:CC:DD:EE:FF', exp=str, pattern=r'([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}')
|
|
758
|
+
# Returns: 'AA:BB:CC:DD:EE:FF'
|
|
759
|
+
validate('not-a-mac', exp=str, pattern=r'([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}', default='invalid')
|
|
760
|
+
# Returns: 'invalid'
|
|
761
|
+
|
|
737
762
|
# Custom converter and validator
|
|
738
763
|
def to_upper(v, *args, **kwargs):
|
|
739
764
|
return str(v).upper()
|
|
@@ -795,6 +820,12 @@ get_datetime('01/15/2024 10:30:00', format_str='%m/%d/%Y %H:%M:%S') # Custom
|
|
|
795
820
|
Extract and post-process a value from a nested data structure, using [jmespath](https://jmespath.org/)
|
|
796
821
|
under the hood (`jmespath` is a required dependency).
|
|
797
822
|
|
|
823
|
+
> **Compilation caching:** compiled jmespath expressions are cached in a process-global
|
|
824
|
+
> `lru_cache(maxsize=256)`, keyed by the expression string. A bare `dig()` call already avoids
|
|
825
|
+
> recompiling a repeated path -- this isn't a `Digger`-only optimization. The cache holds up to
|
|
826
|
+
> 256 distinct expressions across the whole process (shared by every `dig()`/`Digger` caller),
|
|
827
|
+
> LRU-evicted beyond that.
|
|
828
|
+
|
|
798
829
|
- **Parameters:**
|
|
799
830
|
- `src` (Mapping | Sequence): Data structure to navigate
|
|
800
831
|
- `path` (int | str | list | None):
|
|
@@ -811,16 +842,18 @@ under the hood (`jmespath` is a required dependency).
|
|
|
811
842
|
Pass `None` to skip post-processing and get the raw match.
|
|
812
843
|
- `**kwargs`: Passed to `post_processor`
|
|
813
844
|
|
|
814
|
-
- **Returns:** Post-processed value
|
|
815
|
-
|
|
845
|
+
- **Returns:** Post-processed value. If extraction fails or no candidate path matches, the
|
|
846
|
+
post-processor (default `validate()`) still runs and returns `default` (`None` unless set) --
|
|
847
|
+
unless `post_processor=None`, in which case the raw (possibly missing/`None`) value is returned
|
|
848
|
+
untouched with no default substitution.
|
|
816
849
|
|
|
817
850
|
> **Type checkers:** pass `exp=<type>` to get a precise `<type> | None` return type instead of `Any`
|
|
818
851
|
> (`dig(data, path='...', exp=str)` type-checks as `str | None`). Passing `post_processor=None` types
|
|
819
852
|
> as `Any` (raw, unvalidated value). A custom `post_processor` types as that callable's own return type.
|
|
820
853
|
|
|
821
|
-
> **Passing no `exp`:** the default `validate()` post-processor
|
|
822
|
-
>
|
|
823
|
-
>
|
|
854
|
+
> **Passing no `exp`:** the default `validate()` post-processor treats a missing `exp` as "accept
|
|
855
|
+
> any type" -- it does not fail. Pass `exp=<type>` to add an isinstance check, or
|
|
856
|
+
> `post_processor=None` to bypass validation entirely and get the raw match.
|
|
824
857
|
|
|
825
858
|
**Example:**
|
|
826
859
|
|
|
@@ -855,6 +888,39 @@ name = dig(data, path=['response.users[0].display_name', 'response.users[0].name
|
|
|
855
888
|
# Returns: 'Alice'
|
|
856
889
|
```
|
|
857
890
|
|
|
891
|
+
##### `dig()` Behavior Matrix
|
|
892
|
+
|
|
893
|
+
Every row below is a real `dig()` call, backed by a passing unit test in
|
|
894
|
+
`test/unit/parsing/test_parsing.py` -- the test name is the proof, not just a claim. This isn't
|
|
895
|
+
every possible combination of kwargs (that cross product is huge and mostly redundant); it's one
|
|
896
|
+
representative call per distinct decision point in `dig()`/`validate()`. `*(unset)*` means the
|
|
897
|
+
kwarg wasn't passed (the function's own default applies); `--` means it isn't relevant to that row.
|
|
898
|
+
|
|
899
|
+
| `src` | `path` | `exp` | `default` | `post_processor` | `pattern` | `validator` | `pop` | Result | Proven by |
|
|
900
|
+
|---|---|---|---|---|---|---|---|---|---|
|
|
901
|
+
| `{'first': 'a', 'second': ['blah']}` | `'second[0]'` (found) | `str` | *(unset)* | *(unset)* | *(unset)* | *(unset)* | `False` | `'blah'` | `test_should_get_value_from_dict` |
|
|
902
|
+
| `['', ['blah']]` | `'[1][0]'` (found) | `str` | `''` | *(unset)* | *(unset)* | *(unset)* | `False` | `'blah'` (default unused -- value was valid) | `test_should_get_value_from_list` |
|
|
903
|
+
| `{'a': {'b': 'value'}}` | `'x.y.z'` (missing) | `str` | `'default'` | *(unset)* | *(unset)* | *(unset)* | `False` | `'default'` | `test_should_return_default_on_missing_intermediate_key` |
|
|
904
|
+
| `{'items': ['a', 'b', 'c']}` | `'items.invalid_index'` (missing) | `str` | `'default'` | *(unset)* | *(unset)* | *(unset)* | `False` | `'default'` | `test_should_return_default_when_accessing_list_with_string_key` |
|
|
905
|
+
| `{'a': {'b': 'value'}}` | `'x.y.z'` (missing) | `str` | *(unset)* | *(unset)* | *(unset)* | *(unset)* | `False` | `None` (no default set) | `test_should_return_none_for_missing_path_with_exp_and_no_default` |
|
|
906
|
+
| `{'a': {'b': 'c'}}` | `'x.y.z'` (missing) | *(unset)* | *(unset)* | *(unset)* | *(unset)* | *(unset)* | `False` | `None` (`exp=None` means "anything passes"; the found value already is `None`) | `test_should_handle_get_value_with_invalid_path` |
|
|
907
|
+
| `{'key': 'blah'}` | `'key'` (found) | *(unset)* | *(unset)* | *(unset)* | *(unset)* | *(unset)* | `False` | `'blah'` (raw pass-through, no isinstance check) | `test_should_not_crash_when_exp_omitted_from_dig` |
|
|
908
|
+
| `{'a': 'not-an-int'}` | `'a'` (found, wrong type) | `int` | *(unset)* | *(unset)* | *(unset)* | *(unset)* | `False` | `None` | `test_should_return_none_for_dig_result_type_mismatch` |
|
|
909
|
+
| `{'a': 'not-an-int'}` | `'a'` (found, wrong type) | `int` | `-1` | *(unset)* | *(unset)* | *(unset)* | `False` | `-1` | `test_should_return_default_for_dig_result_type_mismatch` |
|
|
910
|
+
| `{'a': 'blah'}` | `'a'` (found, non-type `exp`) | `'str'` | `'fallback'` | *(unset)* | *(unset)* | *(unset)* | `False` | `'fallback'` (garbage `exp` rejected, not raised) | `test_should_return_default_for_dig_with_non_type_expected_type` |
|
|
911
|
+
| `{'key': 'raw_value'}` | `'key'` (found) | -- | -- | `None` | -- | -- | `False` | raw value, unvalidated | `test_should_return_raw_value_when_no_post_processor` |
|
|
912
|
+
| `{'a': 1}` | `'missing'` (missing) | -- | `'default'` (ignored) | `None` | -- | -- | `False` | `None` (`default` is never consulted -- it's `validate()`'s kwarg, and `validate()` never runs) | `test_should_return_none_for_missing_path_with_no_post_processor` |
|
|
913
|
+
| `{'response': {'users': [{'name': 'Alice'}, {'name': 'Bob'}]}}` | `'response.users'` (found) | -- | -- | `len` | -- | -- | `False` | `2` (custom post_processor gets the raw match) | `test_should_use_custom_post_processor_function` |
|
|
914
|
+
| `'blah'` | `None` (src is the value) | `str` | *(unset)* | *(unset)* | *(unset)* | custom lambda | `False` | `'blah'` (custom validator overrides isinstance check) | `test_should_get_value_with_passed_validator` |
|
|
915
|
+
| `'test_string'` | `None` (src is the value) | `str` | *(unset)* | *(unset)* | *(unset)* | *(unset)* | `False` | the src value itself | `test_should_post_process_source_when_path_is_none` |
|
|
916
|
+
| `{'device': {'mac': 'AA:BB:CC:DD:EE:FF'}}` | `'device.mac'` (found) | `str` | *(unset)* | *(unset)* | matching regex | *(unset)* | `False` | matched MAC string | `test_should_validate_dig_result_with_matching_pattern` |
|
|
917
|
+
| `{'device': {'mac': 'not-a-mac'}}` | `'device.mac'` (found) | `str` | `'invalid'` | *(unset)* | non-matching regex | *(unset)* | `False` | `'invalid'` | `test_should_return_default_for_dig_result_with_non_matching_pattern` |
|
|
918
|
+
| `{'a': 1, 'b': 2}` | `'a'` (found) | `int` | *(unset)* | *(unset)* | -- | -- | `True` | `1`, and `'a'` removed from `src` | `test_should_pop_key_from_dict` |
|
|
919
|
+
| `{'items': [{'id': 1}, {'id': 2}]}` | `'items[?id==\`2\`].id'` (found, ambiguous) | `list` | *(unset)* | *(unset)* | -- | -- | `True` (refused) | `[2]`, `src` left unmutated (pop refused + warning logged) | `test_should_refuse_pop_with_ambiguous_jmespath_filter` |
|
|
920
|
+
| `{'user': {'name': 'Alice'}}` | `['user.nickname', 'user.name']` (fallback) | `str` | *(unset)* | *(unset)* | -- | -- | `False` | `'Alice'` (first candidate wins) | `test_should_use_first_matching_path_in_fallback_list` |
|
|
921
|
+
| `{'user': {'name': 'Alice'}}` | `['user.nickname', 'user.alias']` (fallback, none match) | `str` | *(unset)* | *(unset)* | -- | -- | `False` | `None` | `test_should_return_none_when_no_fallback_path_matches` |
|
|
922
|
+
| `{'user': {'name': 'Alice', 'nickname': 'Ali'}}` | `['user.nickname', 'user.name']` (fallback) | `str` | *(unset)* | *(unset)* | -- | -- | `True` | `'Ali'`, only the winning candidate's key removed | `test_should_pop_only_the_winning_fallback_path` |
|
|
923
|
+
|
|
858
924
|
#### `dig_many(src, paths, **common_kwargs)`
|
|
859
925
|
|
|
860
926
|
Extract several named fields from `src` in one call.
|
|
@@ -889,7 +955,9 @@ fields = dig_many(
|
|
|
889
955
|
|
|
890
956
|
A pre-bound, reusable `dig()` call. Validates/compiles the jmespath expression once at
|
|
891
957
|
construction (failing fast on a bad expression instead of on first use), then can be called
|
|
892
|
-
like a function against any number of `src` objects without repeating `path`/`exp`/kwargs.
|
|
958
|
+
like a function against any number of `src` objects without repeating `path`/`exp`/kwargs. A
|
|
959
|
+
string `pattern=` kwarg is likewise pre-compiled to a `re.Pattern` at construction time, so
|
|
960
|
+
repeated calls don't re-compile the regex.
|
|
893
961
|
|
|
894
962
|
**Example:**
|
|
895
963
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
#
|
|
1
|
+
# oj-toolkit
|
|
2
2
|
|
|
3
3
|
Centralized shared utilities library for all ownjoo-org projects. Provides battle-tested functions for type validation, data parsing, and progress logging.
|
|
4
4
|
|
|
@@ -152,11 +152,19 @@ fields = dig_many(
|
|
|
152
152
|
# Bind a path once, reuse it against many objects
|
|
153
153
|
get_name = Digger(path='name', exp=str, default='')
|
|
154
154
|
[get_name(user) for user in data['users']] # Returns: ['Alice', 'Bob']
|
|
155
|
+
|
|
156
|
+
# Constrain a string result with a regex (checked via re.fullmatch after conversion)
|
|
157
|
+
mac = dig({'device': {'mac': 'AA:BB:CC:DD:EE:FF'}}, path='device.mac', exp=str,
|
|
158
|
+
pattern=r'([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}')
|
|
159
|
+
# Returns: 'AA:BB:CC:DD:EE:FF'
|
|
155
160
|
```
|
|
156
161
|
|
|
157
|
-
> **Note:**
|
|
158
|
-
> `
|
|
159
|
-
>
|
|
162
|
+
> **Note:** if the path resolves to nothing (missing key/index, or `src` doesn't contain it),
|
|
163
|
+
> `dig()`'s default `post_processor` (`validate()`) still runs and returns `default` (`None`
|
|
164
|
+
> unless set) -- a missing path never silently drops `default` in favor of a bare `None`.
|
|
165
|
+
> Passing no `exp=` means "accept any type, no isinstance check" -- not "always fail". Pass
|
|
166
|
+
> `exp=<type>` to add a type check, or `post_processor=None` to bypass validation entirely and
|
|
167
|
+
> get the raw match untouched.
|
|
160
168
|
|
|
161
169
|
### Logging Setup
|
|
162
170
|
|
|
@@ -676,18 +684,25 @@ else:
|
|
|
676
684
|
|
|
677
685
|
Returns `True` if the terminal likely supports Unicode characters. Used internally to choose between ASCII and Unicode box/table borders.
|
|
678
686
|
|
|
687
|
+
Checks `NO_COLOR`/`CI`/`TERM` env vars first, then verifies `sys.stdout.encoding` can actually
|
|
688
|
+
encode a box-drawing character -- this catches Windows consoles that default to a codepage
|
|
689
|
+
(`cp1252`, `cp437`, etc.) that can't, even though the platform/terminal is otherwise
|
|
690
|
+
Unicode-capable. Set `PYTHONIOENCODING=utf-8` (or use Windows Terminal with a UTF-8 codepage)
|
|
691
|
+
to get Unicode borders on Windows instead of the ASCII fallback.
|
|
692
|
+
|
|
679
693
|
### `parsing` Module
|
|
680
694
|
|
|
681
|
-
#### `validate(v, exp=None, default=None, converter=None, validator=None, **kwargs)`
|
|
695
|
+
#### `validate(v, exp=None, default=None, converter=None, validator=None, pattern=None, **kwargs)`
|
|
682
696
|
|
|
683
697
|
Generic validation utility that converts and validates a value.
|
|
684
698
|
|
|
685
699
|
- **Parameters:**
|
|
686
700
|
- `v` (Any): The value to validate
|
|
687
|
-
- `exp` (Type): Expected type. Enables automatic converter selection (list → str_to_list, datetime → get_datetime)
|
|
688
|
-
- `default` (Any): Return this if validation fails
|
|
689
|
-
- `converter` (Callable): Custom converter function. Default: auto-selected based on exp
|
|
690
|
-
- `validator` (Callable): Custom validator function(result, exp, **kwargs) → bool. Default: isinstance check
|
|
701
|
+
- `exp` (Type): Expected type. Enables automatic converter selection (list → str_to_list, datetime → get_datetime). If `None`, no isinstance check is performed -- any type passes (see `validator` below)
|
|
702
|
+
- `default` (Any): Return this if validation fails, or `v` is missing/`None`. Default: None
|
|
703
|
+
- `converter` (Callable): Custom converter function. Default: auto-selected based on exp. Auto-selection is limited to `list`/`datetime` -- `bool`/`int`/`float`/etc. are pass-through unless you supply your own `converter=`
|
|
704
|
+
- `validator` (Callable): Custom validator function(result, exp, **kwargs) → bool. Default: isinstance check (`exp=None` always passes; a non-type `exp` is rejected rather than raising)
|
|
705
|
+
- `pattern` (str | re.Pattern): Optional regex checked via `re.fullmatch` against the result, but only when the result is a `str` -- ignored for other types. Applied after conversion/validation succeed; a mismatch falls back to `default`
|
|
691
706
|
- `**kwargs`: Passed to converter and validator
|
|
692
707
|
|
|
693
708
|
- **Returns:** Converted and validated value, or default if validation fails
|
|
@@ -699,6 +714,16 @@ Generic validation utility that converts and validates a value.
|
|
|
699
714
|
numbers = validate('1,2,3', exp=list) # Returns: ['1', '2', '3']
|
|
700
715
|
dt = validate('2024-01-15T10:30:00', exp=datetime) # Returns: datetime object
|
|
701
716
|
|
|
717
|
+
# exp=bool/int/float do NOT auto-coerce strings -- pass converter= explicitly for that
|
|
718
|
+
validate('42', exp=int) # Returns: None ('42' is a str, not an int)
|
|
719
|
+
validate('42', exp=int, converter=int) # Returns: 42
|
|
720
|
+
|
|
721
|
+
# Regex-constrain a string result
|
|
722
|
+
validate('AA:BB:CC:DD:EE:FF', exp=str, pattern=r'([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}')
|
|
723
|
+
# Returns: 'AA:BB:CC:DD:EE:FF'
|
|
724
|
+
validate('not-a-mac', exp=str, pattern=r'([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}', default='invalid')
|
|
725
|
+
# Returns: 'invalid'
|
|
726
|
+
|
|
702
727
|
# Custom converter and validator
|
|
703
728
|
def to_upper(v, *args, **kwargs):
|
|
704
729
|
return str(v).upper()
|
|
@@ -760,6 +785,12 @@ get_datetime('01/15/2024 10:30:00', format_str='%m/%d/%Y %H:%M:%S') # Custom
|
|
|
760
785
|
Extract and post-process a value from a nested data structure, using [jmespath](https://jmespath.org/)
|
|
761
786
|
under the hood (`jmespath` is a required dependency).
|
|
762
787
|
|
|
788
|
+
> **Compilation caching:** compiled jmespath expressions are cached in a process-global
|
|
789
|
+
> `lru_cache(maxsize=256)`, keyed by the expression string. A bare `dig()` call already avoids
|
|
790
|
+
> recompiling a repeated path -- this isn't a `Digger`-only optimization. The cache holds up to
|
|
791
|
+
> 256 distinct expressions across the whole process (shared by every `dig()`/`Digger` caller),
|
|
792
|
+
> LRU-evicted beyond that.
|
|
793
|
+
|
|
763
794
|
- **Parameters:**
|
|
764
795
|
- `src` (Mapping | Sequence): Data structure to navigate
|
|
765
796
|
- `path` (int | str | list | None):
|
|
@@ -776,16 +807,18 @@ under the hood (`jmespath` is a required dependency).
|
|
|
776
807
|
Pass `None` to skip post-processing and get the raw match.
|
|
777
808
|
- `**kwargs`: Passed to `post_processor`
|
|
778
809
|
|
|
779
|
-
- **Returns:** Post-processed value
|
|
780
|
-
|
|
810
|
+
- **Returns:** Post-processed value. If extraction fails or no candidate path matches, the
|
|
811
|
+
post-processor (default `validate()`) still runs and returns `default` (`None` unless set) --
|
|
812
|
+
unless `post_processor=None`, in which case the raw (possibly missing/`None`) value is returned
|
|
813
|
+
untouched with no default substitution.
|
|
781
814
|
|
|
782
815
|
> **Type checkers:** pass `exp=<type>` to get a precise `<type> | None` return type instead of `Any`
|
|
783
816
|
> (`dig(data, path='...', exp=str)` type-checks as `str | None`). Passing `post_processor=None` types
|
|
784
817
|
> as `Any` (raw, unvalidated value). A custom `post_processor` types as that callable's own return type.
|
|
785
818
|
|
|
786
|
-
> **Passing no `exp`:** the default `validate()` post-processor
|
|
787
|
-
>
|
|
788
|
-
>
|
|
819
|
+
> **Passing no `exp`:** the default `validate()` post-processor treats a missing `exp` as "accept
|
|
820
|
+
> any type" -- it does not fail. Pass `exp=<type>` to add an isinstance check, or
|
|
821
|
+
> `post_processor=None` to bypass validation entirely and get the raw match.
|
|
789
822
|
|
|
790
823
|
**Example:**
|
|
791
824
|
|
|
@@ -820,6 +853,39 @@ name = dig(data, path=['response.users[0].display_name', 'response.users[0].name
|
|
|
820
853
|
# Returns: 'Alice'
|
|
821
854
|
```
|
|
822
855
|
|
|
856
|
+
##### `dig()` Behavior Matrix
|
|
857
|
+
|
|
858
|
+
Every row below is a real `dig()` call, backed by a passing unit test in
|
|
859
|
+
`test/unit/parsing/test_parsing.py` -- the test name is the proof, not just a claim. This isn't
|
|
860
|
+
every possible combination of kwargs (that cross product is huge and mostly redundant); it's one
|
|
861
|
+
representative call per distinct decision point in `dig()`/`validate()`. `*(unset)*` means the
|
|
862
|
+
kwarg wasn't passed (the function's own default applies); `--` means it isn't relevant to that row.
|
|
863
|
+
|
|
864
|
+
| `src` | `path` | `exp` | `default` | `post_processor` | `pattern` | `validator` | `pop` | Result | Proven by |
|
|
865
|
+
|---|---|---|---|---|---|---|---|---|---|
|
|
866
|
+
| `{'first': 'a', 'second': ['blah']}` | `'second[0]'` (found) | `str` | *(unset)* | *(unset)* | *(unset)* | *(unset)* | `False` | `'blah'` | `test_should_get_value_from_dict` |
|
|
867
|
+
| `['', ['blah']]` | `'[1][0]'` (found) | `str` | `''` | *(unset)* | *(unset)* | *(unset)* | `False` | `'blah'` (default unused -- value was valid) | `test_should_get_value_from_list` |
|
|
868
|
+
| `{'a': {'b': 'value'}}` | `'x.y.z'` (missing) | `str` | `'default'` | *(unset)* | *(unset)* | *(unset)* | `False` | `'default'` | `test_should_return_default_on_missing_intermediate_key` |
|
|
869
|
+
| `{'items': ['a', 'b', 'c']}` | `'items.invalid_index'` (missing) | `str` | `'default'` | *(unset)* | *(unset)* | *(unset)* | `False` | `'default'` | `test_should_return_default_when_accessing_list_with_string_key` |
|
|
870
|
+
| `{'a': {'b': 'value'}}` | `'x.y.z'` (missing) | `str` | *(unset)* | *(unset)* | *(unset)* | *(unset)* | `False` | `None` (no default set) | `test_should_return_none_for_missing_path_with_exp_and_no_default` |
|
|
871
|
+
| `{'a': {'b': 'c'}}` | `'x.y.z'` (missing) | *(unset)* | *(unset)* | *(unset)* | *(unset)* | *(unset)* | `False` | `None` (`exp=None` means "anything passes"; the found value already is `None`) | `test_should_handle_get_value_with_invalid_path` |
|
|
872
|
+
| `{'key': 'blah'}` | `'key'` (found) | *(unset)* | *(unset)* | *(unset)* | *(unset)* | *(unset)* | `False` | `'blah'` (raw pass-through, no isinstance check) | `test_should_not_crash_when_exp_omitted_from_dig` |
|
|
873
|
+
| `{'a': 'not-an-int'}` | `'a'` (found, wrong type) | `int` | *(unset)* | *(unset)* | *(unset)* | *(unset)* | `False` | `None` | `test_should_return_none_for_dig_result_type_mismatch` |
|
|
874
|
+
| `{'a': 'not-an-int'}` | `'a'` (found, wrong type) | `int` | `-1` | *(unset)* | *(unset)* | *(unset)* | `False` | `-1` | `test_should_return_default_for_dig_result_type_mismatch` |
|
|
875
|
+
| `{'a': 'blah'}` | `'a'` (found, non-type `exp`) | `'str'` | `'fallback'` | *(unset)* | *(unset)* | *(unset)* | `False` | `'fallback'` (garbage `exp` rejected, not raised) | `test_should_return_default_for_dig_with_non_type_expected_type` |
|
|
876
|
+
| `{'key': 'raw_value'}` | `'key'` (found) | -- | -- | `None` | -- | -- | `False` | raw value, unvalidated | `test_should_return_raw_value_when_no_post_processor` |
|
|
877
|
+
| `{'a': 1}` | `'missing'` (missing) | -- | `'default'` (ignored) | `None` | -- | -- | `False` | `None` (`default` is never consulted -- it's `validate()`'s kwarg, and `validate()` never runs) | `test_should_return_none_for_missing_path_with_no_post_processor` |
|
|
878
|
+
| `{'response': {'users': [{'name': 'Alice'}, {'name': 'Bob'}]}}` | `'response.users'` (found) | -- | -- | `len` | -- | -- | `False` | `2` (custom post_processor gets the raw match) | `test_should_use_custom_post_processor_function` |
|
|
879
|
+
| `'blah'` | `None` (src is the value) | `str` | *(unset)* | *(unset)* | *(unset)* | custom lambda | `False` | `'blah'` (custom validator overrides isinstance check) | `test_should_get_value_with_passed_validator` |
|
|
880
|
+
| `'test_string'` | `None` (src is the value) | `str` | *(unset)* | *(unset)* | *(unset)* | *(unset)* | `False` | the src value itself | `test_should_post_process_source_when_path_is_none` |
|
|
881
|
+
| `{'device': {'mac': 'AA:BB:CC:DD:EE:FF'}}` | `'device.mac'` (found) | `str` | *(unset)* | *(unset)* | matching regex | *(unset)* | `False` | matched MAC string | `test_should_validate_dig_result_with_matching_pattern` |
|
|
882
|
+
| `{'device': {'mac': 'not-a-mac'}}` | `'device.mac'` (found) | `str` | `'invalid'` | *(unset)* | non-matching regex | *(unset)* | `False` | `'invalid'` | `test_should_return_default_for_dig_result_with_non_matching_pattern` |
|
|
883
|
+
| `{'a': 1, 'b': 2}` | `'a'` (found) | `int` | *(unset)* | *(unset)* | -- | -- | `True` | `1`, and `'a'` removed from `src` | `test_should_pop_key_from_dict` |
|
|
884
|
+
| `{'items': [{'id': 1}, {'id': 2}]}` | `'items[?id==\`2\`].id'` (found, ambiguous) | `list` | *(unset)* | *(unset)* | -- | -- | `True` (refused) | `[2]`, `src` left unmutated (pop refused + warning logged) | `test_should_refuse_pop_with_ambiguous_jmespath_filter` |
|
|
885
|
+
| `{'user': {'name': 'Alice'}}` | `['user.nickname', 'user.name']` (fallback) | `str` | *(unset)* | *(unset)* | -- | -- | `False` | `'Alice'` (first candidate wins) | `test_should_use_first_matching_path_in_fallback_list` |
|
|
886
|
+
| `{'user': {'name': 'Alice'}}` | `['user.nickname', 'user.alias']` (fallback, none match) | `str` | *(unset)* | *(unset)* | -- | -- | `False` | `None` | `test_should_return_none_when_no_fallback_path_matches` |
|
|
887
|
+
| `{'user': {'name': 'Alice', 'nickname': 'Ali'}}` | `['user.nickname', 'user.name']` (fallback) | `str` | *(unset)* | *(unset)* | -- | -- | `True` | `'Ali'`, only the winning candidate's key removed | `test_should_pop_only_the_winning_fallback_path` |
|
|
888
|
+
|
|
823
889
|
#### `dig_many(src, paths, **common_kwargs)`
|
|
824
890
|
|
|
825
891
|
Extract several named fields from `src` in one call.
|
|
@@ -854,7 +920,9 @@ fields = dig_many(
|
|
|
854
920
|
|
|
855
921
|
A pre-bound, reusable `dig()` call. Validates/compiles the jmespath expression once at
|
|
856
922
|
construction (failing fast on a bad expression instead of on first use), then can be called
|
|
857
|
-
like a function against any number of `src` objects without repeating `path`/`exp`/kwargs.
|
|
923
|
+
like a function against any number of `src` objects without repeating `path`/`exp`/kwargs. A
|
|
924
|
+
string `pattern=` kwarg is likewise pre-compiled to a `re.Pattern` at construction time, so
|
|
925
|
+
repeated calls don't re-compile the regex.
|
|
858
926
|
|
|
859
927
|
**Example:**
|
|
860
928
|
|
|
@@ -48,13 +48,18 @@ def detect_unicode_support() -> bool:
|
|
|
48
48
|
"""Detect if terminal supports Unicode characters.
|
|
49
49
|
|
|
50
50
|
Checks environment variables and terminal type to determine if the terminal
|
|
51
|
-
can display Unicode characters (for pretty borders, etc.)
|
|
51
|
+
can display Unicode characters (for pretty borders, etc.), then verifies
|
|
52
|
+
sys.stdout's actual encoding can render them.
|
|
52
53
|
|
|
53
54
|
Environment checks:
|
|
54
55
|
- NO_COLOR: If set, disables colors/Unicode for accessibility
|
|
55
56
|
- TERM: Terminal type (vt100, xterm, etc.)
|
|
56
57
|
- CI: If in CI environment, assume Unicode support
|
|
57
|
-
-
|
|
58
|
+
- stdout encoding: Windows consoles commonly default to a codepage (cp1252,
|
|
59
|
+
cp437, etc.) that can't encode box-drawing characters even when the
|
|
60
|
+
platform/terminal is otherwise capable -- this is checked directly rather
|
|
61
|
+
than assumed from sys.platform, since PYTHONIOENCODING/chcp/Windows
|
|
62
|
+
Terminal all change what's actually possible independent of the OS.
|
|
58
63
|
|
|
59
64
|
Returns:
|
|
60
65
|
True if terminal likely supports Unicode, False otherwise.
|
|
@@ -78,20 +83,19 @@ def detect_unicode_support() -> bool:
|
|
|
78
83
|
if term in ("dumb", "vt100", "vt220"):
|
|
79
84
|
return False
|
|
80
85
|
|
|
81
|
-
#
|
|
82
|
-
#
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
+
# Verify stdout can actually encode a representative box-drawing character.
|
|
87
|
+
# Windows consoles commonly default to a codepage (cp1252, cp437, etc.) that
|
|
88
|
+
# can't, even when the platform/terminal is otherwise Unicode-capable.
|
|
89
|
+
try:
|
|
90
|
+
"─".encode(sys.stdout.encoding)
|
|
86
91
|
return True
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
92
|
+
except UnicodeEncodeError:
|
|
93
|
+
return False
|
|
94
|
+
except (AttributeError, TypeError, LookupError):
|
|
95
|
+
# stdout has no usable encoding (e.g. a mock/capture stream in tests) --
|
|
96
|
+
# can't verify either way, so don't penalize it.
|
|
90
97
|
return True
|
|
91
98
|
|
|
92
|
-
# Default: attempt Unicode
|
|
93
|
-
return True
|
|
94
|
-
|
|
95
99
|
|
|
96
100
|
def select_style(
|
|
97
101
|
style: str,
|
|
@@ -6,7 +6,10 @@ from typing import Callable
|
|
|
6
6
|
|
|
7
7
|
# pylint: disable=invalid-name
|
|
8
8
|
DEFAULT_CONVERTER: Callable = lambda value, *args, **kwargs: value
|
|
9
|
-
DEFAULT_VALIDATOR: Callable = lambda value, expected_type, *args, **kwargs:
|
|
9
|
+
DEFAULT_VALIDATOR: Callable = lambda value, expected_type, *args, **kwargs: (
|
|
10
|
+
expected_type is None
|
|
11
|
+
or (isinstance(expected_type, (type, tuple)) and isinstance(value, expected_type))
|
|
12
|
+
)
|
|
10
13
|
DEFAULT_SEPARATOR: str = ','
|
|
11
14
|
|
|
12
15
|
|
|
@@ -104,12 +104,13 @@ def get_datetime(
|
|
|
104
104
|
return result
|
|
105
105
|
|
|
106
106
|
|
|
107
|
-
def validate(
|
|
107
|
+
def validate( # pylint: disable=too-many-arguments,too-many-positional-arguments
|
|
108
108
|
v: Any,
|
|
109
109
|
exp: Type[T] = None,
|
|
110
110
|
default: T | None = None,
|
|
111
111
|
converter: Callable = None,
|
|
112
112
|
validator: Callable | None = DEFAULT_VALIDATOR,
|
|
113
|
+
pattern: str | re.Pattern | None = None,
|
|
113
114
|
**kwargs
|
|
114
115
|
) -> T | None:
|
|
115
116
|
"""Validate and optionally convert a value with a custom converter and validator.
|
|
@@ -117,11 +118,13 @@ def validate(
|
|
|
117
118
|
This is a generic validation utility that:
|
|
118
119
|
1. Converts the value using a converter function (with automatic selection for common types)
|
|
119
120
|
2. Validates the result using a validator function
|
|
120
|
-
3.
|
|
121
|
+
3. Optionally checks the result against a regex pattern (str results only)
|
|
122
|
+
4. Returns the result if valid, otherwise returns the default value
|
|
121
123
|
|
|
122
124
|
Args:
|
|
123
125
|
v: The value to validate.
|
|
124
|
-
exp: Expected type of the value. If None, no type conversion is
|
|
126
|
+
exp: Expected type of the value. If None, no type conversion or isinstance check is
|
|
127
|
+
attempted (DEFAULT_VALIDATOR passes any value through unchanged).
|
|
125
128
|
default: The value to return if validation fails or v is None. Default: None.
|
|
126
129
|
converter: Custom converter function. If None, one is selected based on exp:
|
|
127
130
|
- exp=list: uses str_to_list
|
|
@@ -129,6 +132,9 @@ def validate(
|
|
|
129
132
|
- otherwise: uses DEFAULT_CONVERTER (pass-through)
|
|
130
133
|
validator: Custom validator function(result, exp, **kwargs) -> bool.
|
|
131
134
|
If None, uses DEFAULT_VALIDATOR (isinstance check).
|
|
135
|
+
pattern: Optional regex (string or compiled re.Pattern) the result must fully match
|
|
136
|
+
via re.fullmatch. Only applied when the (converted, validated) result is a str;
|
|
137
|
+
checked after conversion/validation succeed. On mismatch, returns default.
|
|
132
138
|
**kwargs: Additional arguments passed to converter and validator functions.
|
|
133
139
|
|
|
134
140
|
Returns:
|
|
@@ -142,6 +148,8 @@ def validate(
|
|
|
142
148
|
0
|
|
143
149
|
>>> validate('a,b,c', exp=list)
|
|
144
150
|
['a', 'b', 'c']
|
|
151
|
+
>>> validate('AA:BB:CC:DD:EE:FF', exp=str, pattern=r'([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}')
|
|
152
|
+
'AA:BB:CC:DD:EE:FF'
|
|
145
153
|
"""
|
|
146
154
|
result: Any = v
|
|
147
155
|
is_valid_result: bool = False
|
|
@@ -173,6 +181,11 @@ def validate(
|
|
|
173
181
|
except Exception as exc_validation:
|
|
174
182
|
logger.debug(f'Failed validation: {validator=}: {exc_validation=}', exc_info=True)
|
|
175
183
|
|
|
184
|
+
if is_valid_result and pattern is not None and isinstance(result, str):
|
|
185
|
+
compiled_pattern = pattern if isinstance(pattern, re.Pattern) else re.compile(pattern)
|
|
186
|
+
if not compiled_pattern.fullmatch(result):
|
|
187
|
+
is_valid_result = False
|
|
188
|
+
|
|
176
189
|
return result if is_valid_result else default
|
|
177
190
|
|
|
178
191
|
|
|
@@ -192,7 +205,13 @@ _JMESPATH_TERMINAL_RE = re.compile(
|
|
|
192
205
|
|
|
193
206
|
@lru_cache(maxsize=256)
|
|
194
207
|
def _compile_jmespath(expr: str) -> Any:
|
|
195
|
-
"""Compile (and cache) a jmespath expression string.
|
|
208
|
+
"""Compile (and cache) a jmespath expression string.
|
|
209
|
+
|
|
210
|
+
This cache is process-global and shared by every dig()/Digger call, keyed on the
|
|
211
|
+
expression string -- a bare dig() call already avoids recompiling a repeated path, not
|
|
212
|
+
just Digger. Capped at 256 distinct expressions (LRU-evicted); Digger's up-front compile
|
|
213
|
+
call at construction populates this same cache rather than maintaining its own.
|
|
214
|
+
"""
|
|
196
215
|
return jmespath.compile(expr)
|
|
197
216
|
|
|
198
217
|
|
|
@@ -337,10 +356,13 @@ def dig(
|
|
|
337
356
|
else:
|
|
338
357
|
result = _dig_expr(src, _path_to_expr(path), pop)
|
|
339
358
|
|
|
340
|
-
#
|
|
341
|
-
|
|
359
|
+
# Always run the post-processor -- even on a missing/None result -- so it can apply
|
|
360
|
+
# its own default handling (validate() already treats v=None as failing an isinstance
|
|
361
|
+
# check and falls back to kwargs['default']). Skipping post_processor entirely is
|
|
362
|
+
# reserved for the explicit post_processor=None "give me the raw value" case.
|
|
363
|
+
if callable(post_processor):
|
|
342
364
|
return post_processor(result, **kwargs)
|
|
343
|
-
return result
|
|
365
|
+
return result
|
|
344
366
|
|
|
345
367
|
|
|
346
368
|
def dig_many(
|
|
@@ -382,7 +404,8 @@ class Digger:
|
|
|
382
404
|
|
|
383
405
|
Validates/compiles the path up front (failing fast on a bad jmespath expression
|
|
384
406
|
instead of on first use) and avoids re-passing the same path/exp/post_processor/kwargs
|
|
385
|
-
on every call.
|
|
407
|
+
on every call. A string `pattern=` kwarg is likewise pre-compiled to a re.Pattern here
|
|
408
|
+
so repeated calls don't re-compile the regex on every invocation.
|
|
386
409
|
|
|
387
410
|
Example:
|
|
388
411
|
>>> get_name = Digger(path='user.name', exp=str, default='')
|
|
@@ -399,6 +422,8 @@ class Digger:
|
|
|
399
422
|
self.path = path
|
|
400
423
|
self.pop = pop
|
|
401
424
|
self.post_processor = post_processor
|
|
425
|
+
if isinstance(kwargs.get('pattern'), str):
|
|
426
|
+
kwargs['pattern'] = re.compile(kwargs['pattern'])
|
|
402
427
|
self.kwargs = kwargs
|
|
403
428
|
for candidate in (path if isinstance(path, (list, tuple)) else [path]):
|
|
404
429
|
if candidate is not None:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: oj-toolkit
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Centralized shared utilities for ownjoo-org projects: standardized logging config, type validation, data parsing, console output, and async utilities
|
|
5
5
|
Author-email: Speedy <speedy@ownjoo.org>
|
|
6
6
|
License: MIT
|
|
@@ -33,7 +33,7 @@ Requires-Dist: mypy>=1.0; extra == "dev"
|
|
|
33
33
|
Requires-Dist: pylint>=2.0; extra == "dev"
|
|
34
34
|
Dynamic: license-file
|
|
35
35
|
|
|
36
|
-
#
|
|
36
|
+
# oj-toolkit
|
|
37
37
|
|
|
38
38
|
Centralized shared utilities library for all ownjoo-org projects. Provides battle-tested functions for type validation, data parsing, and progress logging.
|
|
39
39
|
|
|
@@ -187,11 +187,19 @@ fields = dig_many(
|
|
|
187
187
|
# Bind a path once, reuse it against many objects
|
|
188
188
|
get_name = Digger(path='name', exp=str, default='')
|
|
189
189
|
[get_name(user) for user in data['users']] # Returns: ['Alice', 'Bob']
|
|
190
|
+
|
|
191
|
+
# Constrain a string result with a regex (checked via re.fullmatch after conversion)
|
|
192
|
+
mac = dig({'device': {'mac': 'AA:BB:CC:DD:EE:FF'}}, path='device.mac', exp=str,
|
|
193
|
+
pattern=r'([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}')
|
|
194
|
+
# Returns: 'AA:BB:CC:DD:EE:FF'
|
|
190
195
|
```
|
|
191
196
|
|
|
192
|
-
> **Note:**
|
|
193
|
-
> `
|
|
194
|
-
>
|
|
197
|
+
> **Note:** if the path resolves to nothing (missing key/index, or `src` doesn't contain it),
|
|
198
|
+
> `dig()`'s default `post_processor` (`validate()`) still runs and returns `default` (`None`
|
|
199
|
+
> unless set) -- a missing path never silently drops `default` in favor of a bare `None`.
|
|
200
|
+
> Passing no `exp=` means "accept any type, no isinstance check" -- not "always fail". Pass
|
|
201
|
+
> `exp=<type>` to add a type check, or `post_processor=None` to bypass validation entirely and
|
|
202
|
+
> get the raw match untouched.
|
|
195
203
|
|
|
196
204
|
### Logging Setup
|
|
197
205
|
|
|
@@ -711,18 +719,25 @@ else:
|
|
|
711
719
|
|
|
712
720
|
Returns `True` if the terminal likely supports Unicode characters. Used internally to choose between ASCII and Unicode box/table borders.
|
|
713
721
|
|
|
722
|
+
Checks `NO_COLOR`/`CI`/`TERM` env vars first, then verifies `sys.stdout.encoding` can actually
|
|
723
|
+
encode a box-drawing character -- this catches Windows consoles that default to a codepage
|
|
724
|
+
(`cp1252`, `cp437`, etc.) that can't, even though the platform/terminal is otherwise
|
|
725
|
+
Unicode-capable. Set `PYTHONIOENCODING=utf-8` (or use Windows Terminal with a UTF-8 codepage)
|
|
726
|
+
to get Unicode borders on Windows instead of the ASCII fallback.
|
|
727
|
+
|
|
714
728
|
### `parsing` Module
|
|
715
729
|
|
|
716
|
-
#### `validate(v, exp=None, default=None, converter=None, validator=None, **kwargs)`
|
|
730
|
+
#### `validate(v, exp=None, default=None, converter=None, validator=None, pattern=None, **kwargs)`
|
|
717
731
|
|
|
718
732
|
Generic validation utility that converts and validates a value.
|
|
719
733
|
|
|
720
734
|
- **Parameters:**
|
|
721
735
|
- `v` (Any): The value to validate
|
|
722
|
-
- `exp` (Type): Expected type. Enables automatic converter selection (list → str_to_list, datetime → get_datetime)
|
|
723
|
-
- `default` (Any): Return this if validation fails
|
|
724
|
-
- `converter` (Callable): Custom converter function. Default: auto-selected based on exp
|
|
725
|
-
- `validator` (Callable): Custom validator function(result, exp, **kwargs) → bool. Default: isinstance check
|
|
736
|
+
- `exp` (Type): Expected type. Enables automatic converter selection (list → str_to_list, datetime → get_datetime). If `None`, no isinstance check is performed -- any type passes (see `validator` below)
|
|
737
|
+
- `default` (Any): Return this if validation fails, or `v` is missing/`None`. Default: None
|
|
738
|
+
- `converter` (Callable): Custom converter function. Default: auto-selected based on exp. Auto-selection is limited to `list`/`datetime` -- `bool`/`int`/`float`/etc. are pass-through unless you supply your own `converter=`
|
|
739
|
+
- `validator` (Callable): Custom validator function(result, exp, **kwargs) → bool. Default: isinstance check (`exp=None` always passes; a non-type `exp` is rejected rather than raising)
|
|
740
|
+
- `pattern` (str | re.Pattern): Optional regex checked via `re.fullmatch` against the result, but only when the result is a `str` -- ignored for other types. Applied after conversion/validation succeed; a mismatch falls back to `default`
|
|
726
741
|
- `**kwargs`: Passed to converter and validator
|
|
727
742
|
|
|
728
743
|
- **Returns:** Converted and validated value, or default if validation fails
|
|
@@ -734,6 +749,16 @@ Generic validation utility that converts and validates a value.
|
|
|
734
749
|
numbers = validate('1,2,3', exp=list) # Returns: ['1', '2', '3']
|
|
735
750
|
dt = validate('2024-01-15T10:30:00', exp=datetime) # Returns: datetime object
|
|
736
751
|
|
|
752
|
+
# exp=bool/int/float do NOT auto-coerce strings -- pass converter= explicitly for that
|
|
753
|
+
validate('42', exp=int) # Returns: None ('42' is a str, not an int)
|
|
754
|
+
validate('42', exp=int, converter=int) # Returns: 42
|
|
755
|
+
|
|
756
|
+
# Regex-constrain a string result
|
|
757
|
+
validate('AA:BB:CC:DD:EE:FF', exp=str, pattern=r'([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}')
|
|
758
|
+
# Returns: 'AA:BB:CC:DD:EE:FF'
|
|
759
|
+
validate('not-a-mac', exp=str, pattern=r'([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}', default='invalid')
|
|
760
|
+
# Returns: 'invalid'
|
|
761
|
+
|
|
737
762
|
# Custom converter and validator
|
|
738
763
|
def to_upper(v, *args, **kwargs):
|
|
739
764
|
return str(v).upper()
|
|
@@ -795,6 +820,12 @@ get_datetime('01/15/2024 10:30:00', format_str='%m/%d/%Y %H:%M:%S') # Custom
|
|
|
795
820
|
Extract and post-process a value from a nested data structure, using [jmespath](https://jmespath.org/)
|
|
796
821
|
under the hood (`jmespath` is a required dependency).
|
|
797
822
|
|
|
823
|
+
> **Compilation caching:** compiled jmespath expressions are cached in a process-global
|
|
824
|
+
> `lru_cache(maxsize=256)`, keyed by the expression string. A bare `dig()` call already avoids
|
|
825
|
+
> recompiling a repeated path -- this isn't a `Digger`-only optimization. The cache holds up to
|
|
826
|
+
> 256 distinct expressions across the whole process (shared by every `dig()`/`Digger` caller),
|
|
827
|
+
> LRU-evicted beyond that.
|
|
828
|
+
|
|
798
829
|
- **Parameters:**
|
|
799
830
|
- `src` (Mapping | Sequence): Data structure to navigate
|
|
800
831
|
- `path` (int | str | list | None):
|
|
@@ -811,16 +842,18 @@ under the hood (`jmespath` is a required dependency).
|
|
|
811
842
|
Pass `None` to skip post-processing and get the raw match.
|
|
812
843
|
- `**kwargs`: Passed to `post_processor`
|
|
813
844
|
|
|
814
|
-
- **Returns:** Post-processed value
|
|
815
|
-
|
|
845
|
+
- **Returns:** Post-processed value. If extraction fails or no candidate path matches, the
|
|
846
|
+
post-processor (default `validate()`) still runs and returns `default` (`None` unless set) --
|
|
847
|
+
unless `post_processor=None`, in which case the raw (possibly missing/`None`) value is returned
|
|
848
|
+
untouched with no default substitution.
|
|
816
849
|
|
|
817
850
|
> **Type checkers:** pass `exp=<type>` to get a precise `<type> | None` return type instead of `Any`
|
|
818
851
|
> (`dig(data, path='...', exp=str)` type-checks as `str | None`). Passing `post_processor=None` types
|
|
819
852
|
> as `Any` (raw, unvalidated value). A custom `post_processor` types as that callable's own return type.
|
|
820
853
|
|
|
821
|
-
> **Passing no `exp`:** the default `validate()` post-processor
|
|
822
|
-
>
|
|
823
|
-
>
|
|
854
|
+
> **Passing no `exp`:** the default `validate()` post-processor treats a missing `exp` as "accept
|
|
855
|
+
> any type" -- it does not fail. Pass `exp=<type>` to add an isinstance check, or
|
|
856
|
+
> `post_processor=None` to bypass validation entirely and get the raw match.
|
|
824
857
|
|
|
825
858
|
**Example:**
|
|
826
859
|
|
|
@@ -855,6 +888,39 @@ name = dig(data, path=['response.users[0].display_name', 'response.users[0].name
|
|
|
855
888
|
# Returns: 'Alice'
|
|
856
889
|
```
|
|
857
890
|
|
|
891
|
+
##### `dig()` Behavior Matrix
|
|
892
|
+
|
|
893
|
+
Every row below is a real `dig()` call, backed by a passing unit test in
|
|
894
|
+
`test/unit/parsing/test_parsing.py` -- the test name is the proof, not just a claim. This isn't
|
|
895
|
+
every possible combination of kwargs (that cross product is huge and mostly redundant); it's one
|
|
896
|
+
representative call per distinct decision point in `dig()`/`validate()`. `*(unset)*` means the
|
|
897
|
+
kwarg wasn't passed (the function's own default applies); `--` means it isn't relevant to that row.
|
|
898
|
+
|
|
899
|
+
| `src` | `path` | `exp` | `default` | `post_processor` | `pattern` | `validator` | `pop` | Result | Proven by |
|
|
900
|
+
|---|---|---|---|---|---|---|---|---|---|
|
|
901
|
+
| `{'first': 'a', 'second': ['blah']}` | `'second[0]'` (found) | `str` | *(unset)* | *(unset)* | *(unset)* | *(unset)* | `False` | `'blah'` | `test_should_get_value_from_dict` |
|
|
902
|
+
| `['', ['blah']]` | `'[1][0]'` (found) | `str` | `''` | *(unset)* | *(unset)* | *(unset)* | `False` | `'blah'` (default unused -- value was valid) | `test_should_get_value_from_list` |
|
|
903
|
+
| `{'a': {'b': 'value'}}` | `'x.y.z'` (missing) | `str` | `'default'` | *(unset)* | *(unset)* | *(unset)* | `False` | `'default'` | `test_should_return_default_on_missing_intermediate_key` |
|
|
904
|
+
| `{'items': ['a', 'b', 'c']}` | `'items.invalid_index'` (missing) | `str` | `'default'` | *(unset)* | *(unset)* | *(unset)* | `False` | `'default'` | `test_should_return_default_when_accessing_list_with_string_key` |
|
|
905
|
+
| `{'a': {'b': 'value'}}` | `'x.y.z'` (missing) | `str` | *(unset)* | *(unset)* | *(unset)* | *(unset)* | `False` | `None` (no default set) | `test_should_return_none_for_missing_path_with_exp_and_no_default` |
|
|
906
|
+
| `{'a': {'b': 'c'}}` | `'x.y.z'` (missing) | *(unset)* | *(unset)* | *(unset)* | *(unset)* | *(unset)* | `False` | `None` (`exp=None` means "anything passes"; the found value already is `None`) | `test_should_handle_get_value_with_invalid_path` |
|
|
907
|
+
| `{'key': 'blah'}` | `'key'` (found) | *(unset)* | *(unset)* | *(unset)* | *(unset)* | *(unset)* | `False` | `'blah'` (raw pass-through, no isinstance check) | `test_should_not_crash_when_exp_omitted_from_dig` |
|
|
908
|
+
| `{'a': 'not-an-int'}` | `'a'` (found, wrong type) | `int` | *(unset)* | *(unset)* | *(unset)* | *(unset)* | `False` | `None` | `test_should_return_none_for_dig_result_type_mismatch` |
|
|
909
|
+
| `{'a': 'not-an-int'}` | `'a'` (found, wrong type) | `int` | `-1` | *(unset)* | *(unset)* | *(unset)* | `False` | `-1` | `test_should_return_default_for_dig_result_type_mismatch` |
|
|
910
|
+
| `{'a': 'blah'}` | `'a'` (found, non-type `exp`) | `'str'` | `'fallback'` | *(unset)* | *(unset)* | *(unset)* | `False` | `'fallback'` (garbage `exp` rejected, not raised) | `test_should_return_default_for_dig_with_non_type_expected_type` |
|
|
911
|
+
| `{'key': 'raw_value'}` | `'key'` (found) | -- | -- | `None` | -- | -- | `False` | raw value, unvalidated | `test_should_return_raw_value_when_no_post_processor` |
|
|
912
|
+
| `{'a': 1}` | `'missing'` (missing) | -- | `'default'` (ignored) | `None` | -- | -- | `False` | `None` (`default` is never consulted -- it's `validate()`'s kwarg, and `validate()` never runs) | `test_should_return_none_for_missing_path_with_no_post_processor` |
|
|
913
|
+
| `{'response': {'users': [{'name': 'Alice'}, {'name': 'Bob'}]}}` | `'response.users'` (found) | -- | -- | `len` | -- | -- | `False` | `2` (custom post_processor gets the raw match) | `test_should_use_custom_post_processor_function` |
|
|
914
|
+
| `'blah'` | `None` (src is the value) | `str` | *(unset)* | *(unset)* | *(unset)* | custom lambda | `False` | `'blah'` (custom validator overrides isinstance check) | `test_should_get_value_with_passed_validator` |
|
|
915
|
+
| `'test_string'` | `None` (src is the value) | `str` | *(unset)* | *(unset)* | *(unset)* | *(unset)* | `False` | the src value itself | `test_should_post_process_source_when_path_is_none` |
|
|
916
|
+
| `{'device': {'mac': 'AA:BB:CC:DD:EE:FF'}}` | `'device.mac'` (found) | `str` | *(unset)* | *(unset)* | matching regex | *(unset)* | `False` | matched MAC string | `test_should_validate_dig_result_with_matching_pattern` |
|
|
917
|
+
| `{'device': {'mac': 'not-a-mac'}}` | `'device.mac'` (found) | `str` | `'invalid'` | *(unset)* | non-matching regex | *(unset)* | `False` | `'invalid'` | `test_should_return_default_for_dig_result_with_non_matching_pattern` |
|
|
918
|
+
| `{'a': 1, 'b': 2}` | `'a'` (found) | `int` | *(unset)* | *(unset)* | -- | -- | `True` | `1`, and `'a'` removed from `src` | `test_should_pop_key_from_dict` |
|
|
919
|
+
| `{'items': [{'id': 1}, {'id': 2}]}` | `'items[?id==\`2\`].id'` (found, ambiguous) | `list` | *(unset)* | *(unset)* | -- | -- | `True` (refused) | `[2]`, `src` left unmutated (pop refused + warning logged) | `test_should_refuse_pop_with_ambiguous_jmespath_filter` |
|
|
920
|
+
| `{'user': {'name': 'Alice'}}` | `['user.nickname', 'user.name']` (fallback) | `str` | *(unset)* | *(unset)* | -- | -- | `False` | `'Alice'` (first candidate wins) | `test_should_use_first_matching_path_in_fallback_list` |
|
|
921
|
+
| `{'user': {'name': 'Alice'}}` | `['user.nickname', 'user.alias']` (fallback, none match) | `str` | *(unset)* | *(unset)* | -- | -- | `False` | `None` | `test_should_return_none_when_no_fallback_path_matches` |
|
|
922
|
+
| `{'user': {'name': 'Alice', 'nickname': 'Ali'}}` | `['user.nickname', 'user.name']` (fallback) | `str` | *(unset)* | *(unset)* | -- | -- | `True` | `'Ali'`, only the winning candidate's key removed | `test_should_pop_only_the_winning_fallback_path` |
|
|
923
|
+
|
|
858
924
|
#### `dig_many(src, paths, **common_kwargs)`
|
|
859
925
|
|
|
860
926
|
Extract several named fields from `src` in one call.
|
|
@@ -889,7 +955,9 @@ fields = dig_many(
|
|
|
889
955
|
|
|
890
956
|
A pre-bound, reusable `dig()` call. Validates/compiles the jmespath expression once at
|
|
891
957
|
construction (failing fast on a bad expression instead of on first use), then can be called
|
|
892
|
-
like a function against any number of `src` objects without repeating `path`/`exp`/kwargs.
|
|
958
|
+
like a function against any number of `src` objects without repeating `path`/`exp`/kwargs. A
|
|
959
|
+
string `pattern=` kwarg is likewise pre-compiled to a `re.Pattern` at construction time, so
|
|
960
|
+
repeated calls don't re-compile the regex.
|
|
893
961
|
|
|
894
962
|
**Example:**
|
|
895
963
|
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "oj-toolkit"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.2"
|
|
8
8
|
description = "Centralized shared utilities for ownjoo-org projects: standardized logging config, type validation, data parsing, console output, and async utilities"
|
|
9
9
|
authors = [
|
|
10
10
|
{ name = "Speedy", email = "speedy@ownjoo.org" }
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|