tide2 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tide2/__init__.py +12 -0
- tide2/_version.py +24 -0
- tide2/actors/__init__.py +63 -0
- tide2/actors/anonymizer.py +808 -0
- tide2/actors/llm_recognizer.py +425 -0
- tide2/actors/reassembly.py +98 -0
- tide2/actors/recognizer.py +605 -0
- tide2/actors/transformer.py +549 -0
- tide2/anonymizers/__init__.py +39 -0
- tide2/anonymizers/accession_number_hash.py +129 -0
- tide2/anonymizers/age_grouping.py +334 -0
- tide2/anonymizers/date_jitter.py +476 -0
- tide2/anonymizers/faker.py +192 -0
- tide2/anonymizers/hips_alphanumeric.py +121 -0
- tide2/anonymizers/hips_locations.py +318 -0
- tide2/anonymizers/hips_names.py +190 -0
- tide2/anonymizers/masking.py +40 -0
- tide2/anonymizers/presidio_patches.py +209 -0
- tide2/cli/__init__.py +21 -0
- tide2/cli/main_visualizer.py +36 -0
- tide2/cli/unified_interface.py +1032 -0
- tide2/cryptographic/__init__.py +44 -0
- tide2/cryptographic/date_jitter.py +218 -0
- tide2/cryptographic/fpe_strings.py +394 -0
- tide2/cryptographic/keys_utils.py +208 -0
- tide2/cryptographic/string_selector.py +105 -0
- tide2/recognizers/__init__.py +86 -0
- tide2/recognizers/accession_recognizer.py +160 -0
- tide2/recognizers/address_recognizer.py +291 -0
- tide2/recognizers/base64_image_recognizer.py +144 -0
- tide2/recognizers/cached_results_transformers_recognizer.py +216 -0
- tide2/recognizers/email_recognizer.py +104 -0
- tide2/recognizers/genetic_sequence_recognizer.py +183 -0
- tide2/recognizers/har_recognizer.py +55 -0
- tide2/recognizers/known_values.py +646 -0
- tide2/recognizers/llm_json_recognizer.py +553 -0
- tide2/recognizers/mrn_recognizer.py +78 -0
- tide2/recognizers/passthrough_recognizer.py +77 -0
- tide2/recognizers/phone_recognizer.py +130 -0
- tide2/recognizers/ssn_recognizer.py +215 -0
- tide2/recognizers/transformers_recognizer.py +338 -0
- tide2/recognizers/url_recognizer.py +173 -0
- tide2/resources/__init__.py +0 -0
- tide2/resources/bert_transformer_configuration.json +247 -0
- tide2/resources/cities.tsv +43912 -0
- tide2/resources/countries.tsv +243 -0
- tide2/resources/first_names_with_sex.tsv +34317 -0
- tide2/resources/hospitals.txt +567 -0
- tide2/resources/llm_prompts/__init__.py +0 -0
- tide2/resources/llm_prompts/phi_detection.json +4 -0
- tide2/resources/llm_prompts/phi_detection.txt +233 -0
- tide2/resources/runner_config_example.yaml +96 -0
- tide2/resources/states.tsv +52 -0
- tide2/resources/stopwords.txt +418 -0
- tide2/resources/street_names.tsv +51543 -0
- tide2/resources/surnames.tsv +260174 -0
- tide2/resources/unified_names.tsv +2276 -0
- tide2/resources/zipcodes.tsv +33121 -0
- tide2/runner/__init__.py +62 -0
- tide2/runner/cli.py +526 -0
- tide2/runner/fault_tolerance.py +280 -0
- tide2/runner/local_runner.py +1727 -0
- tide2/runner/transformer.py +280 -0
- tide2/runner/utils.py +166 -0
- tide2/string_parsers/__init__.py +34 -0
- tide2/string_parsers/address_parsers.py +519 -0
- tide2/string_parsers/format_detector.py +745 -0
- tide2/string_parsers/name_parsers.py +297 -0
- tide2/string_parsers/name_tokenizer.py +214 -0
- tide2/transformers/__init__.py +32 -0
- tide2/transformers/config.py +57 -0
- tide2/transformers/core.py +546 -0
- tide2/utils/__init__.py +13 -0
- tide2/utils/batch_columns.py +42 -0
- tide2/utils/constants.py +219 -0
- tide2/utils/gcs_connector.py +826 -0
- tide2/utils/gcs_resource_manager.py +394 -0
- tide2/utils/llm_model.py +491 -0
- tide2/utils/resource_utils.py +219 -0
- tide2/utils/serialization.py +222 -0
- tide2/utils/span_metrics.py +1165 -0
- tide2/utils/text_processing.py +652 -0
- tide2-1.1.0.dist-info/METADATA +432 -0
- tide2-1.1.0.dist-info/RECORD +87 -0
- tide2-1.1.0.dist-info/WHEEL +4 -0
- tide2-1.1.0.dist-info/entry_points.txt +3 -0
- tide2-1.1.0.dist-info/licenses/LICENSE-MIT +21 -0
tide2/__init__.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""
|
|
2
|
+
.. include:: ../../README.md
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
try:
|
|
6
|
+
# Written by hatch-vcs at build time from the git tag (see pyproject.toml).
|
|
7
|
+
from tide2._version import __version__
|
|
8
|
+
except ImportError: # pragma: no cover - source checkout without a build
|
|
9
|
+
__version__ = "0.0.0.dev0"
|
|
10
|
+
|
|
11
|
+
__author__ = "TiDE-CORE 2.0 Team"
|
|
12
|
+
__docformat__ = "google"
|
tide2/_version.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '1.1.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (1, 1, 0)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = None
|
tide2/actors/__init__.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Unified Ray actors for batch processing.
|
|
3
|
+
|
|
4
|
+
This module provides Ray actors for recognition, anonymization, and transformer
|
|
5
|
+
inference that work across all execution modes: local, VM, and cluster.
|
|
6
|
+
|
|
7
|
+
Actors:
|
|
8
|
+
RecognizerActor: PII/PHI recognition using Presidio AnalyzerEngine
|
|
9
|
+
AnonymizerActor: Anonymization using Presidio AnonymizerEngine with HIPS
|
|
10
|
+
TransformerInferenceActor: GPU-based transformer NER inference
|
|
11
|
+
|
|
12
|
+
Factory Functions:
|
|
13
|
+
create_anonymizer_actor: Create AnonymizerActor with keys (bytes or file paths)
|
|
14
|
+
create_transformer_actor: Create TransformerInferenceActor with model config
|
|
15
|
+
|
|
16
|
+
Example:
|
|
17
|
+
from tide2.actors import RecognizerActor, create_anonymizer_actor
|
|
18
|
+
|
|
19
|
+
# Use directly with Ray Data
|
|
20
|
+
ds.map_batches(RecognizerActor, batch_size=100, ...)
|
|
21
|
+
|
|
22
|
+
# Create configured actor with factory
|
|
23
|
+
AnonymizerActorClass = create_anonymizer_actor("/path/to/private.key", "/path/to/public.key")
|
|
24
|
+
ds.map_batches(AnonymizerActorClass, batch_size=100, ...)
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from tide2.actors.anonymizer import AnonymizerActor
|
|
28
|
+
from tide2.actors.anonymizer import create_anonymizer_actor
|
|
29
|
+
from tide2.actors.anonymizer import create_anonymizer_actor_class # Backwards compatibility
|
|
30
|
+
from tide2.actors.llm_recognizer import LlmRecognizerActor
|
|
31
|
+
from tide2.actors.reassembly import ReassemblyActor
|
|
32
|
+
from tide2.actors.recognizer import NoOpContextEnhancer
|
|
33
|
+
from tide2.actors.recognizer import RecognizerActor
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def __getattr__(name: str):
|
|
37
|
+
"""Lazy import for torch-dependent transformer actors."""
|
|
38
|
+
_transformer_exports = {
|
|
39
|
+
"BIOAggregationActor",
|
|
40
|
+
"TransformerInferenceActor",
|
|
41
|
+
"create_transformer_actor",
|
|
42
|
+
"create_transformer_actor_class",
|
|
43
|
+
}
|
|
44
|
+
if name in _transformer_exports:
|
|
45
|
+
from tide2.actors import transformer as _mod
|
|
46
|
+
|
|
47
|
+
return getattr(_mod, name)
|
|
48
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
__all__ = [
|
|
52
|
+
"AnonymizerActor",
|
|
53
|
+
"BIOAggregationActor",
|
|
54
|
+
"LlmRecognizerActor",
|
|
55
|
+
"NoOpContextEnhancer",
|
|
56
|
+
"ReassemblyActor",
|
|
57
|
+
"RecognizerActor",
|
|
58
|
+
"TransformerInferenceActor",
|
|
59
|
+
"create_anonymizer_actor",
|
|
60
|
+
"create_anonymizer_actor_class",
|
|
61
|
+
"create_transformer_actor",
|
|
62
|
+
"create_transformer_actor_class",
|
|
63
|
+
]
|