fmtr.tools 1.2.5__tar.gz → 1.2.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of fmtr.tools might be problematic. Click here for more details.

Files changed (89) hide show
  1. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/PKG-INFO +45 -43
  2. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/dns_tools/client.py +9 -7
  3. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/dns_tools/server.py +10 -17
  4. fmtr_tools-1.2.7/fmtr/tools/pattern_tools.py +232 -0
  5. fmtr_tools-1.2.7/fmtr/tools/version +1 -0
  6. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr.tools.egg-info/PKG-INFO +45 -43
  7. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr.tools.egg-info/requires.txt +44 -42
  8. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/setup.py +1 -1
  9. fmtr_tools-1.2.5/fmtr/tools/pattern_tools.py +0 -175
  10. fmtr_tools-1.2.5/fmtr/tools/version +0 -1
  11. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/LICENSE +0 -0
  12. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/README.md +0 -0
  13. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/__init__.py +0 -0
  14. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/ai_tools/__init__.py +0 -0
  15. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/ai_tools/agentic_tools.py +0 -0
  16. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/ai_tools/inference_tools.py +0 -0
  17. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/api_tools.py +0 -0
  18. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/async_tools.py +0 -0
  19. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/augmentation_tools.py +0 -0
  20. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/caching_tools.py +0 -0
  21. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/constants.py +0 -0
  22. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/data_modelling_tools.py +0 -0
  23. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/dataclass_tools.py +0 -0
  24. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/datatype_tools.py +0 -0
  25. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/debugging_tools.py +0 -0
  26. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/dns_tools/__init__.py +0 -0
  27. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/dns_tools/dm.py +0 -0
  28. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/docker_tools.py +0 -0
  29. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/entrypoints/__init__.py +0 -0
  30. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/entrypoints/cache_hfh.py +0 -0
  31. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/entrypoints/ep_test.py +0 -0
  32. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/entrypoints/remote_debug_test.py +0 -0
  33. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/entrypoints/shell_debug.py +0 -0
  34. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/environment_tools.py +0 -0
  35. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/function_tools.py +0 -0
  36. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/google_api_tools.py +0 -0
  37. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/hash_tools.py +0 -0
  38. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/hfh_tools.py +0 -0
  39. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/html_tools.py +0 -0
  40. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/http_tools.py +0 -0
  41. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/import_tools.py +0 -0
  42. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/inspection_tools.py +0 -0
  43. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/interface_tools.py +0 -0
  44. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/iterator_tools.py +0 -0
  45. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/json_fix_tools.py +0 -0
  46. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/json_tools.py +0 -0
  47. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/logging_tools.py +0 -0
  48. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/merging_tools.py +0 -0
  49. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/metric_tools.py +0 -0
  50. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/name_tools.py +0 -0
  51. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/netrc_tools.py +0 -0
  52. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/openai_tools.py +0 -0
  53. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/packaging_tools.py +0 -0
  54. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/parallel_tools.py +0 -0
  55. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/path_tools/__init__.py +0 -0
  56. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/path_tools/app_path_tools.py +0 -0
  57. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/path_tools/path_tools.py +0 -0
  58. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/path_tools/type_path_tools.py +0 -0
  59. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/pdf_tools.py +0 -0
  60. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/platform_tools.py +0 -0
  61. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/process_tools.py +0 -0
  62. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/profiling_tools.py +0 -0
  63. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/random_tools.py +0 -0
  64. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/semantic_tools.py +0 -0
  65. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/settings_tools.py +0 -0
  66. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/setup_tools/__init__.py +0 -0
  67. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/setup_tools/setup_tools.py +0 -0
  68. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/spaces_tools.py +0 -0
  69. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/string_tools.py +0 -0
  70. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/tabular_tools.py +0 -0
  71. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/tests/__init__.py +0 -0
  72. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/tests/conftest.py +0 -0
  73. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/tests/helpers.py +0 -0
  74. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/tests/test_datatype.py +0 -0
  75. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/tests/test_environment.py +0 -0
  76. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/tests/test_json.py +0 -0
  77. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/tests/test_path.py +0 -0
  78. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/tests/test_yaml.py +0 -0
  79. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/tokenization_tools.py +0 -0
  80. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/tools.py +0 -0
  81. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/unicode_tools.py +0 -0
  82. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/version_tools.py +0 -0
  83. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr/tools/yaml_tools.py +0 -0
  84. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr.tools.egg-info/SOURCES.txt +0 -0
  85. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr.tools.egg-info/dependency_links.txt +0 -0
  86. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr.tools.egg-info/entry_points.txt +0 -0
  87. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/fmtr.tools.egg-info/top_level.txt +0 -0
  88. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/pyproject.toml +0 -0
  89. {fmtr_tools-1.2.5 → fmtr_tools-1.2.7}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fmtr.tools
3
- Version: 1.2.5
3
+ Version: 1.2.7
4
4
  Summary: Collection of high-level tools to simplify everyday development tasks, with a focus on AI/ML
5
5
  Home-page: https://github.com/fmtr/tools
6
6
  Author: Frontmatter
@@ -106,6 +106,8 @@ Requires-Dist: pydevd-pycharm; extra == "debug"
106
106
  Provides-Extra: sets
107
107
  Requires-Dist: pydantic-settings; extra == "sets"
108
108
  Requires-Dist: pydantic; extra == "sets"
109
+ Requires-Dist: yamlscript; extra == "sets"
110
+ Requires-Dist: pyyaml; extra == "sets"
109
111
  Provides-Extra: path-app
110
112
  Requires-Dist: appdirs; extra == "path-app"
111
113
  Provides-Extra: path-type
@@ -128,60 +130,60 @@ Requires-Dist: logfire[httpx]; extra == "http"
128
130
  Provides-Extra: setup
129
131
  Requires-Dist: setuptools; extra == "setup"
130
132
  Provides-Extra: all
131
- Requires-Dist: torchaudio; extra == "all"
132
- Requires-Dist: google-auth-oauthlib; extra == "all"
133
- Requires-Dist: deepmerge; extra == "all"
134
- Requires-Dist: httpx_retries; extra == "all"
135
- Requires-Dist: regex; extra == "all"
136
- Requires-Dist: dask[bag]; extra == "all"
137
- Requires-Dist: pytest-cov; extra == "all"
138
- Requires-Dist: flet-video; extra == "all"
139
- Requires-Dist: uvicorn[standard]; extra == "all"
140
- Requires-Dist: logfire[fastapi]; extra == "all"
141
- Requires-Dist: logfire; extra == "all"
142
- Requires-Dist: bokeh; extra == "all"
133
+ Requires-Dist: distributed; extra == "all"
134
+ Requires-Dist: diskcache; extra == "all"
143
135
  Requires-Dist: sre_yield; extra == "all"
144
- Requires-Dist: google-auth-httplib2; extra == "all"
136
+ Requires-Dist: httpx_retries; extra == "all"
137
+ Requires-Dist: filetype; extra == "all"
138
+ Requires-Dist: logfire[httpx]; extra == "all"
145
139
  Requires-Dist: pymupdf4llm; extra == "all"
146
- Requires-Dist: pydantic-ai[logfire,openai]; extra == "all"
140
+ Requires-Dist: Unidecode; extra == "all"
141
+ Requires-Dist: uvicorn[standard]; extra == "all"
142
+ Requires-Dist: faker; extra == "all"
147
143
  Requires-Dist: semver; extra == "all"
148
- Requires-Dist: yamlscript; extra == "all"
149
- Requires-Dist: torchvision; extra == "all"
150
- Requires-Dist: pydevd-pycharm; extra == "all"
151
- Requires-Dist: tinynetrc; extra == "all"
152
144
  Requires-Dist: pyyaml; extra == "all"
153
- Requires-Dist: openai; extra == "all"
154
- Requires-Dist: sentence_transformers; extra == "all"
155
- Requires-Dist: pymupdf; extra == "all"
145
+ Requires-Dist: regex; extra == "all"
146
+ Requires-Dist: contexttimer; extra == "all"
147
+ Requires-Dist: peft; extra == "all"
148
+ Requires-Dist: flet-webview; extra == "all"
156
149
  Requires-Dist: pydantic-settings; extra == "all"
150
+ Requires-Dist: deepmerge; extra == "all"
157
151
  Requires-Dist: httpx; extra == "all"
158
- Requires-Dist: tabulate; extra == "all"
159
- Requires-Dist: setuptools; extra == "all"
160
- Requires-Dist: diskcache; extra == "all"
161
- Requires-Dist: distributed; extra == "all"
162
- Requires-Dist: huggingface_hub; extra == "all"
163
- Requires-Dist: flet-webview; extra == "all"
164
- Requires-Dist: peft; extra == "all"
165
- Requires-Dist: google-api-python-client; extra == "all"
166
- Requires-Dist: contexttimer; extra == "all"
167
- Requires-Dist: docker; extra == "all"
168
152
  Requires-Dist: pydantic; extra == "all"
169
- Requires-Dist: dnspython[doh]; extra == "all"
153
+ Requires-Dist: pymupdf; extra == "all"
154
+ Requires-Dist: google-auth; extra == "all"
155
+ Requires-Dist: google-api-python-client; extra == "all"
156
+ Requires-Dist: logfire; extra == "all"
170
157
  Requires-Dist: openpyxl; extra == "all"
171
- Requires-Dist: filetype; extra == "all"
158
+ Requires-Dist: html2text; extra == "all"
159
+ Requires-Dist: pytest-cov; extra == "all"
160
+ Requires-Dist: docker; extra == "all"
161
+ Requires-Dist: google-auth-oauthlib; extra == "all"
162
+ Requires-Dist: json_repair; extra == "all"
163
+ Requires-Dist: ollama; extra == "all"
164
+ Requires-Dist: sentence_transformers; extra == "all"
165
+ Requires-Dist: pandas; extra == "all"
166
+ Requires-Dist: setuptools; extra == "all"
167
+ Requires-Dist: logfire[fastapi]; extra == "all"
168
+ Requires-Dist: tabulate; extra == "all"
169
+ Requires-Dist: pydevd-pycharm; extra == "all"
170
+ Requires-Dist: huggingface_hub; extra == "all"
171
+ Requires-Dist: flet-video; extra == "all"
172
+ Requires-Dist: dask[bag]; extra == "all"
173
+ Requires-Dist: google-auth-httplib2; extra == "all"
174
+ Requires-Dist: bokeh; extra == "all"
172
175
  Requires-Dist: fastapi; extra == "all"
173
- Requires-Dist: faker; extra == "all"
176
+ Requires-Dist: dnspython[doh]; extra == "all"
174
177
  Requires-Dist: transformers[sentencepiece]; extra == "all"
175
- Requires-Dist: Unidecode; extra == "all"
178
+ Requires-Dist: yamlscript; extra == "all"
179
+ Requires-Dist: torchaudio; extra == "all"
180
+ Requires-Dist: pydantic-ai[logfire,openai]; extra == "all"
181
+ Requires-Dist: torchvision; extra == "all"
182
+ Requires-Dist: flet[all]; extra == "all"
183
+ Requires-Dist: openai; extra == "all"
184
+ Requires-Dist: tinynetrc; extra == "all"
176
185
  Requires-Dist: appdirs; extra == "all"
177
- Requires-Dist: logfire[httpx]; extra == "all"
178
186
  Requires-Dist: tokenizers; extra == "all"
179
- Requires-Dist: html2text; extra == "all"
180
- Requires-Dist: ollama; extra == "all"
181
- Requires-Dist: json_repair; extra == "all"
182
- Requires-Dist: flet[all]; extra == "all"
183
- Requires-Dist: google-auth; extra == "all"
184
- Requires-Dist: pandas; extra == "all"
185
187
  Dynamic: author
186
188
  Dynamic: author-email
187
189
  Dynamic: description
@@ -1,4 +1,5 @@
1
1
  import dns
2
+ from dataclasses import dataclass
2
3
  from dns import query
3
4
  from functools import cached_property
4
5
  from httpx_retries import Retry, RetryTransport
@@ -39,6 +40,7 @@ class ClientBasePlain:
39
40
  exchange.response_upstream = Response.from_message(response)
40
41
 
41
42
 
43
+ @dataclass
42
44
  class ClientDoH:
43
45
  """
44
46
 
@@ -47,19 +49,19 @@ class ClientDoH:
47
49
  """
48
50
 
49
51
  HEADERS = {"Content-Type": "application/dns-message"}
50
- client = HTTPClientDoH()
52
+ CLIENT = HTTPClientDoH()
53
+ BOOTSTRAP = ClientBasePlain('8.8.8.8')
54
+
55
+ host: str
56
+ url: str
51
57
 
52
- def __init__(self, host, url):
53
- self.host = host
54
- self.url = url
55
- self.bootstrap = ClientBasePlain('8.8.8.8')
56
58
 
57
59
  @cached_property
58
60
  def ip(self):
59
61
  message = dns.message.make_query(self.host, dns.rdatatype.A, flags=0)
60
62
  request = Request.from_message(message)
61
63
  exchange = Exchange(request=request, ip=None, port=None)
62
- self.bootstrap.resolve(exchange)
64
+ self.BOOTSTRAP.resolve(exchange)
63
65
  ip = next(iter(exchange.response_upstream.answer.items.keys())).address
64
66
  return ip
65
67
 
@@ -72,7 +74,7 @@ class ClientDoH:
72
74
  request = exchange.request
73
75
  headers = self.HEADERS | dict(Host=self.host)
74
76
  url = self.url.format(host=self.ip)
75
- response_doh = self.client.post(url, headers=headers, content=request.wire)
77
+ response_doh = self.CLIENT.post(url, headers=headers, content=request.wire)
76
78
  response_doh.raise_for_status()
77
79
  response = Response.from_http(response_doh)
78
80
  exchange.response_upstream = response
@@ -1,10 +1,12 @@
1
1
  import socket
2
+ from dataclasses import dataclass
2
3
 
3
4
  from fmtr.tools import logger
4
5
  from fmtr.tools.dns_tools.client import ClientDoH
5
- from fmtr.tools.dns_tools.dm import Exchange, Response
6
+ from fmtr.tools.dns_tools.dm import Exchange
6
7
 
7
8
 
9
+ @dataclass
8
10
  class ServerBasePlain:
9
11
  """
10
12
 
@@ -12,9 +14,11 @@ class ServerBasePlain:
12
14
 
13
15
  """
14
16
 
15
- def __init__(self, host, port):
16
- self.host = host
17
- self.port = port
17
+ host: str
18
+ port: int
19
+
20
+ def __post_init__(self):
21
+
18
22
  self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
19
23
 
20
24
  def resolve(self, exchange: Exchange):
@@ -36,6 +40,7 @@ class ServerBasePlain:
36
40
  sock.sendto(exchange.response.wire, (ip, port))
37
41
 
38
42
 
43
+ @dataclass
39
44
  class ServerBaseDoHProxy(ServerBasePlain):
40
45
  """
41
46
 
@@ -43,9 +48,7 @@ class ServerBaseDoHProxy(ServerBasePlain):
43
48
 
44
49
  """
45
50
 
46
- def __init__(self, host, port, client: ClientDoH):
47
- super().__init__(host, port)
48
- self.client = client
51
+ client: ClientDoH
49
52
 
50
53
  def process_question(self, exchange: Exchange):
51
54
  return
@@ -53,16 +56,6 @@ class ServerBaseDoHProxy(ServerBasePlain):
53
56
  def process_upstream(self, exchange: Exchange):
54
57
  return
55
58
 
56
- def from_upstream(self, exchange: Exchange) -> Exchange:
57
-
58
- request = exchange.request
59
- response_doh = self.client.post(self.URL, headers=self.HEADERS, content=request.wire)
60
- response_doh.raise_for_status()
61
- response = Response.from_http(response_doh)
62
- exchange.response_upstream = response
63
-
64
- return exchange
65
-
66
59
  def resolve(self, exchange: Exchange):
67
60
  """
68
61
 
@@ -0,0 +1,232 @@
1
+ import regex as re
2
+ from dataclasses import dataclass, asdict
3
+ from functools import cached_property
4
+ from typing import List, Any
5
+
6
+ from fmtr.tools.logging_tools import logger
7
+ from fmtr.tools.string_tools import join
8
+
9
+
10
+ class RewriteCircularLoopError(Exception):
11
+ """
12
+
13
+ Circular loop error
14
+
15
+ """
16
+
17
+
18
+ MASK_GROUP = '(?:{pattern})'
19
+ MASK_NAMED = r"(?P<{key}>{pattern})"
20
+
21
+
22
+ def alt(*patterns):
23
+ patterns = sorted(patterns, key=len, reverse=True)
24
+ pattern = '|'.join(patterns)
25
+ pattern = MASK_GROUP.format(pattern=pattern)
26
+ return pattern
27
+
28
+
29
+
30
+
31
+
32
+
33
+ @dataclass
34
+ class Key:
35
+ RECORD_SEP = '␞'
36
+
37
+ def flatten(self, data):
38
+ """
39
+
40
+ Flatten/serialise dictionary data
41
+
42
+ """
43
+ pairs = [f'{value}' for key, value in data.items()]
44
+ string = self.RECORD_SEP.join(pairs)
45
+ return string
46
+
47
+ @cached_property
48
+ def pattern(self):
49
+ """
50
+
51
+ Serialise to pattern
52
+
53
+ """
54
+ data = {key: MASK_NAMED.format(key=key, pattern=value) for key, value in asdict(self).items()}
55
+ pattern = self.flatten(data)
56
+ return pattern
57
+
58
+ @cached_property
59
+ def rx(self):
60
+ """
61
+
62
+ Compile to Regular Expression
63
+
64
+ """
65
+ return re.compile(self.pattern)
66
+
67
+ @cached_property
68
+ def string(self):
69
+ """
70
+
71
+ Serialise to string
72
+
73
+ """
74
+ string = self.flatten(asdict(self))
75
+ return string
76
+
77
+ def transform(self, match: re.Match):
78
+ """
79
+
80
+ Transform match object into a new object of the same type.
81
+
82
+ """
83
+ groupdict = match.groupdict()
84
+ data = {key: value.format(**groupdict) for key, value in asdict(self).items()}
85
+ obj = self.__class__(**data)
86
+ return obj
87
+
88
+
89
+ @dataclass
90
+ class Item:
91
+ """
92
+
93
+ Key-value pair
94
+
95
+ """
96
+ key: Key
97
+ value: Key
98
+
99
+ @dataclass
100
+ class Mapper:
101
+ """
102
+
103
+ Pattern-based, dictionary-like mapper.
104
+ Compiles a single regex pattern from a list of rules, and determines which rule matched.
105
+ It supports initialization from structured rule data, execution of a single lookup pass, and
106
+ recursive lookups until a stable state is reached.
107
+
108
+ """
109
+ PREFIX_GROUP = '__'
110
+ items: List[Item]
111
+ default: Any = None
112
+ is_recursive: bool = False
113
+
114
+ @cached_property
115
+ def pattern(self):
116
+ """
117
+
118
+ Provides a dynamically generated regex pattern based on the rules provided.
119
+
120
+ """
121
+ patterns = [
122
+ MASK_NAMED.format(key=f'{self.PREFIX_GROUP}{i}', pattern=item.key.pattern)
123
+ for i, item in enumerate(self.items)
124
+ ]
125
+ pattern = alt(*patterns)
126
+ return pattern
127
+
128
+ @cached_property
129
+ def rx(self):
130
+ """
131
+
132
+ Regex object.
133
+
134
+ """
135
+ return re.compile(self.pattern)
136
+
137
+ def get_default(self, key: Key):
138
+ if self.is_recursive:
139
+ return key
140
+ else:
141
+ return self.default
142
+
143
+ def get(self, key: Key) -> Key:
144
+ """
145
+
146
+ Use recursive or single lookup pass, depending on whether recursive lookups have been specified.
147
+
148
+ """
149
+ if self.is_recursive:
150
+ return self.get_recursive(key)
151
+ else:
152
+ return self.get_one(key)
153
+
154
+ def get_one(self, key: Key):
155
+ """
156
+
157
+ Single lookup pass.
158
+ Lookup the source string based on the matching rule.
159
+
160
+ """
161
+
162
+ match = self.rx.fullmatch(key.string)
163
+
164
+ if not match:
165
+ value = self.get_default(key)
166
+ logger.debug(f'No match for {key=}.')
167
+ else:
168
+
169
+ match_ids = {name: v for name, v in match.groupdict().items() if v}
170
+ rule_ids = {
171
+ int(id.removeprefix(self.PREFIX_GROUP))
172
+ for id in match_ids.keys() if id.startswith(self.PREFIX_GROUP)
173
+ }
174
+
175
+ if len(rule_ids) != 1:
176
+ msg = f'Multiple group matches: {rule_ids}'
177
+ raise ValueError(msg)
178
+
179
+ rule_id = next(iter(rule_ids))
180
+ rule = self.items[rule_id]
181
+
182
+ if isinstance(rule.value, Key):
183
+ value = rule.value.transform(match)
184
+ else:
185
+ value = rule.value
186
+
187
+ logger.debug(f'Matched using {rule_id=}: {key=} → {value=}')
188
+
189
+ return value
190
+
191
+ def get_recursive(self, key: Key) -> Key:
192
+ """
193
+
194
+ Lookup the provided text by continuously applying lookup rules until no changes are made
195
+ or a circular loop is detected.
196
+
197
+ """
198
+ history = []
199
+ previous = key
200
+
201
+ def get_history_str():
202
+ return join(history, sep=' → ')
203
+
204
+ with logger.span(f'Matching {key=}...'):
205
+ while True:
206
+ if previous in history:
207
+ history.append(previous)
208
+ msg = f'Loop detected on node "{previous}": {get_history_str()}'
209
+ raise RewriteCircularLoopError(msg)
210
+
211
+ history.append(previous)
212
+
213
+ new = previous
214
+
215
+ new = self.get_one(new)
216
+
217
+ if new == previous:
218
+ break
219
+
220
+ previous = new
221
+
222
+ if len(history) == 1:
223
+ history_str = 'No matching performed.'
224
+ else:
225
+ history_str = get_history_str()
226
+ logger.debug(f'Finished matching: {history_str}')
227
+
228
+ return previous
229
+
230
+
231
+ if __name__ == '__main__':
232
+ ...
@@ -0,0 +1 @@
1
+ 1.2.7
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fmtr.tools
3
- Version: 1.2.5
3
+ Version: 1.2.7
4
4
  Summary: Collection of high-level tools to simplify everyday development tasks, with a focus on AI/ML
5
5
  Home-page: https://github.com/fmtr/tools
6
6
  Author: Frontmatter
@@ -106,6 +106,8 @@ Requires-Dist: pydevd-pycharm; extra == "debug"
106
106
  Provides-Extra: sets
107
107
  Requires-Dist: pydantic-settings; extra == "sets"
108
108
  Requires-Dist: pydantic; extra == "sets"
109
+ Requires-Dist: yamlscript; extra == "sets"
110
+ Requires-Dist: pyyaml; extra == "sets"
109
111
  Provides-Extra: path-app
110
112
  Requires-Dist: appdirs; extra == "path-app"
111
113
  Provides-Extra: path-type
@@ -128,60 +130,60 @@ Requires-Dist: logfire[httpx]; extra == "http"
128
130
  Provides-Extra: setup
129
131
  Requires-Dist: setuptools; extra == "setup"
130
132
  Provides-Extra: all
131
- Requires-Dist: torchaudio; extra == "all"
132
- Requires-Dist: google-auth-oauthlib; extra == "all"
133
- Requires-Dist: deepmerge; extra == "all"
134
- Requires-Dist: httpx_retries; extra == "all"
135
- Requires-Dist: regex; extra == "all"
136
- Requires-Dist: dask[bag]; extra == "all"
137
- Requires-Dist: pytest-cov; extra == "all"
138
- Requires-Dist: flet-video; extra == "all"
139
- Requires-Dist: uvicorn[standard]; extra == "all"
140
- Requires-Dist: logfire[fastapi]; extra == "all"
141
- Requires-Dist: logfire; extra == "all"
142
- Requires-Dist: bokeh; extra == "all"
133
+ Requires-Dist: distributed; extra == "all"
134
+ Requires-Dist: diskcache; extra == "all"
143
135
  Requires-Dist: sre_yield; extra == "all"
144
- Requires-Dist: google-auth-httplib2; extra == "all"
136
+ Requires-Dist: httpx_retries; extra == "all"
137
+ Requires-Dist: filetype; extra == "all"
138
+ Requires-Dist: logfire[httpx]; extra == "all"
145
139
  Requires-Dist: pymupdf4llm; extra == "all"
146
- Requires-Dist: pydantic-ai[logfire,openai]; extra == "all"
140
+ Requires-Dist: Unidecode; extra == "all"
141
+ Requires-Dist: uvicorn[standard]; extra == "all"
142
+ Requires-Dist: faker; extra == "all"
147
143
  Requires-Dist: semver; extra == "all"
148
- Requires-Dist: yamlscript; extra == "all"
149
- Requires-Dist: torchvision; extra == "all"
150
- Requires-Dist: pydevd-pycharm; extra == "all"
151
- Requires-Dist: tinynetrc; extra == "all"
152
144
  Requires-Dist: pyyaml; extra == "all"
153
- Requires-Dist: openai; extra == "all"
154
- Requires-Dist: sentence_transformers; extra == "all"
155
- Requires-Dist: pymupdf; extra == "all"
145
+ Requires-Dist: regex; extra == "all"
146
+ Requires-Dist: contexttimer; extra == "all"
147
+ Requires-Dist: peft; extra == "all"
148
+ Requires-Dist: flet-webview; extra == "all"
156
149
  Requires-Dist: pydantic-settings; extra == "all"
150
+ Requires-Dist: deepmerge; extra == "all"
157
151
  Requires-Dist: httpx; extra == "all"
158
- Requires-Dist: tabulate; extra == "all"
159
- Requires-Dist: setuptools; extra == "all"
160
- Requires-Dist: diskcache; extra == "all"
161
- Requires-Dist: distributed; extra == "all"
162
- Requires-Dist: huggingface_hub; extra == "all"
163
- Requires-Dist: flet-webview; extra == "all"
164
- Requires-Dist: peft; extra == "all"
165
- Requires-Dist: google-api-python-client; extra == "all"
166
- Requires-Dist: contexttimer; extra == "all"
167
- Requires-Dist: docker; extra == "all"
168
152
  Requires-Dist: pydantic; extra == "all"
169
- Requires-Dist: dnspython[doh]; extra == "all"
153
+ Requires-Dist: pymupdf; extra == "all"
154
+ Requires-Dist: google-auth; extra == "all"
155
+ Requires-Dist: google-api-python-client; extra == "all"
156
+ Requires-Dist: logfire; extra == "all"
170
157
  Requires-Dist: openpyxl; extra == "all"
171
- Requires-Dist: filetype; extra == "all"
158
+ Requires-Dist: html2text; extra == "all"
159
+ Requires-Dist: pytest-cov; extra == "all"
160
+ Requires-Dist: docker; extra == "all"
161
+ Requires-Dist: google-auth-oauthlib; extra == "all"
162
+ Requires-Dist: json_repair; extra == "all"
163
+ Requires-Dist: ollama; extra == "all"
164
+ Requires-Dist: sentence_transformers; extra == "all"
165
+ Requires-Dist: pandas; extra == "all"
166
+ Requires-Dist: setuptools; extra == "all"
167
+ Requires-Dist: logfire[fastapi]; extra == "all"
168
+ Requires-Dist: tabulate; extra == "all"
169
+ Requires-Dist: pydevd-pycharm; extra == "all"
170
+ Requires-Dist: huggingface_hub; extra == "all"
171
+ Requires-Dist: flet-video; extra == "all"
172
+ Requires-Dist: dask[bag]; extra == "all"
173
+ Requires-Dist: google-auth-httplib2; extra == "all"
174
+ Requires-Dist: bokeh; extra == "all"
172
175
  Requires-Dist: fastapi; extra == "all"
173
- Requires-Dist: faker; extra == "all"
176
+ Requires-Dist: dnspython[doh]; extra == "all"
174
177
  Requires-Dist: transformers[sentencepiece]; extra == "all"
175
- Requires-Dist: Unidecode; extra == "all"
178
+ Requires-Dist: yamlscript; extra == "all"
179
+ Requires-Dist: torchaudio; extra == "all"
180
+ Requires-Dist: pydantic-ai[logfire,openai]; extra == "all"
181
+ Requires-Dist: torchvision; extra == "all"
182
+ Requires-Dist: flet[all]; extra == "all"
183
+ Requires-Dist: openai; extra == "all"
184
+ Requires-Dist: tinynetrc; extra == "all"
176
185
  Requires-Dist: appdirs; extra == "all"
177
- Requires-Dist: logfire[httpx]; extra == "all"
178
186
  Requires-Dist: tokenizers; extra == "all"
179
- Requires-Dist: html2text; extra == "all"
180
- Requires-Dist: ollama; extra == "all"
181
- Requires-Dist: json_repair; extra == "all"
182
- Requires-Dist: flet[all]; extra == "all"
183
- Requires-Dist: google-auth; extra == "all"
184
- Requires-Dist: pandas; extra == "all"
185
187
  Dynamic: author
186
188
  Dynamic: author-email
187
189
  Dynamic: description
@@ -15,60 +15,60 @@ pydantic-ai[logfire,openai]
15
15
  ollama
16
16
 
17
17
  [all]
18
- torchaudio
19
- google-auth-oauthlib
20
- deepmerge
21
- httpx_retries
22
- regex
23
- dask[bag]
24
- pytest-cov
25
- flet-video
26
- uvicorn[standard]
27
- logfire[fastapi]
28
- logfire
29
- bokeh
18
+ distributed
19
+ diskcache
30
20
  sre_yield
31
- google-auth-httplib2
21
+ httpx_retries
22
+ filetype
23
+ logfire[httpx]
32
24
  pymupdf4llm
33
- pydantic-ai[logfire,openai]
25
+ Unidecode
26
+ uvicorn[standard]
27
+ faker
34
28
  semver
35
- yamlscript
36
- torchvision
37
- pydevd-pycharm
38
- tinynetrc
39
29
  pyyaml
40
- openai
41
- sentence_transformers
42
- pymupdf
30
+ regex
31
+ contexttimer
32
+ peft
33
+ flet-webview
43
34
  pydantic-settings
35
+ deepmerge
44
36
  httpx
45
- tabulate
46
- setuptools
47
- diskcache
48
- distributed
49
- huggingface_hub
50
- flet-webview
51
- peft
52
- google-api-python-client
53
- contexttimer
54
- docker
55
37
  pydantic
56
- dnspython[doh]
38
+ pymupdf
39
+ google-auth
40
+ google-api-python-client
41
+ logfire
57
42
  openpyxl
58
- filetype
43
+ html2text
44
+ pytest-cov
45
+ docker
46
+ google-auth-oauthlib
47
+ json_repair
48
+ ollama
49
+ sentence_transformers
50
+ pandas
51
+ setuptools
52
+ logfire[fastapi]
53
+ tabulate
54
+ pydevd-pycharm
55
+ huggingface_hub
56
+ flet-video
57
+ dask[bag]
58
+ google-auth-httplib2
59
+ bokeh
59
60
  fastapi
60
- faker
61
+ dnspython[doh]
61
62
  transformers[sentencepiece]
62
- Unidecode
63
+ yamlscript
64
+ torchaudio
65
+ pydantic-ai[logfire,openai]
66
+ torchvision
67
+ flet[all]
68
+ openai
69
+ tinynetrc
63
70
  appdirs
64
- logfire[httpx]
65
71
  tokenizers
66
- html2text
67
- ollama
68
- json_repair
69
- flet[all]
70
- google-auth
71
- pandas
72
72
 
73
73
  [api]
74
74
  fastapi
@@ -183,6 +183,8 @@ openpyxl
183
183
  [sets]
184
184
  pydantic-settings
185
185
  pydantic
186
+ yamlscript
187
+ pyyaml
186
188
 
187
189
  [setup]
188
190
  setuptools
@@ -32,7 +32,7 @@ DEPENDENCIES = {
32
32
  'caching': ['diskcache'],
33
33
  'pdf': ['pymupdf', 'dm', 'pymupdf4llm'],
34
34
  'debug': ['pydevd-pycharm'],
35
- 'sets': ['pydantic-settings', 'dm'],
35
+ 'sets': ['pydantic-settings', 'dm', 'yaml'],
36
36
  'path.app': ['appdirs'],
37
37
  'path.type': ['filetype'],
38
38
  'dns': ['dnspython[doh]', 'http'],
@@ -1,175 +0,0 @@
1
- import regex as re
2
- from dataclasses import dataclass
3
- from functools import cached_property
4
- from typing import List
5
-
6
- from fmtr.tools.logging_tools import logger
7
-
8
-
9
- class RewriteCircularLoopError(Exception):
10
- """
11
-
12
- Circular loop error
13
-
14
- """
15
-
16
-
17
- @dataclass
18
- class Rewrite:
19
- """
20
- Represents a single rule for pattern matching and target string replacement.
21
-
22
- This class is used to define a rule with a pattern and a target string.
23
- The `pattern` is a regular expression used to identify matches in input text.
24
- The `target` allows rewriting the identified matches with a formatted string.
25
- It provides properties for generating a unique identifier for use as a regex group name and compiling the provided pattern into a regular expression object.
26
-
27
- """
28
- pattern: str
29
- target: str
30
-
31
- @cached_property
32
- def id(self):
33
- """
34
-
35
- Regex group name.
36
-
37
- """
38
- return f'id{abs(hash(self.pattern))}'
39
-
40
- @cached_property
41
- def rx(self):
42
- """
43
-
44
- Regex object.
45
-
46
- """
47
- return re.compile(self.pattern)
48
-
49
- def apply(self, match: re.Match):
50
- """
51
-
52
- Rewrite using the target string and match groups.
53
-
54
- """
55
- target = self.target.format(**match.groupdict())
56
- return target
57
-
58
-
59
- @dataclass
60
- class Rewriter:
61
- """
62
-
63
- Represents a Rewriter class that handles pattern matching, rule application, and text rewriting.
64
- Compiles a single regex pattern from a list of rules, and determines which rule matched.
65
- It supports initialization from structured rule data, execution of a single rewrite pass, and
66
- recursive rewriting until a stable state is reached.
67
-
68
- """
69
- rules: List[Rewrite]
70
-
71
- @cached_property
72
- def pattern(self):
73
- """
74
-
75
- Provides a dynamically generated regex pattern based on the rules provided.
76
-
77
- """
78
- patterns = [fr"(?P<{rule.id}>{rule.pattern})" for rule in self.rules]
79
- sorted(patterns, key=len, reverse=True)
80
- pattern = '|'.join(patterns)
81
- return pattern
82
-
83
- @cached_property
84
- def rule_lookup(self):
85
- """
86
-
87
- Dictionary mapping rule identifiers to their corresponding rules.
88
- """
89
-
90
- return {rule.id: rule for rule in self.rules}
91
-
92
- @cached_property
93
- def rx(self):
94
- """
95
-
96
- Regex object.
97
-
98
- """
99
- return re.compile(self.pattern)
100
-
101
- def rewrite_pass(self, source: str):
102
- """
103
-
104
- Single rewrite pass.
105
- Rewrites the provided source string based on the matching rule.
106
-
107
- """
108
-
109
- match = self.rx.fullmatch(source)
110
-
111
- if not match:
112
- return source
113
-
114
- match_ids = {k: v for k, v in match.groupdict().items() if v}
115
- match_id = match_ids & self.rule_lookup.keys()
116
-
117
- if len(match_id) != 1:
118
- msg = f'Multiple group matches: {match_id}'
119
- raise ValueError(msg)
120
-
121
- match_id = next(iter(match_id))
122
- rule = self.rule_lookup[match_id]
123
- target = rule.apply(match)
124
-
125
- logger.debug(f'Rewrote using {match_id=}: {source=} -> {target=}')
126
-
127
- return target
128
-
129
- def rewrite(self, source: str) -> str:
130
- """
131
-
132
- Rewrites the provided text by continuously applying rewrite rules until no changes are made
133
- or a circular loop is detected.
134
-
135
- """
136
- history = []
137
- previous = source
138
-
139
- def get_history_str():
140
- return ' -> '.join(history)
141
-
142
- with logger.span(f'Rewriting "{source}"...'):
143
- while True:
144
- if previous in history:
145
- history.append(previous)
146
- msg = f'Loop detected on node "{previous}": {get_history_str()}'
147
- raise RewriteCircularLoopError(msg)
148
-
149
- history.append(previous)
150
-
151
- new = previous
152
-
153
- new = self.rewrite_pass(new)
154
-
155
- if new == previous:
156
- break
157
-
158
- previous = new
159
-
160
- if len(history) == 1:
161
- history_str = 'No rewrites performed.'
162
- else:
163
- history_str = get_history_str()
164
- logger.debug(f'Finished rewriting: {history_str}')
165
-
166
- return previous
167
-
168
- @classmethod
169
- def from_data(cls, data):
170
- rules = [Rewrite(*pair) for pair in data.items()]
171
- self = cls(rules=rules)
172
- return self
173
-
174
-
175
-
@@ -1 +0,0 @@
1
- 1.2.5
File without changes
File without changes
File without changes
File without changes