fmtr.tools 1.1.16__py3-none-any.whl → 1.1.17__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of fmtr.tools might be problematic. Click here for more details.
- fmtr/tools/__init__.py +16 -1
- fmtr/tools/dns_tools.py +57 -0
- fmtr/tools/pattern_tools.py +237 -0
- fmtr/tools/setup_tools.py +268 -0
- fmtr/tools/string_tools.py +0 -6
- fmtr/tools/version +1 -1
- {fmtr_tools-1.1.16.dist-info → fmtr_tools-1.1.17.dist-info}/METADATA +56 -38
- {fmtr_tools-1.1.16.dist-info → fmtr_tools-1.1.17.dist-info}/RECORD +12 -9
- {fmtr_tools-1.1.16.dist-info → fmtr_tools-1.1.17.dist-info}/WHEEL +1 -1
- {fmtr_tools-1.1.16.dist-info → fmtr_tools-1.1.17.dist-info}/entry_points.txt +0 -0
- {fmtr_tools-1.1.16.dist-info → fmtr_tools-1.1.17.dist-info}/licenses/LICENSE +0 -0
- {fmtr_tools-1.1.16.dist-info → fmtr_tools-1.1.17.dist-info}/top_level.txt +0 -0
fmtr/tools/__init__.py
CHANGED
|
@@ -163,4 +163,19 @@ except ImportError as exception:
|
|
|
163
163
|
try:
|
|
164
164
|
from fmtr.tools import settings_tools as sets
|
|
165
165
|
except ImportError as exception:
|
|
166
|
-
sets = MissingExtraMockModule('sets', exception)
|
|
166
|
+
sets = MissingExtraMockModule('sets', exception)
|
|
167
|
+
|
|
168
|
+
try:
|
|
169
|
+
from fmtr.tools import pattern_tools as patterns
|
|
170
|
+
except ImportError as exception:
|
|
171
|
+
patterns = MissingExtraMockModule('patterns', exception)
|
|
172
|
+
|
|
173
|
+
try:
|
|
174
|
+
from fmtr.tools import dns_tools as dns
|
|
175
|
+
except ImportError as exception:
|
|
176
|
+
dns = MissingExtraMockModule('dns', exception)
|
|
177
|
+
|
|
178
|
+
try:
|
|
179
|
+
from fmtr.tools import setup_tools as setup
|
|
180
|
+
except ImportError as exception:
|
|
181
|
+
setup = MissingExtraMockModule('setup', exception)
|
fmtr/tools/dns_tools.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import dns
|
|
2
|
+
import httpx
|
|
3
|
+
import socket
|
|
4
|
+
from dns.message import Message, QueryMessage
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Server:
|
|
8
|
+
|
|
9
|
+
def __init__(self, host, port):
|
|
10
|
+
self.host = host
|
|
11
|
+
self.port = port
|
|
12
|
+
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
13
|
+
|
|
14
|
+
def resolve(self, message: Message) -> QueryMessage:
|
|
15
|
+
raise NotImplemented
|
|
16
|
+
|
|
17
|
+
def start(self):
|
|
18
|
+
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
19
|
+
sock.bind((self.host, self.port))
|
|
20
|
+
print(f"Listening on {self.host}:{self.port}")
|
|
21
|
+
while True:
|
|
22
|
+
data, addr = sock.recvfrom(512)
|
|
23
|
+
question = dns.message.from_wire(data)
|
|
24
|
+
answer = self.resolve(question)
|
|
25
|
+
sock.sendto(answer.to_wire(), addr)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class Proxy(Server):
|
|
29
|
+
url = "https://cloudflare-dns.com/dns-query"
|
|
30
|
+
headers = {"accept": "application/dns-json"}
|
|
31
|
+
|
|
32
|
+
def resolve(self, question: Message) -> Message:
|
|
33
|
+
qname = question.question[0].name.to_text()
|
|
34
|
+
qtype = dns.rdatatype.to_text(question.question[0].rdtype)
|
|
35
|
+
params = {"name": qname, "type": qtype}
|
|
36
|
+
|
|
37
|
+
response = httpx.get(self.url, headers=self.headers, params=params)
|
|
38
|
+
response.raise_for_status()
|
|
39
|
+
|
|
40
|
+
answer = dns.message.make_response(question)
|
|
41
|
+
answer.flags |= dns.flags.RA
|
|
42
|
+
|
|
43
|
+
data = response.json()
|
|
44
|
+
for answer_rr in data.get("Answer", []):
|
|
45
|
+
name = dns.name.from_text(answer_rr["name"])
|
|
46
|
+
rtype = answer_rr["type"]
|
|
47
|
+
ttl = answer_rr["TTL"]
|
|
48
|
+
rdata = dns.rdata.from_text(dns.rdataclass.IN, rtype, answer_rr["data"])
|
|
49
|
+
rrset = dns.rrset.from_rdata(name, ttl, rdata)
|
|
50
|
+
answer.answer.append(rrset)
|
|
51
|
+
|
|
52
|
+
return answer
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
if __name__ == "__main__":
|
|
56
|
+
proxy = Proxy("127.0.0.1", 5354)
|
|
57
|
+
proxy.start()
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
import regex as re
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from functools import cached_property
|
|
4
|
+
from typing import List
|
|
5
|
+
|
|
6
|
+
from fmtr.tools.logging_tools import logger
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class RewriteCircularLoopError(Exception):
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
Circular loop error
|
|
13
|
+
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class Rule:
|
|
19
|
+
"""
|
|
20
|
+
Represents a single rule for pattern matching and target string replacement.
|
|
21
|
+
|
|
22
|
+
This class is used to define a rule with a pattern and a target string.
|
|
23
|
+
The `pattern` is a regular expression used to identify matches in input text.
|
|
24
|
+
The `target` allows rewriting the identified matches with a formatted string.
|
|
25
|
+
It provides properties for generating a unique identifier for use as a regex group name and compiling the provided pattern into a regular expression object.
|
|
26
|
+
|
|
27
|
+
"""
|
|
28
|
+
pattern: str
|
|
29
|
+
target: str
|
|
30
|
+
|
|
31
|
+
@cached_property
|
|
32
|
+
def id(self):
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
Regex group name.
|
|
36
|
+
|
|
37
|
+
"""
|
|
38
|
+
return f'id{abs(hash(self.pattern))}'
|
|
39
|
+
|
|
40
|
+
@cached_property
|
|
41
|
+
def rx(self):
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
Regex object.
|
|
45
|
+
|
|
46
|
+
"""
|
|
47
|
+
return re.compile(self.pattern)
|
|
48
|
+
|
|
49
|
+
def apply(self, match: re.Match):
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
Rewrite using the target string and match groups.
|
|
53
|
+
|
|
54
|
+
"""
|
|
55
|
+
target = self.target.format(**match.groupdict())
|
|
56
|
+
return target
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass
|
|
60
|
+
class Rewriter:
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
Represents a Rewriter class that handles pattern matching, rule application, and text rewriting.
|
|
64
|
+
Compiles a single regex pattern from a list of rules, and determines which rule matched.
|
|
65
|
+
It supports initialization from structured rule data, execution of a single rewrite pass, and
|
|
66
|
+
recursive rewriting until a stable state is reached.
|
|
67
|
+
|
|
68
|
+
"""
|
|
69
|
+
rules: List[Rule]
|
|
70
|
+
|
|
71
|
+
@cached_property
|
|
72
|
+
def pattern(self):
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
Provides a dynamically generated regex pattern based on the rules provided.
|
|
76
|
+
|
|
77
|
+
"""
|
|
78
|
+
patterns = [fr"(?P<{rule.id}>{rule.pattern})" for rule in self.rules]
|
|
79
|
+
sorted(patterns, key=len, reverse=True)
|
|
80
|
+
pattern = '|'.join(patterns)
|
|
81
|
+
return pattern
|
|
82
|
+
|
|
83
|
+
@cached_property
|
|
84
|
+
def rule_lookup(self):
|
|
85
|
+
"""
|
|
86
|
+
|
|
87
|
+
Dictionary mapping rule identifiers to their corresponding rules.
|
|
88
|
+
"""
|
|
89
|
+
|
|
90
|
+
return {rule.id: rule for rule in self.rules}
|
|
91
|
+
|
|
92
|
+
@cached_property
|
|
93
|
+
def rx(self):
|
|
94
|
+
"""
|
|
95
|
+
|
|
96
|
+
Regex object.
|
|
97
|
+
|
|
98
|
+
"""
|
|
99
|
+
return re.compile(self.pattern)
|
|
100
|
+
|
|
101
|
+
def rewrite_pass(self, source: str):
|
|
102
|
+
"""
|
|
103
|
+
|
|
104
|
+
Single rewrite pass.
|
|
105
|
+
Rewrites the provided source string based on the matching rule.
|
|
106
|
+
|
|
107
|
+
"""
|
|
108
|
+
|
|
109
|
+
match = self.rx.match(source)
|
|
110
|
+
|
|
111
|
+
if not match:
|
|
112
|
+
return source
|
|
113
|
+
|
|
114
|
+
match_ids = {k: v for k, v in match.groupdict().items() if v}
|
|
115
|
+
match_id = match_ids & self.rule_lookup.keys()
|
|
116
|
+
|
|
117
|
+
if len(match_id) != 1:
|
|
118
|
+
msg = f'Multiple group matches: {match_id}'
|
|
119
|
+
raise ValueError(msg)
|
|
120
|
+
|
|
121
|
+
match_id = next(iter(match_id))
|
|
122
|
+
rule = self.rule_lookup[match_id]
|
|
123
|
+
target = rule.apply(match)
|
|
124
|
+
|
|
125
|
+
logger.debug(f'Rewrote using {match_id=}: {source=} -> {target=}')
|
|
126
|
+
|
|
127
|
+
return target
|
|
128
|
+
|
|
129
|
+
def rewrite(self, source: str) -> str:
|
|
130
|
+
"""
|
|
131
|
+
|
|
132
|
+
Rewrites the provided text by continuously applying rewrite rules until no changes are made
|
|
133
|
+
or a circular loop is detected.
|
|
134
|
+
|
|
135
|
+
"""
|
|
136
|
+
history = []
|
|
137
|
+
previous = source
|
|
138
|
+
|
|
139
|
+
def get_history_str():
|
|
140
|
+
return ' -> '.join(history)
|
|
141
|
+
|
|
142
|
+
with logger.span(f'Rewriting "{source}"...'):
|
|
143
|
+
while True:
|
|
144
|
+
if previous in history:
|
|
145
|
+
history.append(previous)
|
|
146
|
+
msg = f'Loop detected on node "{previous}": {get_history_str()}'
|
|
147
|
+
raise RewriteCircularLoopError(msg)
|
|
148
|
+
|
|
149
|
+
history.append(previous)
|
|
150
|
+
|
|
151
|
+
new = previous
|
|
152
|
+
|
|
153
|
+
new = self.rewrite_pass(new)
|
|
154
|
+
|
|
155
|
+
if new == previous:
|
|
156
|
+
break
|
|
157
|
+
|
|
158
|
+
previous = new
|
|
159
|
+
|
|
160
|
+
logger.debug(f'Finished rewriting: {get_history_str()}')
|
|
161
|
+
|
|
162
|
+
return previous
|
|
163
|
+
|
|
164
|
+
@classmethod
|
|
165
|
+
def from_data(cls, data):
|
|
166
|
+
rules = [Rule(*pair) for pair in data.items()]
|
|
167
|
+
self = cls(rules=rules)
|
|
168
|
+
return self
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
if __name__ == '__main__':
|
|
172
|
+
data = {
|
|
173
|
+
|
|
174
|
+
r'(?P<name>[a-z]+)\.dev\.example\.com': '{name}.test.example.com',
|
|
175
|
+
r'img\d+\.static\.cdn\.example\.com': 'images.cdn.example.com',
|
|
176
|
+
r'service\.(?P<env>dev|staging|prod)\.example\.org': '{env}-service.example.org',
|
|
177
|
+
r'legacy\.(?P<region>[a-z]+)\.oldsite\.com': '{region}.newsitenow.com',
|
|
178
|
+
r'shop\.(?P<country_code>[a-z]{2})\.example\.net': 'store.{country_code}.example.net',
|
|
179
|
+
r'(?P<user>[a-z]+)\.mail\.example\.com': '{user}.email.example.com',
|
|
180
|
+
r'app1\.cluster(?P<num>[0-9]+)\.example\.cloud': 'service{num}.example.cloud',
|
|
181
|
+
r'(?P<project>[a-z]+)\.research\.corp\.com': '{project}.lab.corp.com',
|
|
182
|
+
r'cdn\.(?P<version>v[0-9]+)\.content\.net': 'static.{version}.content.net',
|
|
183
|
+
# Literal rule without named group
|
|
184
|
+
r'corp\.secureaccess\.com': 'access.corp.com',
|
|
185
|
+
# Literal rule without named group
|
|
186
|
+
r'redirect\.oldsite\.org': 'homepage.newsite.org',
|
|
187
|
+
# # Recursive rules
|
|
188
|
+
|
|
189
|
+
r'archive\.(?P<year>\d{4})\.oldsite\.net': 'legacy.{year}.oldsite.net', # Recursive matching
|
|
190
|
+
r'legacy\.(?P<year>\d{4})\.oldsite\.net': 'archive-backup.{year}.net', # Continuation
|
|
191
|
+
|
|
192
|
+
# r'(?P<subd>[a-zA-Z]+)\.vpn': '{subd}.loop.ts.net',
|
|
193
|
+
# r'(?P<subd>[a-zA-Z]+)\.loop\.ts\.net': '{subd}.vpn', # Recursive loop back to .vpn
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
rewriter = Rewriter.from_data(data)
|
|
197
|
+
|
|
198
|
+
tests = [
|
|
199
|
+
"sales.vpn",
|
|
200
|
+
"marketing.dev.example.com",
|
|
201
|
+
"img01.static.cdn.example.com",
|
|
202
|
+
"service.dev.example.org",
|
|
203
|
+
"legacy.eu.oldsite.com",
|
|
204
|
+
"shop.us.example.net",
|
|
205
|
+
"alice.mail.example.com",
|
|
206
|
+
"app1.cluster1.example.cloud",
|
|
207
|
+
"genetics.research.corp.com",
|
|
208
|
+
"cdn.v2.content.net",
|
|
209
|
+
"corp.secureaccess.com",
|
|
210
|
+
"redirect.oldsite.org",
|
|
211
|
+
"support.bbb.ts.net",
|
|
212
|
+
"support.ccc.ts.net",
|
|
213
|
+
"archive.2022.oldsite.net",
|
|
214
|
+
"legacy.2022.oldsite.net",
|
|
215
|
+
"finance.vpn",
|
|
216
|
+
"engineering.dev.example.com",
|
|
217
|
+
"img02.static.cdn.example.com",
|
|
218
|
+
"service.staging.example.org",
|
|
219
|
+
"legacy.apac.oldsite.com",
|
|
220
|
+
"shop.uk.example.net",
|
|
221
|
+
"bob.mail.example.com",
|
|
222
|
+
"app1.cluster2.example.cloud",
|
|
223
|
+
"astrophysics.research.corp.com",
|
|
224
|
+
"cdn.v3.content.net",
|
|
225
|
+
"archive.2021.oldsite.net",
|
|
226
|
+
"legacy.2021.oldsite.net",
|
|
227
|
+
"quality.bbb.ts.net",
|
|
228
|
+
"quality.ccc.ts.net"
|
|
229
|
+
]
|
|
230
|
+
|
|
231
|
+
logger.warning('hello?')
|
|
232
|
+
for test in tests:
|
|
233
|
+
print(test)
|
|
234
|
+
text = rewriter.rewrite(test)
|
|
235
|
+
text
|
|
236
|
+
|
|
237
|
+
tests
|
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
from itertools import chain
|
|
2
|
+
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
from functools import cached_property
|
|
5
|
+
from setuptools import find_namespace_packages, find_packages
|
|
6
|
+
from typing import List, Dict
|
|
7
|
+
|
|
8
|
+
from fmtr.tools.constants import Constants
|
|
9
|
+
from fmtr.tools.path_tools import Path
|
|
10
|
+
from fmtr.tools.path_tools.path_tools import PathsBase, path
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class SetupPaths(PathsBase):
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
Canonical paths for a package.
|
|
17
|
+
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
SKIP_DIRS = {'data'}
|
|
21
|
+
|
|
22
|
+
def __init__(self, path=None):
|
|
23
|
+
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
Use calling module path as default path, if not otherwise specified.
|
|
27
|
+
|
|
28
|
+
"""
|
|
29
|
+
if not path:
|
|
30
|
+
path = self.from_caller()
|
|
31
|
+
|
|
32
|
+
self.repo = Path(path)
|
|
33
|
+
|
|
34
|
+
@property
|
|
35
|
+
def path(self):
|
|
36
|
+
if self.is_namespace:
|
|
37
|
+
return self.repo / self.org / self.name
|
|
38
|
+
else:
|
|
39
|
+
return self.repo / self.name
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def readme(self):
|
|
43
|
+
return self.repo / 'README.md'
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def version(self):
|
|
47
|
+
return self.path / Constants.FILENAME_VERSION
|
|
48
|
+
|
|
49
|
+
@cached_property
|
|
50
|
+
def layout(self):
|
|
51
|
+
|
|
52
|
+
directories = [
|
|
53
|
+
dir for dir in self.repo.iterdir()
|
|
54
|
+
if dir.is_dir() and not dir.name.startswith('.') and dir.name not in self.SKIP_DIRS
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
if len(directories) != 1:
|
|
58
|
+
raise ValueError(f'Expected exactly one directory in "{path}", found {','.join(directories)}')
|
|
59
|
+
|
|
60
|
+
target = next(iter(directories))
|
|
61
|
+
|
|
62
|
+
contents = list(target.iterdir())
|
|
63
|
+
if len(contents) == 1 and (item := next(iter(contents))).is_dir():
|
|
64
|
+
return True, target.name, item.name
|
|
65
|
+
|
|
66
|
+
else:
|
|
67
|
+
return False, None, target.name
|
|
68
|
+
|
|
69
|
+
@property
|
|
70
|
+
def is_namespace(self) -> str:
|
|
71
|
+
is_namespace, org, name = self.layout
|
|
72
|
+
return is_namespace
|
|
73
|
+
|
|
74
|
+
@property
|
|
75
|
+
def org(self) -> str:
|
|
76
|
+
is_namespace, org, name = self.layout
|
|
77
|
+
return org
|
|
78
|
+
|
|
79
|
+
@property
|
|
80
|
+
def name(self) -> str:
|
|
81
|
+
is_namespace, org, name = self.layout
|
|
82
|
+
return name
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class Setup:
|
|
86
|
+
AUTHOR = 'Frontmatter'
|
|
87
|
+
AUTHOR_EMAIL = 'innovative.fowler@mask.pro.fmtr.dev'
|
|
88
|
+
|
|
89
|
+
paths = SetupPaths(path=Path(__file__).absolute().parent.parent.parent)
|
|
90
|
+
|
|
91
|
+
def __init__(self, dependencies, console_scripts=None, client=None, **kwargs):
|
|
92
|
+
|
|
93
|
+
self.client = client
|
|
94
|
+
self.kwargs = kwargs
|
|
95
|
+
self.dependencies = dependencies
|
|
96
|
+
self.paths
|
|
97
|
+
|
|
98
|
+
self.console_scripts = console_scripts
|
|
99
|
+
|
|
100
|
+
def get_entrypoint_path(self, key, value):
|
|
101
|
+
if value:
|
|
102
|
+
return f'{self.name}.{value}:{key}'
|
|
103
|
+
else:
|
|
104
|
+
return f'{self.name}:{key}'
|
|
105
|
+
|
|
106
|
+
@property
|
|
107
|
+
def entrypoints(self):
|
|
108
|
+
if self.console_scripts:
|
|
109
|
+
return dict(
|
|
110
|
+
console_scripts=[f'{key} = {self.get_entrypoint_path(key, value)}' for key, value in self.console_scripts.items()],
|
|
111
|
+
)
|
|
112
|
+
else:
|
|
113
|
+
return dict()
|
|
114
|
+
|
|
115
|
+
@property
|
|
116
|
+
def name(self):
|
|
117
|
+
if self.paths.is_namespace:
|
|
118
|
+
return f'{self.paths.org}.{self.paths.name}'
|
|
119
|
+
return self.paths.name
|
|
120
|
+
|
|
121
|
+
@property
|
|
122
|
+
def author(self):
|
|
123
|
+
if self.client:
|
|
124
|
+
return f'{self.AUTHOR} on behalf of {self.client}'
|
|
125
|
+
return self.AUTHOR
|
|
126
|
+
|
|
127
|
+
@property
|
|
128
|
+
def copyright(self):
|
|
129
|
+
if self.client:
|
|
130
|
+
return self.client
|
|
131
|
+
return self.AUTHOR
|
|
132
|
+
|
|
133
|
+
@property
|
|
134
|
+
def long_description(self):
|
|
135
|
+
|
|
136
|
+
return self.paths.readme.read_text()
|
|
137
|
+
|
|
138
|
+
@property
|
|
139
|
+
def version(self):
|
|
140
|
+
return self.paths.version.read_text().strip()
|
|
141
|
+
|
|
142
|
+
@property
|
|
143
|
+
def packages(self):
|
|
144
|
+
if self.paths.is_namespace:
|
|
145
|
+
return find_namespace_packages(where=str(self.paths.repo))
|
|
146
|
+
else:
|
|
147
|
+
return find_packages(where=str(self.paths.repo))
|
|
148
|
+
|
|
149
|
+
@property
|
|
150
|
+
def package_dir(self):
|
|
151
|
+
if self.paths.is_namespace:
|
|
152
|
+
return {'': str(self.paths.repo)}
|
|
153
|
+
else:
|
|
154
|
+
return None
|
|
155
|
+
|
|
156
|
+
@property
|
|
157
|
+
def package_data(self):
|
|
158
|
+
return {self.name: [Constants.FILENAME_VERSION]}
|
|
159
|
+
|
|
160
|
+
@property
|
|
161
|
+
def url(self):
|
|
162
|
+
return f'https://github.com/{self.paths.org}/{self.paths.name}'
|
|
163
|
+
|
|
164
|
+
def get_data_setup(self):
|
|
165
|
+
return dict(
|
|
166
|
+
name=self.name,
|
|
167
|
+
version=self.version,
|
|
168
|
+
author=self.author,
|
|
169
|
+
author_email=self.AUTHOR_EMAIL,
|
|
170
|
+
url=self.url,
|
|
171
|
+
license=f'Copyright © {datetime.now().year} {self.copyright}. All rights reserved.',
|
|
172
|
+
long_description=self.long_description,
|
|
173
|
+
long_description_content_type='text/markdown',
|
|
174
|
+
packages=self.packages,
|
|
175
|
+
package_dir=self.package_dir,
|
|
176
|
+
package_data=self.package_data,
|
|
177
|
+
entry_points=self.entrypoints,
|
|
178
|
+
install_requires=self.dependencies.install,
|
|
179
|
+
extras_require=self.dependencies.extras,
|
|
180
|
+
) | self.kwargs
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
class Entrypoints:
|
|
184
|
+
ALL = 'all'
|
|
185
|
+
INSTALL = 'install'
|
|
186
|
+
|
|
187
|
+
def __init__(self, console_scripts=None, **kwargs):
|
|
188
|
+
self.kwargs = kwargs
|
|
189
|
+
self._console_scripts = console_scripts
|
|
190
|
+
|
|
191
|
+
@property
|
|
192
|
+
def console_scripts(self):
|
|
193
|
+
return [f'{key} = {value}:{key}' for key, value in self._console_scripts.items()]
|
|
194
|
+
|
|
195
|
+
@property
|
|
196
|
+
def data(self):
|
|
197
|
+
return dict(
|
|
198
|
+
console_scripts=self.console_scripts,
|
|
199
|
+
) | self.kwargs
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
class Dependencies:
|
|
203
|
+
ALL = 'all'
|
|
204
|
+
INSTALL = 'install'
|
|
205
|
+
|
|
206
|
+
def __init__(self, **kwargs):
|
|
207
|
+
self.dependencies = kwargs
|
|
208
|
+
|
|
209
|
+
def resolve_values(self, key) -> List[str]:
|
|
210
|
+
"""
|
|
211
|
+
|
|
212
|
+
Flatten a list of values.
|
|
213
|
+
|
|
214
|
+
"""
|
|
215
|
+
values_resolved = []
|
|
216
|
+
values = self.dependencies[key]
|
|
217
|
+
|
|
218
|
+
for value in values:
|
|
219
|
+
if value == key or value not in self.dependencies:
|
|
220
|
+
# Add the value directly if it references itself or is not a dependency key.
|
|
221
|
+
values_resolved.append(value)
|
|
222
|
+
else:
|
|
223
|
+
# Recurse into nested dependencies.
|
|
224
|
+
values_resolved += self.resolve_values(value)
|
|
225
|
+
|
|
226
|
+
return values_resolved
|
|
227
|
+
|
|
228
|
+
@property
|
|
229
|
+
def extras(self) -> Dict[str, List[str]]:
|
|
230
|
+
"""
|
|
231
|
+
|
|
232
|
+
Flatten dependencies.
|
|
233
|
+
|
|
234
|
+
"""
|
|
235
|
+
resolved = {key: self.resolve_values(key) for key in self.dependencies.keys()}
|
|
236
|
+
resolved.pop(self.INSTALL, None)
|
|
237
|
+
resolved[self.ALL] = list(set(chain.from_iterable(resolved.values())))
|
|
238
|
+
return resolved
|
|
239
|
+
|
|
240
|
+
@property
|
|
241
|
+
def install(self):
|
|
242
|
+
return self.resolve_values(self.INSTALL)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
if __name__ == '__main__':
|
|
246
|
+
ds = Dependencies(
|
|
247
|
+
install=['version', 'yaml'],
|
|
248
|
+
|
|
249
|
+
yaml=['yamlscript', 'pyyaml'],
|
|
250
|
+
logging=['logfire', 'version'],
|
|
251
|
+
version=['semver', 'av'],
|
|
252
|
+
av=['av']
|
|
253
|
+
# Add the rest of your dependencies...
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
ds
|
|
257
|
+
|
|
258
|
+
setup = Setup(
|
|
259
|
+
# client='Acme',
|
|
260
|
+
dependencies=ds,
|
|
261
|
+
description='some tools test',
|
|
262
|
+
console_scripts=dict(
|
|
263
|
+
cache_hfh='console_script_tools',
|
|
264
|
+
test=None,
|
|
265
|
+
)
|
|
266
|
+
)
|
|
267
|
+
data = setup.get_data_setup()
|
|
268
|
+
data
|
fmtr/tools/string_tools.py
CHANGED
fmtr/tools/version
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
1.1.
|
|
1
|
+
1.1.17
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fmtr.tools
|
|
3
|
-
Version: 1.1.
|
|
3
|
+
Version: 1.1.17
|
|
4
4
|
Summary: Collection of high-level tools to simplify everyday development tasks, with a focus on AI/ML
|
|
5
5
|
Home-page: https://github.com/fmtr/fmtr.tools
|
|
6
6
|
Author: Frontmatter
|
|
@@ -9,54 +9,60 @@ License: Copyright © 2025 Frontmatter. All rights reserved.
|
|
|
9
9
|
Description-Content-Type: text/markdown
|
|
10
10
|
License-File: LICENSE
|
|
11
11
|
Provides-Extra: test
|
|
12
|
+
Requires-Dist: fastapi; extra == "test"
|
|
13
|
+
Requires-Dist: tabulate; extra == "test"
|
|
14
|
+
Requires-Dist: torchaudio; extra == "test"
|
|
15
|
+
Requires-Dist: semver; extra == "test"
|
|
12
16
|
Requires-Dist: distributed; extra == "test"
|
|
17
|
+
Requires-Dist: filetype; extra == "test"
|
|
18
|
+
Requires-Dist: pandas; extra == "test"
|
|
19
|
+
Requires-Dist: dnspython[doh]; extra == "test"
|
|
20
|
+
Requires-Dist: sentence_transformers; extra == "test"
|
|
21
|
+
Requires-Dist: logfire[httpx]; extra == "test"
|
|
22
|
+
Requires-Dist: transformers[sentencepiece]; extra == "test"
|
|
23
|
+
Requires-Dist: openai; extra == "test"
|
|
24
|
+
Requires-Dist: regex; extra == "test"
|
|
25
|
+
Requires-Dist: contexttimer; extra == "test"
|
|
13
26
|
Requires-Dist: uvicorn[standard]; extra == "test"
|
|
14
|
-
Requires-Dist:
|
|
27
|
+
Requires-Dist: flet-webview; extra == "test"
|
|
28
|
+
Requires-Dist: pyyaml; extra == "test"
|
|
29
|
+
Requires-Dist: tinynetrc; extra == "test"
|
|
30
|
+
Requires-Dist: pytest-cov; extra == "test"
|
|
15
31
|
Requires-Dist: tokenizers; extra == "test"
|
|
16
|
-
Requires-Dist:
|
|
17
|
-
Requires-Dist:
|
|
18
|
-
Requires-Dist:
|
|
32
|
+
Requires-Dist: logfire[fastapi]; extra == "test"
|
|
33
|
+
Requires-Dist: json_repair; extra == "test"
|
|
34
|
+
Requires-Dist: faker; extra == "test"
|
|
19
35
|
Requires-Dist: huggingface_hub; extra == "test"
|
|
20
|
-
Requires-Dist:
|
|
21
|
-
Requires-Dist:
|
|
36
|
+
Requires-Dist: flet[all]; extra == "test"
|
|
37
|
+
Requires-Dist: torchvision; extra == "test"
|
|
38
|
+
Requires-Dist: httpx; extra == "test"
|
|
39
|
+
Requires-Dist: pydantic-settings; extra == "test"
|
|
40
|
+
Requires-Dist: google-auth-oauthlib; extra == "test"
|
|
41
|
+
Requires-Dist: setuptools; extra == "test"
|
|
42
|
+
Requires-Dist: pydantic-ai[logfire,openai]; extra == "test"
|
|
43
|
+
Requires-Dist: logfire; extra == "test"
|
|
44
|
+
Requires-Dist: dask[bag]; extra == "test"
|
|
22
45
|
Requires-Dist: html2text; extra == "test"
|
|
23
|
-
Requires-Dist:
|
|
24
|
-
Requires-Dist:
|
|
25
|
-
Requires-Dist:
|
|
46
|
+
Requires-Dist: yamlscript; extra == "test"
|
|
47
|
+
Requires-Dist: pymupdf4llm; extra == "test"
|
|
48
|
+
Requires-Dist: peft; extra == "test"
|
|
49
|
+
Requires-Dist: bokeh; extra == "test"
|
|
26
50
|
Requires-Dist: Unidecode; extra == "test"
|
|
27
|
-
Requires-Dist:
|
|
28
|
-
Requires-Dist:
|
|
29
|
-
Requires-Dist:
|
|
30
|
-
Requires-Dist: pydantic-settings; extra == "test"
|
|
31
|
-
Requires-Dist: contexttimer; extra == "test"
|
|
51
|
+
Requires-Dist: google-auth; extra == "test"
|
|
52
|
+
Requires-Dist: appdirs; extra == "test"
|
|
53
|
+
Requires-Dist: openpyxl; extra == "test"
|
|
32
54
|
Requires-Dist: diskcache; extra == "test"
|
|
33
|
-
Requires-Dist:
|
|
34
|
-
Requires-Dist: logfire; extra == "test"
|
|
35
|
-
Requires-Dist: openai; extra == "test"
|
|
55
|
+
Requires-Dist: pydantic; extra == "test"
|
|
36
56
|
Requires-Dist: pydevd-pycharm; extra == "test"
|
|
37
|
-
Requires-Dist: pandas; extra == "test"
|
|
38
|
-
Requires-Dist: flet-video; extra == "test"
|
|
39
|
-
Requires-Dist: yamlscript; extra == "test"
|
|
40
57
|
Requires-Dist: sre_yield; extra == "test"
|
|
41
|
-
Requires-Dist:
|
|
58
|
+
Requires-Dist: docker; extra == "test"
|
|
59
|
+
Requires-Dist: ollama; extra == "test"
|
|
42
60
|
Requires-Dist: google-api-python-client; extra == "test"
|
|
43
|
-
Requires-Dist:
|
|
44
|
-
Requires-Dist:
|
|
45
|
-
Requires-Dist:
|
|
46
|
-
Requires-Dist: openpyxl; extra == "test"
|
|
47
|
-
Requires-Dist: flet[all]; extra == "test"
|
|
48
|
-
Requires-Dist: bokeh; extra == "test"
|
|
49
|
-
Requires-Dist: pyyaml; extra == "test"
|
|
50
|
-
Requires-Dist: json_repair; extra == "test"
|
|
51
|
-
Requires-Dist: pymupdf4llm; extra == "test"
|
|
52
|
-
Requires-Dist: peft; extra == "test"
|
|
61
|
+
Requires-Dist: pymupdf; extra == "test"
|
|
62
|
+
Requires-Dist: httpx_retries; extra == "test"
|
|
63
|
+
Requires-Dist: flet-video; extra == "test"
|
|
53
64
|
Requires-Dist: deepmerge; extra == "test"
|
|
54
|
-
Requires-Dist:
|
|
55
|
-
Requires-Dist: faker; extra == "test"
|
|
56
|
-
Requires-Dist: tinynetrc; extra == "test"
|
|
57
|
-
Requires-Dist: google-auth-oauthlib; extra == "test"
|
|
58
|
-
Requires-Dist: semver; extra == "test"
|
|
59
|
-
Requires-Dist: transformers[sentencepiece]; extra == "test"
|
|
65
|
+
Requires-Dist: google-auth-httplib2; extra == "test"
|
|
60
66
|
Provides-Extra: yaml
|
|
61
67
|
Requires-Dist: yamlscript; extra == "yaml"
|
|
62
68
|
Requires-Dist: pyyaml; extra == "yaml"
|
|
@@ -157,6 +163,18 @@ Provides-Extra: path-app
|
|
|
157
163
|
Requires-Dist: appdirs; extra == "path-app"
|
|
158
164
|
Provides-Extra: path-type
|
|
159
165
|
Requires-Dist: filetype; extra == "path-type"
|
|
166
|
+
Provides-Extra: dns
|
|
167
|
+
Requires-Dist: dnspython[doh]; extra == "dns"
|
|
168
|
+
Provides-Extra: patterns
|
|
169
|
+
Requires-Dist: regex; extra == "patterns"
|
|
170
|
+
Provides-Extra: http
|
|
171
|
+
Requires-Dist: httpx; extra == "http"
|
|
172
|
+
Requires-Dist: httpx_retries; extra == "http"
|
|
173
|
+
Requires-Dist: logfire; extra == "http"
|
|
174
|
+
Requires-Dist: semver; extra == "http"
|
|
175
|
+
Requires-Dist: logfire[httpx]; extra == "http"
|
|
176
|
+
Provides-Extra: setup
|
|
177
|
+
Requires-Dist: setuptools; extra == "setup"
|
|
160
178
|
Dynamic: author
|
|
161
179
|
Dynamic: author-email
|
|
162
180
|
Dynamic: description
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
fmtr/tools/__init__.py,sha256=
|
|
1
|
+
fmtr/tools/__init__.py,sha256=KFSvu0OIyX5QsE4_rkFrFhhSxyvYFTaGsLemJqvNAys,5683
|
|
2
2
|
fmtr/tools/api_tools.py,sha256=w8Zrp_EwN5KlUghwLoTCXo4z1irg5tAsReCqDLjASfE,2133
|
|
3
3
|
fmtr/tools/async_tools.py,sha256=ewz757WcveQJd-G5SVr2JDOQVbdLGecCgl-tsBGVZz4,284
|
|
4
4
|
fmtr/tools/augmentation_tools.py,sha256=-6ESbO4CDlKqVOV1J1V6qBeoBMzbFIinkDHRHnCBej0,55
|
|
@@ -9,6 +9,7 @@ fmtr/tools/data_modelling_tools.py,sha256=0BFm-F_cYzVTxftWQwORkPd0FM2BTLVh9-s0-r
|
|
|
9
9
|
fmtr/tools/dataclass_tools.py,sha256=0Gt6KeLhtPgubo_2tYkIVqB8oQ91Qzag8OAGZDdjvMU,1209
|
|
10
10
|
fmtr/tools/datatype_tools.py,sha256=3P4AWIFGkJ-UqvXlj0Jc9IvkIIgTOE9jRrOk3NVbpH8,1508
|
|
11
11
|
fmtr/tools/debugging_tools.py,sha256=_xzqS0V5BpL8d06j-jVQjGgI7T020QsqVXKAKMz7Du8,2082
|
|
12
|
+
fmtr/tools/dns_tools.py,sha256=oWX4w33NKbMTJM7fUlg7adwVi5beUpWs8NKDnsGzNFM,1781
|
|
12
13
|
fmtr/tools/docker_tools.py,sha256=rdaZje2xhlmnfQqZnR7IHgRdWncTLjrJcViUTt5oEwk,617
|
|
13
14
|
fmtr/tools/environment_tools.py,sha256=ZO2e8pTzbQ8jNXnmKpaZF7_3qkVM6U5iqku5YSwOszE,1849
|
|
14
15
|
fmtr/tools/function_tools.py,sha256=_oW3-HZXMst2pcU-I-U7KTMmzo0g9MIQKmX-c2_NEoE,858
|
|
@@ -30,6 +31,7 @@ fmtr/tools/netrc_tools.py,sha256=PpNpz_mWlQi6VHGromKwFfTyLpHUXsd4LY6-OKLCbeI,376
|
|
|
30
31
|
fmtr/tools/openai_tools.py,sha256=6SUgejgzUzmlKKct2_ePXntvMegu3FJgfk9x7aqtqYc,742
|
|
31
32
|
fmtr/tools/packaging_tools.py,sha256=FlgOTnDRHZWQL2iR-wucTsyGEHRE-MlddKL30MPmUqE,253
|
|
32
33
|
fmtr/tools/parallel_tools.py,sha256=QEb_gN1StkxsqYaH4HSjiJX8Y3gpb2uKNsOzG4uFpaM,3071
|
|
34
|
+
fmtr/tools/pattern_tools.py,sha256=of-a26bRf-DgbzI0xJWSNkAONZozYylE_9E1hmpp6mw,6793
|
|
33
35
|
fmtr/tools/pdf_tools.py,sha256=xvv9B84uAF81rFJRnXhSsxYuP42vY9ZdPVFrSMVe8G8,4069
|
|
34
36
|
fmtr/tools/platform_tools.py,sha256=7p69CmAHe_sF68Fx9uVhns1k5EewTHTWgUYzkl6ZQKA,308
|
|
35
37
|
fmtr/tools/process_tools.py,sha256=Ysh5Dk2QFBhXQerArjKdt7xZd3JrN5Ho02AaOjH0Nnw,1425
|
|
@@ -37,13 +39,14 @@ fmtr/tools/profiling_tools.py,sha256=jpXVjaNKPydTasEQVNXvxzGtMhXPit08AnJddkU8uIc
|
|
|
37
39
|
fmtr/tools/random_tools.py,sha256=4VlQdk5THbR8ka4pZaLbk_ZO_4yy6PF_lHZes_rgenY,2223
|
|
38
40
|
fmtr/tools/semantic_tools.py,sha256=cxY9NSAHWj4nEc6Oj4qA1omR3dWbl2OuH7_PkINc6_E,1386
|
|
39
41
|
fmtr/tools/settings_tools.py,sha256=o11W3T60UZSvCTkh_eEQq1Mx74GycQ6JxUr0plBDbsk,2356
|
|
42
|
+
fmtr/tools/setup_tools.py,sha256=JymAPBPgHlF2gw9d4TrLT2LvPS9B2riGyJ2suD2gw6U,6944
|
|
40
43
|
fmtr/tools/spaces_tools.py,sha256=D_he3mve6DruB3OPS6QyzqD05ChHnRTb4buViKPe7To,1099
|
|
41
|
-
fmtr/tools/string_tools.py,sha256=
|
|
44
|
+
fmtr/tools/string_tools.py,sha256=mjO7NSqhtYMtnX7fKb8eXZ7MO2lXWGMl0kXxnZznupM,3764
|
|
42
45
|
fmtr/tools/tabular_tools.py,sha256=tpIpZzYku1HcJrHZJL6BC39LmN3WUWVhFbK2N7nDVmE,120
|
|
43
46
|
fmtr/tools/tokenization_tools.py,sha256=me-IBzSLyNYejLybwjO9CNB6Mj2NYfKPaOVThXyaGNg,4268
|
|
44
47
|
fmtr/tools/tools.py,sha256=CAsApa1YwVdNE6H66Vjivs_mXYvOas3rh7fPELAnTpk,795
|
|
45
48
|
fmtr/tools/unicode_tools.py,sha256=yS_9wpu8ogNoiIL7s1G_8bETFFO_YQlo4LNPv1NLDeY,52
|
|
46
|
-
fmtr/tools/version,sha256=
|
|
49
|
+
fmtr/tools/version,sha256=GkdkUXF_QoQgGaivlyso9zz8QA1rg2DseoZxAQwvOEc,6
|
|
47
50
|
fmtr/tools/version_tools.py,sha256=yNs_CGqWpqE4jbK9wsPIi14peJVXYbhIcMqHAFOw3yE,1480
|
|
48
51
|
fmtr/tools/yaml_tools.py,sha256=9kuYChqJelWQIjGlSnK4iDdOWWH06P0gp9jIcRrC3UI,1903
|
|
49
52
|
fmtr/tools/ai_tools/__init__.py,sha256=JZrLuOFNV1A3wvJgonxOgz_4WS-7MfCuowGWA5uYCjs,372
|
|
@@ -61,9 +64,9 @@ fmtr/tools/tests/test_environment.py,sha256=iHaiMQfECYZPkPKwfuIZV9uHuWe3aE-p_dN_
|
|
|
61
64
|
fmtr/tools/tests/test_json.py,sha256=IeSP4ziPvRcmS8kq7k9tHonC9rN5YYq9GSNT2ul6Msk,287
|
|
62
65
|
fmtr/tools/tests/test_path.py,sha256=AkZQa6_8BQ-VaCyL_J-iKmdf2ZaM-xFYR37Kun3k4_g,2188
|
|
63
66
|
fmtr/tools/tests/test_yaml.py,sha256=jc0TwwKu9eC0LvFGNMERdgBue591xwLxYXFbtsRwXVM,287
|
|
64
|
-
fmtr_tools-1.1.
|
|
65
|
-
fmtr_tools-1.1.
|
|
66
|
-
fmtr_tools-1.1.
|
|
67
|
-
fmtr_tools-1.1.
|
|
68
|
-
fmtr_tools-1.1.
|
|
69
|
-
fmtr_tools-1.1.
|
|
67
|
+
fmtr_tools-1.1.17.dist-info/licenses/LICENSE,sha256=FW9aa6vVN5IjRQWLT43hs4_koYSmpcbIovlKeAJ0_cI,10757
|
|
68
|
+
fmtr_tools-1.1.17.dist-info/METADATA,sha256=RrfqTgYjKQLca5VUuvhO3oGRxwLY3vXmsSG4NUaci5I,15735
|
|
69
|
+
fmtr_tools-1.1.17.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
70
|
+
fmtr_tools-1.1.17.dist-info/entry_points.txt,sha256=e-eOW1Ml13tbxHC6Er1MnVOEDePeWw7DKwlE-l-gCY0,205
|
|
71
|
+
fmtr_tools-1.1.17.dist-info/top_level.txt,sha256=LXem9xCgNOD72tE2gRKESdiQTL902mfFkwWb6-dlwEE,5
|
|
72
|
+
fmtr_tools-1.1.17.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|