fmtr.tools 1.2.1__py3-none-any.whl → 1.2.3__py3-none-any.whl

fmtr/tools/ai_tools/agentic_tools.py

@@ -1,17 +1,31 @@
-from typing import List, Optional
-
 import pydantic_ai
-from pydantic_ai import RunContext
+from pydantic_ai import RunContext, ModelRetry
 from pydantic_ai.agent import AgentRunResult, Agent
 from pydantic_ai.models.openai import OpenAIModel
 from pydantic_ai.providers.openai import OpenAIProvider
+from typing import List, Optional, Any
 
 from fmtr.tools import environment_tools as env
 from fmtr.tools.constants import Constants
+from fmtr.tools.logging_tools import logger
 from fmtr.tools.string_tools import truncate_mid
 
 pydantic_ai.Agent.instrument_all()
 
+
+class Validator:
+    """
+
+    Subclassable validator
+
+    """
+
+    async def validate(self, ctx: RunContext[Any], output: Any) -> List[str]:
+        raise NotImplementedError()
+
+
+
+
 class Task:
     """
 
@@ -31,6 +45,7 @@ class Task:
     DEPS_TYPE = str
     RESULT_TYPE = str
     RESULT_RETRIES = 5
+    VALIDATORS: List[Validator] = []
 
     def __init__(self, *args, **kwargs):
         """
@@ -77,9 +92,18 @@ class Task:
     async def validate(self, ctx: RunContext[DEPS_TYPE], output: RESULT_TYPE) -> RESULT_TYPE:
         """
 
-        Dummy validator
+        Aggregate any validation failures and combine them into a single ModelRetry exception
 
         """
+        msgs = []
+        for validator in self.VALIDATORS:
+            msgs += validator.validate(ctx, output)
+
+        if msgs:
+            msg = '. '.join(msgs)
+            logger.warning(msg)
+            raise ModelRetry(msg)
+
         return output
 
     def get_prompt(self, deps: Optional[DEPS_TYPE]) -> Optional[str]:

fmtr/tools/dns_tools.py

@@ -1,57 +1,221 @@
 import dns
 import httpx
 import socket
-from dns.message import Message, QueryMessage
+from dataclasses import dataclass
+from dns.message import Message
+from functools import cached_property
+from httpx_retries import Retry, RetryTransport
+from typing import Optional, Self
 
+from fmtr.tools import Client, logger
 
-class Server:
+RETRY_STRATEGY = Retry(
+    total=2,  # initial + 1 retry
+    allowed_methods={"GET", "POST"},
+    status_forcelist={502, 503, 504},
+    retry_on_exceptions=None,  # defaults to httpx.TransportError etc.
+    backoff_factor=0.25,  # short backoff (e.g. 0.25s, 0.5s)
+    max_backoff_wait=0.75,  # max total delay before giving up
+    backoff_jitter=0.1,  # small jitter to avoid retry bursts
+    respect_retry_after_header=False,  # DoH resolvers probably won't set this
+)
+
+
+class HTTPClientDoH(Client):
+    """
+
+    Base HTTP client for DoH-appropriate retry strategy.
+
+    """
+    TRANSPORT = RetryTransport(retry=RETRY_STRATEGY)
+
+
+@dataclass
+class BaseDNSData:
+    """
+
+    DNS response object.
+
+    """
+    wire: bytes
+
+    @cached_property
+    def message(self) -> Message:
+        return dns.message.from_wire(self.wire)
+
+    @classmethod
+    def from_message(cls, message: Message) -> Self:
+        return cls(message.to_wire())
+
+
+@dataclass
+class Response(BaseDNSData):
+    """
+
+    DNS response object.
+
+    """
+
+    http: Optional[httpx.Response] = None
+
+    @classmethod
+    def from_http(cls, response: httpx.Response) -> Self:
+        self = cls(response.content, http=response)
+        return self
+
+
+@dataclass
+class Request(BaseDNSData):
+    """
+
+    DNS request object.
+
+    """
+    wire: bytes
+
+    @cached_property
+    def question(self):
+        return self.message.question[0]
+
+    @cached_property
+    def is_valid(self):
+        return len(self.message.question) != 0
+
+    @cached_property
+    def type(self):
+        return self.question.rdtype
+
+    @cached_property
+    def type_text(self):
+        return dns.rdatatype.to_text(self.type)
+
+    @cached_property
+    def name(self):
+        return self.question.name
+
+    @cached_property
+    def name_text(self):
+        return self.name.to_text()
+
+    @cached_property
+    def blackhole(self) -> Response:
+        blackhole = dns.message.make_response(self.message)
+        blackhole.flags |= dns.flags.RA
+        blackhole.set_rcode(dns.rcode.NXDOMAIN)
+        response = Response.from_message(blackhole)
+        return response
+
+
+@dataclass
+class Exchange:
+    """
+
+    Entire DNS exchange for a DNS Proxy: request -> upstream response -> response
+
+    """
+    ip: str
+    port: int
+
+    request: Request
+    response: Optional[Response] = None
+    response_upstream: Optional[Response] = None
+
+    @classmethod
+    def from_wire(cls, wire: bytes, ip: str, port: int) -> Self:
+        request = Request(wire)
+        return cls(request=request, ip=ip, port=port)
+
+    @cached_property
+    def client(self):
+        return f'{self.ip}:{self.port}'
+
+
+class BasePlain:
+    """
+
+    Base for starting a plain DNS server
+
+    """
 
     def __init__(self, host, port):
         self.host = host
         self.port = port
         self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
 
-    def resolve(self, message: Message) -> QueryMessage:
+    def resolve(self, exchange: Exchange):
         raise NotImplemented
 
     def start(self):
+        """
+
+        Listen and resolve via overridden resolve method.
+
+        """
         sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
         sock.bind((self.host, self.port))
         print(f"Listening on {self.host}:{self.port}")
         while True:
-            data, addr = sock.recvfrom(512)
-            question = dns.message.from_wire(data)
-            answer = self.resolve(question)
-            sock.sendto(answer.to_wire(), addr)
+            data, (ip, port) = sock.recvfrom(512)
+            exchange = Exchange.from_wire(data, ip=ip, port=port)
+            self.resolve(exchange)
+            sock.sendto(exchange.response.wire, (ip, port))
+
+
+class BaseDoHProxy(BasePlain):
+    """
+
+    Base for a DNS Proxy server
+
+    """
+
+    URL = None
+    HEADERS = {"Content-Type": "application/dns-message"}
+    client = HTTPClientDoH()
+
+    def process_question(self, exchange: Exchange):
+        return
+
+    def process_upstream(self, exchange: Exchange):
+        return
+
+    def from_upstream(self, exchange: Exchange) -> Exchange:
+
+        request = exchange.request
+        response_doh = self.client.post(self.URL, headers=self.HEADERS, content=request.wire)
+        response_doh.raise_for_status()
+        response = Response.from_http(response_doh)
+        exchange.response_upstream = response
+
+        return exchange
+
+    def resolve(self, exchange: Exchange):
+        """
+
+        Resolve a request, processing each stage, initial question, upstream response etc.
+        Subclasses can override the relevant processing methods to implement custom behaviour.
 
+        """
 
-class Proxy(Server):
-    url = "https://cloudflare-dns.com/dns-query"
-    headers = {"accept": "application/dns-json"}
+        request = exchange.request
 
-    def resolve(self, question: Message) -> Message:
-        qname = question.question[0].name.to_text()
-        qtype = dns.rdatatype.to_text(question.question[0].rdtype)
-        params = {"name": qname, "type": qtype}
+        with logger.span(f'Handling request for {request.name_text} from {exchange.client}...'):
 
-        response = httpx.get(self.url, headers=self.headers, params=params)
-        response.raise_for_status()
+            if not request.is_valid:
+                raise ValueError(f'Only one question per request is supported. Got {len(request.question)} questions.')
 
-        answer = dns.message.make_response(question)
-        answer.flags |= dns.flags.RA
+            with logger.span(f'Processing question...'):
+                self.process_question(exchange)
+            if exchange.response:
+                return
 
-        data = response.json()
-        for answer_rr in data.get("Answer", []):
-            name = dns.name.from_text(answer_rr["name"])
-            rtype = answer_rr["type"]
-            ttl = answer_rr["TTL"]
-            rdata = dns.rdata.from_text(dns.rdataclass.IN, rtype, answer_rr["data"])
-            rrset = dns.rrset.from_rdata(name, ttl, rdata)
-            answer.answer.append(rrset)
+            with logger.span(f'Making upstream request for {request.name_text}...'):
+                self.from_upstream(exchange)
 
-        return answer
+            with logger.span(f'Processing upstream response...'):
+                self.process_upstream(exchange)
 
+            if exchange.response:
+                return
 
-if __name__ == "__main__":
-    proxy = Proxy("127.0.0.1", 5354)
-    proxy.start()
+            exchange.response = exchange.response_upstream
+            return

fmtr/tools/version

@@ -1 +1 @@
-1.2.1
+1.2.3

{fmtr_tools-1.2.1.dist-info → fmtr_tools-1.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fmtr.tools
-Version: 1.2.1
+Version: 1.2.3
 Summary: Collection of high-level tools to simplify everyday development tasks, with a focus on AI/ML
 Home-page: https://github.com/fmtr/tools
 Author: Frontmatter
@@ -112,6 +112,11 @@ Provides-Extra: path-type
 Requires-Dist: filetype; extra == "path-type"
 Provides-Extra: dns
 Requires-Dist: dnspython[doh]; extra == "dns"
+Requires-Dist: httpx; extra == "dns"
+Requires-Dist: httpx_retries; extra == "dns"
+Requires-Dist: logfire; extra == "dns"
+Requires-Dist: semver; extra == "dns"
+Requires-Dist: logfire[httpx]; extra == "dns"
 Provides-Extra: patterns
 Requires-Dist: regex; extra == "patterns"
 Provides-Extra: http
@@ -123,60 +128,60 @@ Requires-Dist: logfire[httpx]; extra == "http"
 Provides-Extra: setup
 Requires-Dist: setuptools; extra == "setup"
 Provides-Extra: all
-Requires-Dist: logfire[fastapi]; extra == "all"
-Requires-Dist: torchaudio; extra == "all"
+Requires-Dist: deepmerge; extra == "all"
+Requires-Dist: pydantic-settings; extra == "all"
+Requires-Dist: sentence_transformers; extra == "all"
+Requires-Dist: diskcache; extra == "all"
+Requires-Dist: google-auth; extra == "all"
+Requires-Dist: faker; extra == "all"
+Requires-Dist: tinynetrc; extra == "all"
+Requires-Dist: sre_yield; extra == "all"
+Requires-Dist: html2text; extra == "all"
+Requires-Dist: json_repair; extra == "all"
 Requires-Dist: pymupdf; extra == "all"
-Requires-Dist: ollama; extra == "all"
-Requires-Dist: transformers[sentencepiece]; extra == "all"
+Requires-Dist: peft; extra == "all"
+Requires-Dist: appdirs; extra == "all"
+Requires-Dist: google-api-python-client; extra == "all"
+Requires-Dist: distributed; extra == "all"
+Requires-Dist: openai; extra == "all"
+Requires-Dist: yamlscript; extra == "all"
+Requires-Dist: fastapi; extra == "all"
+Requires-Dist: tabulate; extra == "all"
+Requires-Dist: bokeh; extra == "all"
 Requires-Dist: uvicorn[standard]; extra == "all"
-Requires-Dist: tokenizers; extra == "all"
-Requires-Dist: tinynetrc; extra == "all"
+Requires-Dist: pydantic-ai[logfire,openai]; extra == "all"
+Requires-Dist: transformers[sentencepiece]; extra == "all"
+Requires-Dist: docker; extra == "all"
+Requires-Dist: pymupdf4llm; extra == "all"
+Requires-Dist: torchaudio; extra == "all"
+Requires-Dist: pydevd-pycharm; extra == "all"
+Requires-Dist: regex; extra == "all"
+Requires-Dist: Unidecode; extra == "all"
+Requires-Dist: huggingface_hub; extra == "all"
+Requires-Dist: contexttimer; extra == "all"
 Requires-Dist: pandas; extra == "all"
-Requires-Dist: openai; extra == "all"
-Requires-Dist: google-api-python-client; extra == "all"
-Requires-Dist: pydantic-settings; extra == "all"
-Requires-Dist: deepmerge; extra == "all"
+Requires-Dist: flet[all]; extra == "all"
+Requires-Dist: flet-video; extra == "all"
 Requires-Dist: pyyaml; extra == "all"
-Requires-Dist: httpx_retries; extra == "all"
-Requires-Dist: yamlscript; extra == "all"
+Requires-Dist: google-auth-httplib2; extra == "all"
+Requires-Dist: ollama; extra == "all"
 Requires-Dist: setuptools; extra == "all"
+Requires-Dist: logfire[httpx]; extra == "all"
+Requires-Dist: openpyxl; extra == "all"
 Requires-Dist: dask[bag]; extra == "all"
-Requires-Dist: faker; extra == "all"
-Requires-Dist: logfire; extra == "all"
-Requires-Dist: Unidecode; extra == "all"
-Requires-Dist: google-auth-oauthlib; extra == "all"
-Requires-Dist: flet-video; extra == "all"
-Requires-Dist: docker; extra == "all"
-Requires-Dist: google-auth; extra == "all"
-Requires-Dist: tabulate; extra == "all"
-Requires-Dist: pydevd-pycharm; extra == "all"
 Requires-Dist: semver; extra == "all"
-Requires-Dist: dnspython[doh]; extra == "all"
-Requires-Dist: distributed; extra == "all"
-Requires-Dist: json_repair; extra == "all"
-Requires-Dist: torchvision; extra == "all"
-Requires-Dist: diskcache; extra == "all"
-Requires-Dist: openpyxl; extra == "all"
-Requires-Dist: google-auth-httplib2; extra == "all"
-Requires-Dist: httpx; extra == "all"
-Requires-Dist: regex; extra == "all"
-Requires-Dist: bokeh; extra == "all"
-Requires-Dist: fastapi; extra == "all"
-Requires-Dist: html2text; extra == "all"
+Requires-Dist: logfire; extra == "all"
+Requires-Dist: pytest-cov; extra == "all"
 Requires-Dist: filetype; extra == "all"
-Requires-Dist: peft; extra == "all"
-Requires-Dist: pydantic-ai[logfire,openai]; extra == "all"
-Requires-Dist: pydantic; extra == "all"
-Requires-Dist: sre_yield; extra == "all"
+Requires-Dist: tokenizers; extra == "all"
 Requires-Dist: flet-webview; extra == "all"
-Requires-Dist: sentence_transformers; extra == "all"
-Requires-Dist: pytest-cov; extra == "all"
-Requires-Dist: flet[all]; extra == "all"
-Requires-Dist: logfire[httpx]; extra == "all"
-Requires-Dist: pymupdf4llm; extra == "all"
-Requires-Dist: appdirs; extra == "all"
-Requires-Dist: contexttimer; extra == "all"
-Requires-Dist: huggingface_hub; extra == "all"
+Requires-Dist: google-auth-oauthlib; extra == "all"
+Requires-Dist: httpx_retries; extra == "all"
+Requires-Dist: pydantic; extra == "all"
+Requires-Dist: logfire[fastapi]; extra == "all"
+Requires-Dist: httpx; extra == "all"
+Requires-Dist: torchvision; extra == "all"
+Requires-Dist: dnspython[doh]; extra == "all"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{fmtr_tools-1.2.1.dist-info → fmtr_tools-1.2.3.dist-info}/RECORD

@@ -8,7 +8,7 @@ fmtr/tools/data_modelling_tools.py,sha256=0BFm-F_cYzVTxftWQwORkPd0FM2BTLVh9-s0-r
 fmtr/tools/dataclass_tools.py,sha256=0Gt6KeLhtPgubo_2tYkIVqB8oQ91Qzag8OAGZDdjvMU,1209
 fmtr/tools/datatype_tools.py,sha256=3P4AWIFGkJ-UqvXlj0Jc9IvkIIgTOE9jRrOk3NVbpH8,1508
 fmtr/tools/debugging_tools.py,sha256=_xzqS0V5BpL8d06j-jVQjGgI7T020QsqVXKAKMz7Du8,2082
-fmtr/tools/dns_tools.py,sha256=oWX4w33NKbMTJM7fUlg7adwVi5beUpWs8NKDnsGzNFM,1781
+fmtr/tools/dns_tools.py,sha256=7OoELFd9lZ3ULdJ6h8QwHYFNUYS9atqYrflKEzzFW-g,5461
 fmtr/tools/docker_tools.py,sha256=rdaZje2xhlmnfQqZnR7IHgRdWncTLjrJcViUTt5oEwk,617
 fmtr/tools/environment_tools.py,sha256=ZO2e8pTzbQ8jNXnmKpaZF7_3qkVM6U5iqku5YSwOszE,1849
 fmtr/tools/function_tools.py,sha256=_oW3-HZXMst2pcU-I-U7KTMmzo0g9MIQKmX-c2_NEoE,858
@@ -45,11 +45,11 @@ fmtr/tools/tabular_tools.py,sha256=tpIpZzYku1HcJrHZJL6BC39LmN3WUWVhFbK2N7nDVmE,1
 fmtr/tools/tokenization_tools.py,sha256=me-IBzSLyNYejLybwjO9CNB6Mj2NYfKPaOVThXyaGNg,4268
 fmtr/tools/tools.py,sha256=CAsApa1YwVdNE6H66Vjivs_mXYvOas3rh7fPELAnTpk,795
 fmtr/tools/unicode_tools.py,sha256=yS_9wpu8ogNoiIL7s1G_8bETFFO_YQlo4LNPv1NLDeY,52
-fmtr/tools/version,sha256=IGYgplfxzMivcrhm_c7yDmQcSPOc4n8zjOgxOUnGPlM,5
+fmtr/tools/version,sha256=xH9bGLikMOaYuf4V5R9hGZhOeDNLzz9F4hDTDDfvL54,5
 fmtr/tools/version_tools.py,sha256=yNs_CGqWpqE4jbK9wsPIi14peJVXYbhIcMqHAFOw3yE,1480
 fmtr/tools/yaml_tools.py,sha256=9kuYChqJelWQIjGlSnK4iDdOWWH06P0gp9jIcRrC3UI,1903
 fmtr/tools/ai_tools/__init__.py,sha256=JZrLuOFNV1A3wvJgonxOgz_4WS-7MfCuowGWA5uYCjs,372
-fmtr/tools/ai_tools/agentic_tools.py,sha256=YebH7R9ovVo3GzfWAZxY49e2fNt13APDMe290xx7zks,3720
+fmtr/tools/ai_tools/agentic_tools.py,sha256=acSEPFS-aguDXanWGs3fAAlRyJSYPZW7L-Kb2qDLm-I,4300
 fmtr/tools/ai_tools/inference_tools.py,sha256=2UP2gXEyOJUjyyV6zmFIYmIxUsh1rXkRH0IbFvr2bRs,11908
 fmtr/tools/entrypoints/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fmtr/tools/entrypoints/cache_hfh.py,sha256=fQNs4J9twQuZH_Yj98-oOvEX7-LrSUP3kO8nzw2HrHs,60
@@ -70,9 +70,9 @@ fmtr/tools/tests/test_environment.py,sha256=iHaiMQfECYZPkPKwfuIZV9uHuWe3aE-p_dN_
 fmtr/tools/tests/test_json.py,sha256=IeSP4ziPvRcmS8kq7k9tHonC9rN5YYq9GSNT2ul6Msk,287
 fmtr/tools/tests/test_path.py,sha256=AkZQa6_8BQ-VaCyL_J-iKmdf2ZaM-xFYR37Kun3k4_g,2188
 fmtr/tools/tests/test_yaml.py,sha256=jc0TwwKu9eC0LvFGNMERdgBue591xwLxYXFbtsRwXVM,287
-fmtr_tools-1.2.1.dist-info/licenses/LICENSE,sha256=FW9aa6vVN5IjRQWLT43hs4_koYSmpcbIovlKeAJ0_cI,10757
-fmtr_tools-1.2.1.dist-info/METADATA,sha256=jNFEE6r7_TrdiJIO4PjjyPtPOgWHJ2IJOKmPHMG8MjM,15738
-fmtr_tools-1.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-fmtr_tools-1.2.1.dist-info/entry_points.txt,sha256=fSQrDGNctdQXbUxpMWYVfVQ0mhZMDyaEDG3D3a0zOSc,278
-fmtr_tools-1.2.1.dist-info/top_level.txt,sha256=LXem9xCgNOD72tE2gRKESdiQTL902mfFkwWb6-dlwEE,5
-fmtr_tools-1.2.1.dist-info/RECORD,,
+fmtr_tools-1.2.3.dist-info/licenses/LICENSE,sha256=FW9aa6vVN5IjRQWLT43hs4_koYSmpcbIovlKeAJ0_cI,10757
+fmtr_tools-1.2.3.dist-info/METADATA,sha256=ogcH2Hf0wFsWkOT5H3KOLMlR_7DNWGl_BxecAmUthZc,15943
+fmtr_tools-1.2.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+fmtr_tools-1.2.3.dist-info/entry_points.txt,sha256=fSQrDGNctdQXbUxpMWYVfVQ0mhZMDyaEDG3D3a0zOSc,278
+fmtr_tools-1.2.3.dist-info/top_level.txt,sha256=LXem9xCgNOD72tE2gRKESdiQTL902mfFkwWb6-dlwEE,5
+fmtr_tools-1.2.3.dist-info/RECORD,,