fmtr.tools 1.3.15__tar.gz → 1.3.17__tar.gz

{fmtr_tools-1.3.15 → fmtr_tools-1.3.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fmtr.tools
-Version: 1.3.15
+Version: 1.3.17
 Summary: Collection of high-level tools to simplify everyday development tasks, with a focus on AI/ML
 Home-page: https://github.com/fmtr/fmtr.tools
 Author: Frontmatter
@@ -126,61 +126,61 @@ Requires-Dist: logfire[httpx]; extra == "http"
 Provides-Extra: setup
 Requires-Dist: setuptools; extra == "setup"
 Provides-Extra: all
-Requires-Dist: flet-video; extra == "all"
-Requires-Dist: regex; extra == "all"
-Requires-Dist: dnspython[doh]; extra == "all"
-Requires-Dist: distributed; extra == "all"
-Requires-Dist: tinynetrc; extra == "all"
-Requires-Dist: pandas; extra == "all"
-Requires-Dist: contexttimer; extra == "all"
-Requires-Dist: peft; extra == "all"
-Requires-Dist: pydantic; extra == "all"
-Requires-Dist: google-auth; extra == "all"
-Requires-Dist: tokenizers; extra == "all"
-Requires-Dist: diskcache; extra == "all"
-Requires-Dist: cachetools; extra == "all"
-Requires-Dist: pymupdf; extra == "all"
-Requires-Dist: bokeh; extra == "all"
-Requires-Dist: tabulate; extra == "all"
-Requires-Dist: ollama; extra == "all"
 Requires-Dist: Unidecode; extra == "all"
-Requires-Dist: logfire; extra == "all"
-Requires-Dist: sre_yield; extra == "all"
-Requires-Dist: pyyaml; extra == "all"
-Requires-Dist: deepmerge; extra == "all"
-Requires-Dist: huggingface_hub; extra == "all"
-Requires-Dist: openai; extra == "all"
-Requires-Dist: sentence_transformers; extra == "all"
-Requires-Dist: openpyxl; extra == "all"
+Requires-Dist: semver; extra == "all"
+Requires-Dist: google-auth-httplib2; extra == "all"
+Requires-Dist: setuptools; extra == "all"
+Requires-Dist: dask[bag]; extra == "all"
 Requires-Dist: httpx_retries; extra == "all"
 Requires-Dist: flet-webview; extra == "all"
-Requires-Dist: pytest-cov; extra == "all"
+Requires-Dist: docker; extra == "all"
+Requires-Dist: bokeh; extra == "all"
+Requires-Dist: filetype; extra == "all"
 Requires-Dist: torchvision; extra == "all"
-Requires-Dist: torchaudio; extra == "all"
 Requires-Dist: faker; extra == "all"
-Requires-Dist: logfire[httpx]; extra == "all"
-Requires-Dist: flet[all]; extra == "all"
-Requires-Dist: google-api-python-client; extra == "all"
-Requires-Dist: json_repair; extra == "all"
-Requires-Dist: google-auth-oauthlib; extra == "all"
+Requires-Dist: uvicorn[standard]; extra == "all"
 Requires-Dist: fastapi; extra == "all"
-Requires-Dist: yamlscript; extra == "all"
-Requires-Dist: setuptools; extra == "all"
-Requires-Dist: semver; extra == "all"
-Requires-Dist: pymupdf4llm; extra == "all"
-Requires-Dist: pydevd-pycharm~=251.25410.159; extra == "all"
+Requires-Dist: regex; extra == "all"
+Requires-Dist: openai; extra == "all"
 Requires-Dist: appdirs; extra == "all"
-Requires-Dist: google-auth-httplib2; extra == "all"
-Requires-Dist: pydantic-settings; extra == "all"
-Requires-Dist: uvicorn[standard]; extra == "all"
 Requires-Dist: httpx; extra == "all"
-Requires-Dist: docker; extra == "all"
-Requires-Dist: transformers[sentencepiece]; extra == "all"
-Requires-Dist: dask[bag]; extra == "all"
-Requires-Dist: filetype; extra == "all"
-Requires-Dist: logfire[fastapi]; extra == "all"
+Requires-Dist: distributed; extra == "all"
+Requires-Dist: google-api-python-client; extra == "all"
+Requires-Dist: peft; extra == "all"
+Requires-Dist: google-auth-oauthlib; extra == "all"
+Requires-Dist: pyyaml; extra == "all"
+Requires-Dist: json_repair; extra == "all"
+Requires-Dist: deepmerge; extra == "all"
+Requires-Dist: cachetools; extra == "all"
+Requires-Dist: flet[all]; extra == "all"
+Requires-Dist: pydevd-pycharm~=251.25410.159; extra == "all"
 Requires-Dist: html2text; extra == "all"
+Requires-Dist: ollama; extra == "all"
+Requires-Dist: pandas; extra == "all"
+Requires-Dist: pymupdf4llm; extra == "all"
+Requires-Dist: pydantic-settings; extra == "all"
+Requires-Dist: flet-video; extra == "all"
+Requires-Dist: sentence_transformers; extra == "all"
+Requires-Dist: pymupdf; extra == "all"
+Requires-Dist: logfire[httpx]; extra == "all"
+Requires-Dist: huggingface_hub; extra == "all"
+Requires-Dist: openpyxl; extra == "all"
+Requires-Dist: contexttimer; extra == "all"
+Requires-Dist: yamlscript; extra == "all"
+Requires-Dist: pydantic; extra == "all"
+Requires-Dist: sre_yield; extra == "all"
 Requires-Dist: pydantic-ai[logfire,openai]; extra == "all"
+Requires-Dist: tokenizers; extra == "all"
+Requires-Dist: torchaudio; extra == "all"
+Requires-Dist: logfire[fastapi]; extra == "all"
+Requires-Dist: pytest-cov; extra == "all"
+Requires-Dist: tinynetrc; extra == "all"
+Requires-Dist: tabulate; extra == "all"
+Requires-Dist: diskcache; extra == "all"
+Requires-Dist: transformers[sentencepiece]; extra == "all"
+Requires-Dist: google-auth; extra == "all"
+Requires-Dist: logfire; extra == "all"
+Requires-Dist: dnspython[doh]; extra == "all"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{fmtr_tools-1.3.15 → fmtr_tools-1.3.17}/fmtr/tools/dns_tools/dm.py

@@ -10,6 +10,19 @@ from typing import Self, Optional, List
 
 from fmtr.tools.string_tools import join
 
+TTL_CODE_DEFAULTS = {
+    dnsrcode.NOERROR: 300,  # Successful query
+    dnsrcode.FORMERR: 60,  # Format error
+    dnsrcode.SERVFAIL: 10,  # Server failure
+    dnsrcode.NXDOMAIN: 60 * 60,  # Non-existent domain
+    dnsrcode.NOTIMP: 60,  # Not implemented
+    dnsrcode.REFUSED: 60,  # Refused
+    dnsrcode.YXDOMAIN: 600,  # Name exists when it should not
+    dnsrcode.YXRRSET: 600,  # RR Set exists when it should not
+    dnsrcode.NXRRSET: 300,  # RR Set that should exist does not
+    dnsrcode.NOTAUTH: 60,  # Not authorized
+    dnsrcode.NOTZONE: 60  # Name not contained in zone
+}
 
 @dataclass
 class BaseDNSData:
@@ -68,6 +81,24 @@ class Response(BaseDNSData):
     def rcode_text(self) -> str:
         return dnsrcode.to_text(self.rcode)
 
+    @property
+    def ttl(self) -> int:
+        """
+
+        Get median TTL from answers, falling back to authority, then to error-code defaults.
+
+        """
+        answers = self.message.answer or self.message.authority
+        if answers:
+            ttls = [answer.ttl for answer in answers]
+            ttl = min(ttls)
+            return ttl
+
+        ttl = TTL_CODE_DEFAULTS.get(self.rcode, dnsrcode.NXDOMAIN)
+        return ttl
+
+
+
     def __str__(self):
         """
 
@@ -148,6 +179,9 @@ class Exchange:
     client_name: Optional[str] = None
     is_complete: bool = False
 
+    @property
+    def addr(self):
+        return self.ip, self.port
 
     @classmethod
     def from_wire(cls, wire: bytes, **kwargs) -> Self:
@@ -163,26 +197,25 @@ class Exchange:
     def question_last(self) -> RRset:
         """
 
-        Contrive an RRset representing the latest/current question. This can be the original question - or a hybrid one if we've injected our own answers into the exchange.
+        Create an RRset surrogate representing the latest/current question. This can be the original question - or a hybrid one if we've injected our own answers into the Exchange.
 
         """
         if self.answers_pre:
             rrset = self.answers_pre[-1]
-            ty = self.request.type
+            rdtype = self.request.type
             ttl = self.request.question.ttl
             rdclass = self.request.question.rdclass
             name = next(iter(rrset.items.keys())).to_text()
-
-            rrset_contrived = dns.rrset.from_text(
+            rrset_surrogate = dns.rrset.from_text(
                 name=name,
                 ttl=ttl,
-                rdtype=ty,
+                rdtype=rdtype,
                 rdclass=rdclass,
             )
 
-            return rrset_contrived
+            return rrset_surrogate
         else:
-            return self.request.question  # Solves the issue of digging out the name.
+            return self.request.question
 
     @property
     def query_last(self) -> QueryMessage:

{fmtr_tools-1.3.15 → fmtr_tools-1.3.17}/fmtr/tools/dns_tools/proxy.py

@@ -39,7 +39,7 @@ class Proxy(server.Plain):
         """
         exchange.is_complete = True
 
-    def resolve(self, exchange: Exchange) -> Exchange:
+    async def resolve(self, exchange: Exchange) -> Exchange:
         """
 
         Resolve a request, processing each stage, initial question, upstream response etc.

{fmtr_tools-1.3.15 → fmtr_tools-1.3.17}/fmtr/tools/dns_tools/server.py

@@ -1,7 +1,9 @@
-import socket
-from dataclasses import dataclass
+import asyncio
+import dns.rcode
+from dataclasses import dataclass, field
 from datetime import timedelta
 from functools import cached_property
+from typing import Optional
 
 from fmtr.tools import caching_tools as caching
 from fmtr.tools.dns_tools.dm import Exchange
@@ -9,59 +11,61 @@ from fmtr.tools.logging_tools import logger
 
 
 @dataclass(kw_only=True, eq=False)
-class Plain:
+class Plain(asyncio.DatagramProtocol):
     """
 
-    Base for starting a plain DNS server
-
+    Async base class for a plain DNS server using asyncio DatagramProtocol.
     """
 
     host: str
     port: int
+    transport: Optional[asyncio.DatagramTransport] = field(default=None, init=False)
 
     @cached_property
-    def sock(self):
-        return socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    def loop(self):
+        return asyncio.get_event_loop()
+
 
     @cached_property
     def cache(self):
         """
 
         Overridable cache.
-
         """
         cache = caching.TLRU(maxsize=1_024, ttu_static=timedelta(hours=1), desc='DNS Request')
         return cache
 
-    def start(self):
-        """
+    def connection_made(self, transport: asyncio.DatagramTransport):
+        self.transport = transport
+        logger.info(f'Listening on {self.host}:{self.port}')
 
-        Listen and resolve via overridden resolve method.
+    def datagram_received(self, data: bytes, addr):
+        ip, port = addr
+        exchange = Exchange.from_wire(data, ip=ip, port=port)
+        asyncio.create_task(self.handle(exchange))
 
+    async def start(self):
         """
-        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-        sock.bind((self.host, self.port))
-        logger.info(f'Listening on {self.host}:{self.port}')
-        while True:
-            data, (ip, port) = sock.recvfrom(512)
-            exchange = Exchange.from_wire(data, ip=ip, port=port)
-            self.handle(exchange)
-            sock.sendto(exchange.response.message.to_wire(), (ip, port))
 
-    def resolve(self, exchange: Exchange) -> Exchange:
+        Start the async UDP server.
         """
 
-        Defined in subclasses
+        logger.info(f'Starting async DNS server on {self.host}:{self.port}...')
+        await self.loop.create_datagram_endpoint(
+            lambda: self,
+            local_addr=(self.host, self.port)
+        )
+        await asyncio.Future()  # Prevent exit by blocking forever
 
+    async def resolve(self, exchange: Exchange) -> Exchange:
         """
-        raise NotImplemented
 
-    def check_cache(self, exchange: Exchange):
-        """
-
-        Check cache, patch in in new ID and mark complete
+        To be defined in subclasses.
 
         """
+        raise NotImplementedError
+
+    def check_cache(self, exchange: Exchange):
         if exchange.key in self.cache:
             logger.info(f'Request found in cache.')
             exchange.response = self.cache[exchange.key]
@@ -88,25 +92,30 @@ class Plain:
         """
         request = exchange.request
         response = exchange.response
-
         logger.info(
             f'Resolution complete {exchange.client_name=} {request.message.id=} {request.type_text} {request.name_text} {request.question=} {exchange.is_complete=} {response.rcode=} {response.rcode_text=} {response.answer=} {response.blocked_by=}...'
         )
 
+    def log_dns_errors(self, exchange: Exchange):
+        """
 
+        Warn about any errors
 
-    def handle(self, exchange: Exchange):
         """
+        if exchange.response.rcode != dns.rcode.NOERROR:
+            logger.warning(f'Error {exchange.response.rcode_text=}')
 
-        Check validity of request, reverse lookup client address, check presence in cache and resolve.
-
+    async def handle(self, exchange: Exchange):
         """
 
+        Warn about any errors
+
+        """
         if not exchange.request.is_valid:
             raise ValueError(f'Only one question per request is supported. Got {len(exchange.request.question)} questions.')
 
         if not exchange.is_internal:
-            self.handle(exchange.reverse)
+            await self.handle(exchange.reverse)
             client_name = exchange.reverse.question_last.name.to_text()
             if not exchange.reverse.response.answer:
                 logger.warning(f'Client name could not be resolved {client_name=}.')
@@ -117,10 +126,10 @@ class Plain:
                 self.check_cache(exchange)
 
             if not exchange.is_complete:
-                exchange = self.resolve(exchange)
-                exchange.is_complete = True
+                exchange = await self.resolve(exchange)
+                self.cache[exchange.key] = exchange.response
 
-            self.cache[exchange.key] = exchange.response
+            self.log_dns_errors(exchange)
             self.log_response(exchange)
 
-        return exchange
+        self.transport.sendto(exchange.response.message.to_wire(), exchange.addr)

fmtr_tools-1.3.17/fmtr/tools/version

@@ -0,0 +1 @@
+1.3.17

{fmtr_tools-1.3.15 → fmtr_tools-1.3.17}/fmtr.tools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fmtr.tools
-Version: 1.3.15
+Version: 1.3.17
 Summary: Collection of high-level tools to simplify everyday development tasks, with a focus on AI/ML
 Home-page: https://github.com/fmtr/fmtr.tools
 Author: Frontmatter
@@ -126,61 +126,61 @@ Requires-Dist: logfire[httpx]; extra == "http"
 Provides-Extra: setup
 Requires-Dist: setuptools; extra == "setup"
 Provides-Extra: all
-Requires-Dist: flet-video; extra == "all"
-Requires-Dist: regex; extra == "all"
-Requires-Dist: dnspython[doh]; extra == "all"
-Requires-Dist: distributed; extra == "all"
-Requires-Dist: tinynetrc; extra == "all"
-Requires-Dist: pandas; extra == "all"
-Requires-Dist: contexttimer; extra == "all"
-Requires-Dist: peft; extra == "all"
-Requires-Dist: pydantic; extra == "all"
-Requires-Dist: google-auth; extra == "all"
-Requires-Dist: tokenizers; extra == "all"
-Requires-Dist: diskcache; extra == "all"
-Requires-Dist: cachetools; extra == "all"
-Requires-Dist: pymupdf; extra == "all"
-Requires-Dist: bokeh; extra == "all"
-Requires-Dist: tabulate; extra == "all"
-Requires-Dist: ollama; extra == "all"
 Requires-Dist: Unidecode; extra == "all"
-Requires-Dist: logfire; extra == "all"
-Requires-Dist: sre_yield; extra == "all"
-Requires-Dist: pyyaml; extra == "all"
-Requires-Dist: deepmerge; extra == "all"
-Requires-Dist: huggingface_hub; extra == "all"
-Requires-Dist: openai; extra == "all"
-Requires-Dist: sentence_transformers; extra == "all"
-Requires-Dist: openpyxl; extra == "all"
+Requires-Dist: semver; extra == "all"
+Requires-Dist: google-auth-httplib2; extra == "all"
+Requires-Dist: setuptools; extra == "all"
+Requires-Dist: dask[bag]; extra == "all"
 Requires-Dist: httpx_retries; extra == "all"
 Requires-Dist: flet-webview; extra == "all"
-Requires-Dist: pytest-cov; extra == "all"
+Requires-Dist: docker; extra == "all"
+Requires-Dist: bokeh; extra == "all"
+Requires-Dist: filetype; extra == "all"
 Requires-Dist: torchvision; extra == "all"
-Requires-Dist: torchaudio; extra == "all"
 Requires-Dist: faker; extra == "all"
-Requires-Dist: logfire[httpx]; extra == "all"
-Requires-Dist: flet[all]; extra == "all"
-Requires-Dist: google-api-python-client; extra == "all"
-Requires-Dist: json_repair; extra == "all"
-Requires-Dist: google-auth-oauthlib; extra == "all"
+Requires-Dist: uvicorn[standard]; extra == "all"
 Requires-Dist: fastapi; extra == "all"
-Requires-Dist: yamlscript; extra == "all"
-Requires-Dist: setuptools; extra == "all"
-Requires-Dist: semver; extra == "all"
-Requires-Dist: pymupdf4llm; extra == "all"
-Requires-Dist: pydevd-pycharm~=251.25410.159; extra == "all"
+Requires-Dist: regex; extra == "all"
+Requires-Dist: openai; extra == "all"
 Requires-Dist: appdirs; extra == "all"
-Requires-Dist: google-auth-httplib2; extra == "all"
-Requires-Dist: pydantic-settings; extra == "all"
-Requires-Dist: uvicorn[standard]; extra == "all"
 Requires-Dist: httpx; extra == "all"
-Requires-Dist: docker; extra == "all"
-Requires-Dist: transformers[sentencepiece]; extra == "all"
-Requires-Dist: dask[bag]; extra == "all"
-Requires-Dist: filetype; extra == "all"
-Requires-Dist: logfire[fastapi]; extra == "all"
+Requires-Dist: distributed; extra == "all"
+Requires-Dist: google-api-python-client; extra == "all"
+Requires-Dist: peft; extra == "all"
+Requires-Dist: google-auth-oauthlib; extra == "all"
+Requires-Dist: pyyaml; extra == "all"
+Requires-Dist: json_repair; extra == "all"
+Requires-Dist: deepmerge; extra == "all"
+Requires-Dist: cachetools; extra == "all"
+Requires-Dist: flet[all]; extra == "all"
+Requires-Dist: pydevd-pycharm~=251.25410.159; extra == "all"
 Requires-Dist: html2text; extra == "all"
+Requires-Dist: ollama; extra == "all"
+Requires-Dist: pandas; extra == "all"
+Requires-Dist: pymupdf4llm; extra == "all"
+Requires-Dist: pydantic-settings; extra == "all"
+Requires-Dist: flet-video; extra == "all"
+Requires-Dist: sentence_transformers; extra == "all"
+Requires-Dist: pymupdf; extra == "all"
+Requires-Dist: logfire[httpx]; extra == "all"
+Requires-Dist: huggingface_hub; extra == "all"
+Requires-Dist: openpyxl; extra == "all"
+Requires-Dist: contexttimer; extra == "all"
+Requires-Dist: yamlscript; extra == "all"
+Requires-Dist: pydantic; extra == "all"
+Requires-Dist: sre_yield; extra == "all"
 Requires-Dist: pydantic-ai[logfire,openai]; extra == "all"
+Requires-Dist: tokenizers; extra == "all"
+Requires-Dist: torchaudio; extra == "all"
+Requires-Dist: logfire[fastapi]; extra == "all"
+Requires-Dist: pytest-cov; extra == "all"
+Requires-Dist: tinynetrc; extra == "all"
+Requires-Dist: tabulate; extra == "all"
+Requires-Dist: diskcache; extra == "all"
+Requires-Dist: transformers[sentencepiece]; extra == "all"
+Requires-Dist: google-auth; extra == "all"
+Requires-Dist: logfire; extra == "all"
+Requires-Dist: dnspython[doh]; extra == "all"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

fmtr_tools-1.3.15/fmtr/tools/version

@@ -1 +0,0 @@
-1.3.15

{fmtr_tools-1.3.15 → fmtr_tools-1.3.17}/fmtr.tools.egg-info/requires.txt

@@ -14,61 +14,61 @@ pydantic-ai[logfire,openai]
 ollama
 
 [all]
-flet-video
-regex
-dnspython[doh]
-distributed
-tinynetrc
-pandas
-contexttimer
-peft
-pydantic
-google-auth
-tokenizers
-diskcache
-cachetools
-pymupdf
-bokeh
-tabulate
-ollama
 Unidecode
-logfire
-sre_yield
-pyyaml
-deepmerge
-huggingface_hub
-openai
-sentence_transformers
-openpyxl
+semver
+google-auth-httplib2
+setuptools
+dask[bag]
 httpx_retries
 flet-webview
-pytest-cov
+docker
+bokeh
+filetype
 torchvision
-torchaudio
 faker
-logfire[httpx]
-flet[all]
-google-api-python-client
-json_repair
-google-auth-oauthlib
+uvicorn[standard]
 fastapi
-yamlscript
-setuptools
-semver
-pymupdf4llm
-pydevd-pycharm~=251.25410.159
+regex
+openai
 appdirs
-google-auth-httplib2
-pydantic-settings
-uvicorn[standard]
 httpx
-docker
-transformers[sentencepiece]
-dask[bag]
-filetype
-logfire[fastapi]
+distributed
+google-api-python-client
+peft
+google-auth-oauthlib
+pyyaml
+json_repair
+deepmerge
+cachetools
+flet[all]
+pydevd-pycharm~=251.25410.159
 html2text
+ollama
+pandas
+pymupdf4llm
+pydantic-settings
+flet-video
+sentence_transformers
+pymupdf
+logfire[httpx]
+huggingface_hub
+openpyxl
+contexttimer
+yamlscript
+pydantic
+sre_yield
 pydantic-ai[logfire,openai]
+tokenizers
+torchaudio
+logfire[fastapi]
+pytest-cov
+tinynetrc
+tabulate
+diskcache
+transformers[sentencepiece]
+google-auth
+logfire
+dnspython[doh]
 
 [api]
 fastapi