fmtr.tools 1.3.14__tar.gz → 1.3.16__tar.gz

{fmtr_tools-1.3.14 → fmtr_tools-1.3.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fmtr.tools
-Version: 1.3.14
+Version: 1.3.16
 Summary: Collection of high-level tools to simplify everyday development tasks, with a focus on AI/ML
 Home-page: https://github.com/fmtr/fmtr.tools
 Author: Frontmatter
@@ -126,61 +126,61 @@ Requires-Dist: logfire[httpx]; extra == "http"
 Provides-Extra: setup
 Requires-Dist: setuptools; extra == "setup"
 Provides-Extra: all
-Requires-Dist: deepmerge; extra == "all"
-Requires-Dist: google-auth-httplib2; extra == "all"
-Requires-Dist: fastapi; extra == "all"
-Requires-Dist: bokeh; extra == "all"
-Requires-Dist: pydantic; extra == "all"
+Requires-Dist: filetype; extra == "all"
+Requires-Dist: pyyaml; extra == "all"
+Requires-Dist: huggingface_hub; extra == "all"
+Requires-Dist: tinynetrc; extra == "all"
+Requires-Dist: flet[all]; extra == "all"
 Requires-Dist: tabulate; extra == "all"
-Requires-Dist: dask[bag]; extra == "all"
+Requires-Dist: pydantic-settings; extra == "all"
+Requires-Dist: flet-webview; extra == "all"
+Requires-Dist: docker; extra == "all"
+Requires-Dist: Unidecode; extra == "all"
 Requires-Dist: openpyxl; extra == "all"
-Requires-Dist: filetype; extra == "all"
+Requires-Dist: sentence_transformers; extra == "all"
+Requires-Dist: deepmerge; extra == "all"
 Requires-Dist: pymupdf; extra == "all"
-Requires-Dist: docker; extra == "all"
-Requires-Dist: flet-webview; extra == "all"
-Requires-Dist: dnspython[doh]; extra == "all"
+Requires-Dist: google-auth-oauthlib; extra == "all"
+Requires-Dist: torchaudio; extra == "all"
 Requires-Dist: logfire; extra == "all"
-Requires-Dist: contexttimer; extra == "all"
-Requires-Dist: torchvision; extra == "all"
-Requires-Dist: huggingface_hub; extra == "all"
-Requires-Dist: transformers[sentencepiece]; extra == "all"
+Requires-Dist: diskcache; extra == "all"
 Requires-Dist: cachetools; extra == "all"
-Requires-Dist: httpx; extra == "all"
-Requires-Dist: semver; extra == "all"
-Requires-Dist: sre_yield; extra == "all"
-Requires-Dist: html2text; extra == "all"
-Requires-Dist: pyyaml; extra == "all"
-Requires-Dist: google-auth; extra == "all"
-Requires-Dist: regex; extra == "all"
-Requires-Dist: httpx_retries; extra == "all"
+Requires-Dist: setuptools; extra == "all"
+Requires-Dist: distributed; extra == "all"
 Requires-Dist: tokenizers; extra == "all"
-Requires-Dist: sentence_transformers; extra == "all"
-Requires-Dist: flet-video; extra == "all"
-Requires-Dist: pymupdf4llm; extra == "all"
-Requires-Dist: google-auth-oauthlib; extra == "all"
-Requires-Dist: pytest-cov; extra == "all"
-Requires-Dist: logfire[httpx]; extra == "all"
-Requires-Dist: ollama; extra == "all"
 Requires-Dist: faker; extra == "all"
-Requires-Dist: torchaudio; extra == "all"
-Requires-Dist: pandas; extra == "all"
-Requires-Dist: google-api-python-client; extra == "all"
-Requires-Dist: uvicorn[standard]; extra == "all"
-Requires-Dist: appdirs; extra == "all"
-Requires-Dist: flet[all]; extra == "all"
-Requires-Dist: setuptools; extra == "all"
 Requires-Dist: logfire[fastapi]; extra == "all"
+Requires-Dist: google-auth; extra == "all"
+Requires-Dist: fastapi; extra == "all"
+Requires-Dist: peft; extra == "all"
+Requires-Dist: transformers[sentencepiece]; extra == "all"
 Requires-Dist: yamlscript; extra == "all"
+Requires-Dist: contexttimer; extra == "all"
+Requires-Dist: torchvision; extra == "all"
+Requires-Dist: appdirs; extra == "all"
+Requires-Dist: google-auth-httplib2; extra == "all"
+Requires-Dist: logfire[httpx]; extra == "all"
+Requires-Dist: pytest-cov; extra == "all"
+Requires-Dist: regex; extra == "all"
+Requires-Dist: pydantic; extra == "all"
 Requires-Dist: pydevd-pycharm~=251.25410.159; extra == "all"
-Requires-Dist: tinynetrc; extra == "all"
-Requires-Dist: pydantic-ai[logfire,openai]; extra == "all"
-Requires-Dist: Unidecode; extra == "all"
-Requires-Dist: pydantic-settings; extra == "all"
-Requires-Dist: json_repair; extra == "all"
-Requires-Dist: peft; extra == "all"
-Requires-Dist: distributed; extra == "all"
 Requires-Dist: openai; extra == "all"
-Requires-Dist: diskcache; extra == "all"
+Requires-Dist: dask[bag]; extra == "all"
+Requires-Dist: pandas; extra == "all"
+Requires-Dist: json_repair; extra == "all"
+Requires-Dist: uvicorn[standard]; extra == "all"
+Requires-Dist: html2text; extra == "all"
+Requires-Dist: pydantic-ai[logfire,openai]; extra == "all"
+Requires-Dist: semver; extra == "all"
+Requires-Dist: dnspython[doh]; extra == "all"
+Requires-Dist: ollama; extra == "all"
+Requires-Dist: flet-video; extra == "all"
+Requires-Dist: google-api-python-client; extra == "all"
+Requires-Dist: httpx; extra == "all"
+Requires-Dist: httpx_retries; extra == "all"
+Requires-Dist: sre_yield; extra == "all"
+Requires-Dist: bokeh; extra == "all"
+Requires-Dist: pymupdf4llm; extra == "all"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{fmtr_tools-1.3.14 → fmtr_tools-1.3.16}/fmtr/tools/dns_tools/client.py

@@ -3,6 +3,7 @@ from dataclasses import dataclass
 from dns import query
 from functools import cached_property
 from httpx_retries import Retry, RetryTransport
+from typing import Optional
 
 from fmtr.tools import http_tools as http
 from fmtr.tools.dns_tools.dm import Exchange, Response
@@ -38,14 +39,17 @@ class Plain:
     """
     host: str
     port: int = 53
+    ttl_min: Optional[int] = None
 
     def resolve(self, exchange: Exchange):
 
         with logger.span(f'UDP {self.host}:{self.port}'):
             response_plain = query.udp(q=exchange.query_last, where=self.host, port=self.port)
             response = Response.from_message(response_plain)
+            for answer in response.message.answer:
+                answer.ttl = max(answer.ttl, self.ttl_min or answer.ttl)
 
-        exchange.response.message.answer += response.message.answer
+        exchange.response = response
 
 
 @dataclass
@@ -86,7 +90,7 @@ class HTTP:
             response_doh = self.CLIENT.post(url, headers=headers, content=exchange.query_last.to_wire())
             response_doh.raise_for_status()
             response = Response.from_http(response_doh)
-            exchange.response.message.answer += response.message.answer
+            exchange.response = response
 
         except Exception as exception:
             exchange.response.message.set_rcode(dnspython.rcode.SERVFAIL)

{fmtr_tools-1.3.14 → fmtr_tools-1.3.16}/fmtr/tools/dns_tools/dm.py

@@ -1,15 +1,28 @@
 import dns
 import httpx
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from dns import rcode as dnsrcode
 from dns import reversename
 from dns.message import Message, QueryMessage
 from dns.rrset import RRset
 from functools import cached_property
-from typing import Self, Optional
+from typing import Self, Optional, List
 
 from fmtr.tools.string_tools import join
 
+TTL_CODE_DEFAULTS = {
+    dnsrcode.NOERROR: 300,  # Successful query
+    dnsrcode.FORMERR: 60,  # Format error
+    dnsrcode.SERVFAIL: 10,  # Server failure
+    dnsrcode.NXDOMAIN: 60 * 60,  # Non-existent domain
+    dnsrcode.NOTIMP: 60,  # Not implemented
+    dnsrcode.REFUSED: 60,  # Refused
+    dnsrcode.YXDOMAIN: 600,  # Name exists when it should not
+    dnsrcode.YXRRSET: 600,  # RR Set exists when it should not
+    dnsrcode.NXRRSET: 300,  # RR Set that should exist does not
+    dnsrcode.NOTAUTH: 60,  # Not authorized
+    dnsrcode.NOTZONE: 60  # Name not contained in zone
+}
 
 @dataclass
 class BaseDNSData:
@@ -37,7 +50,6 @@ class Response(BaseDNSData):
     """
 
     http: Optional[httpx.Response] = None
-    is_complete: bool = False
     blocked_by: Optional[str] = None
 
     @classmethod
@@ -69,6 +81,24 @@ class Response(BaseDNSData):
     def rcode_text(self) -> str:
         return dnsrcode.to_text(self.rcode)
 
+    @property
+    def ttl(self) -> int:
+        """
+
+        Get median TTL from answers, falling back to authority, then to error-code defaults.
+
+        """
+        answers = self.message.answer or self.message.authority
+        if answers:
+            ttls = [answer.ttl for answer in answers]
+            ttl = min(ttls)
+            return ttl
+
+        ttl = TTL_CODE_DEFAULTS.get(self.rcode, dnsrcode.NXDOMAIN)
+        return ttl
+
+
+
     def __str__(self):
         """
 
@@ -144,9 +174,10 @@ class Exchange:
 
     request: Request
     response: Optional[Response] = None
+    answers_pre: List[RRset] = field(default_factory=list)
     is_internal: bool = False
     client_name: Optional[str] = None
-
+    is_complete: bool = False
 
 
     @classmethod
@@ -163,27 +194,25 @@ class Exchange:
     def question_last(self) -> RRset:
         """
 
-        Contrive an RRset representing the latest/current question. This can be the original question - or a hybrid one if we've injected our own answers into the exchange.
+        Create an RRset surrogate representing the latest/current question. This can be the original question - or a hybrid one if we've injected our own answers into the Exchange.
 
         """
-        if self.response.answer:
-            rrset = self.response.answer
-            ty = self.request.type
+        if self.answers_pre:
+            rrset = self.answers_pre[-1]
+            rdtype = self.request.type
             ttl = self.request.question.ttl
             rdclass = self.request.question.rdclass
             name = next(iter(rrset.items.keys())).to_text()
-
-            rrset_contrived = dns.rrset.from_text(
+            rrset_surrogate = dns.rrset.from_text(
                 name=name,
                 ttl=ttl,
-                rdtype=ty,
+                rdtype=rdtype,
                 rdclass=rdclass,
-
             )
 
-            return rrset_contrived
+            return rrset_surrogate
         else:
-            return self.request.question  # Solves the issue of digging out the name.
+            return self.request.question
 
     @property
     def query_last(self) -> QueryMessage:
@@ -194,7 +223,7 @@ class Exchange:
         """
 
         question_last = self.question_last
-        query = dns.message.make_query(qname=question_last.name, rdclass=question_last.rdclass, rdtype=question_last.rdtype)
+        query = dns.message.make_query(qname=question_last.name, rdclass=question_last.rdclass, rdtype=question_last.rdtype, id=self.request.message.id)
         return query
 
     @property

{fmtr_tools-1.3.14 → fmtr_tools-1.3.16}/fmtr/tools/dns_tools/proxy.py

@@ -31,6 +31,14 @@ class Proxy(server.Plain):
         """
         return
 
+    def finalize(self, exchange: Exchange):
+        """
+
+        Finalize a still open exchange.
+
+        """
+        exchange.is_complete = True
+
     def resolve(self, exchange: Exchange) -> Exchange:
         """
 
@@ -38,20 +46,21 @@ class Proxy(server.Plain):
         Subclasses can override the relevant processing methods to implement custom behaviour.
 
         """
-
         with logger.span(f'Processing question...'):
             self.process_question(exchange)
-        if exchange.response.is_complete:
+        if exchange.is_complete:
             return exchange
 
         with logger.span(f'Making upstream request...'):
             self.client.resolve(exchange)
-        if exchange.response.is_complete:
+        if exchange.is_complete:
             return exchange
 
         with logger.span(f'Processing upstream response...'):
             self.process_upstream(exchange)
-        if exchange.response.is_complete:
+        if exchange.is_complete:
             return exchange
 
+        self.finalize(exchange)
+
         return exchange

{fmtr_tools-1.3.14 → fmtr_tools-1.3.16}/fmtr/tools/dns_tools/server.py

@@ -1,3 +1,4 @@
+import dns
 import socket
 from dataclasses import dataclass
 from datetime import timedelta
@@ -66,7 +67,7 @@ class Plain:
             logger.info(f'Request found in cache.')
             exchange.response = self.cache[exchange.key]
             exchange.response.message.id = exchange.request.message.id
-            exchange.response.is_complete = True
+            exchange.is_complete = True
 
     def get_span(self, exchange: Exchange):
         """
@@ -90,10 +91,20 @@ class Plain:
         response = exchange.response
 
         logger.info(
-            f'Resolution complete {exchange.client_name=} {request.message.id=} {request.type_text} {request.name_text} {request.question=} {response.is_complete=} {response.rcode=} {response.rcode_text=} {response.answer=} {response.blocked_by=}...'
+            f'Resolution complete {exchange.client_name=} {request.message.id=} {request.type_text} {request.name_text} {request.question=} {exchange.is_complete=} {response.rcode=} {response.rcode_text=} {response.answer=} {response.blocked_by=}...'
         )
 
+    def log_dns_errors(self, exchange: Exchange):
+        """
+
+        Warn about any errors
+
+        """
+
+        if exchange.response.rcode == dns.rcode.NOERROR:
+            return
 
+        logger.warning(f'Error {exchange.response.rcode_text=}')
 
     def handle(self, exchange: Exchange):
         """
@@ -116,11 +127,11 @@ class Plain:
             with logger.span(f'Checking cache...'):
                 self.check_cache(exchange)
 
-            if not exchange.response.is_complete:
+            if not exchange.is_complete:
                 exchange = self.resolve(exchange)
-                exchange.response.is_complete = True
+                self.cache[exchange.key] = exchange.response
 
-            self.cache[exchange.key] = exchange.response
+            self.log_dns_errors(exchange)
             self.log_response(exchange)
 
         return exchange

fmtr_tools-1.3.16/fmtr/tools/version

@@ -0,0 +1 @@
+1.3.16

{fmtr_tools-1.3.14 → fmtr_tools-1.3.16}/fmtr.tools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fmtr.tools
-Version: 1.3.14
+Version: 1.3.16
 Summary: Collection of high-level tools to simplify everyday development tasks, with a focus on AI/ML
 Home-page: https://github.com/fmtr/fmtr.tools
 Author: Frontmatter
@@ -126,61 +126,61 @@ Requires-Dist: logfire[httpx]; extra == "http"
 Provides-Extra: setup
 Requires-Dist: setuptools; extra == "setup"
 Provides-Extra: all
-Requires-Dist: deepmerge; extra == "all"
-Requires-Dist: google-auth-httplib2; extra == "all"
-Requires-Dist: fastapi; extra == "all"
-Requires-Dist: bokeh; extra == "all"
-Requires-Dist: pydantic; extra == "all"
+Requires-Dist: filetype; extra == "all"
+Requires-Dist: pyyaml; extra == "all"
+Requires-Dist: huggingface_hub; extra == "all"
+Requires-Dist: tinynetrc; extra == "all"
+Requires-Dist: flet[all]; extra == "all"
 Requires-Dist: tabulate; extra == "all"
-Requires-Dist: dask[bag]; extra == "all"
+Requires-Dist: pydantic-settings; extra == "all"
+Requires-Dist: flet-webview; extra == "all"
+Requires-Dist: docker; extra == "all"
+Requires-Dist: Unidecode; extra == "all"
 Requires-Dist: openpyxl; extra == "all"
-Requires-Dist: filetype; extra == "all"
+Requires-Dist: sentence_transformers; extra == "all"
+Requires-Dist: deepmerge; extra == "all"
 Requires-Dist: pymupdf; extra == "all"
-Requires-Dist: docker; extra == "all"
-Requires-Dist: flet-webview; extra == "all"
-Requires-Dist: dnspython[doh]; extra == "all"
+Requires-Dist: google-auth-oauthlib; extra == "all"
+Requires-Dist: torchaudio; extra == "all"
 Requires-Dist: logfire; extra == "all"
-Requires-Dist: contexttimer; extra == "all"
-Requires-Dist: torchvision; extra == "all"
-Requires-Dist: huggingface_hub; extra == "all"
-Requires-Dist: transformers[sentencepiece]; extra == "all"
+Requires-Dist: diskcache; extra == "all"
 Requires-Dist: cachetools; extra == "all"
-Requires-Dist: httpx; extra == "all"
-Requires-Dist: semver; extra == "all"
-Requires-Dist: sre_yield; extra == "all"
-Requires-Dist: html2text; extra == "all"
-Requires-Dist: pyyaml; extra == "all"
-Requires-Dist: google-auth; extra == "all"
-Requires-Dist: regex; extra == "all"
-Requires-Dist: httpx_retries; extra == "all"
+Requires-Dist: setuptools; extra == "all"
+Requires-Dist: distributed; extra == "all"
 Requires-Dist: tokenizers; extra == "all"
-Requires-Dist: sentence_transformers; extra == "all"
-Requires-Dist: flet-video; extra == "all"
-Requires-Dist: pymupdf4llm; extra == "all"
-Requires-Dist: google-auth-oauthlib; extra == "all"
-Requires-Dist: pytest-cov; extra == "all"
-Requires-Dist: logfire[httpx]; extra == "all"
-Requires-Dist: ollama; extra == "all"
 Requires-Dist: faker; extra == "all"
-Requires-Dist: torchaudio; extra == "all"
-Requires-Dist: pandas; extra == "all"
-Requires-Dist: google-api-python-client; extra == "all"
-Requires-Dist: uvicorn[standard]; extra == "all"
-Requires-Dist: appdirs; extra == "all"
-Requires-Dist: flet[all]; extra == "all"
-Requires-Dist: setuptools; extra == "all"
 Requires-Dist: logfire[fastapi]; extra == "all"
+Requires-Dist: google-auth; extra == "all"
+Requires-Dist: fastapi; extra == "all"
+Requires-Dist: peft; extra == "all"
+Requires-Dist: transformers[sentencepiece]; extra == "all"
 Requires-Dist: yamlscript; extra == "all"
+Requires-Dist: contexttimer; extra == "all"
+Requires-Dist: torchvision; extra == "all"
+Requires-Dist: appdirs; extra == "all"
+Requires-Dist: google-auth-httplib2; extra == "all"
+Requires-Dist: logfire[httpx]; extra == "all"
+Requires-Dist: pytest-cov; extra == "all"
+Requires-Dist: regex; extra == "all"
+Requires-Dist: pydantic; extra == "all"
 Requires-Dist: pydevd-pycharm~=251.25410.159; extra == "all"
-Requires-Dist: tinynetrc; extra == "all"
-Requires-Dist: pydantic-ai[logfire,openai]; extra == "all"
-Requires-Dist: Unidecode; extra == "all"
-Requires-Dist: pydantic-settings; extra == "all"
-Requires-Dist: json_repair; extra == "all"
-Requires-Dist: peft; extra == "all"
-Requires-Dist: distributed; extra == "all"
 Requires-Dist: openai; extra == "all"
-Requires-Dist: diskcache; extra == "all"
+Requires-Dist: dask[bag]; extra == "all"
+Requires-Dist: pandas; extra == "all"
+Requires-Dist: json_repair; extra == "all"
+Requires-Dist: uvicorn[standard]; extra == "all"
+Requires-Dist: html2text; extra == "all"
+Requires-Dist: pydantic-ai[logfire,openai]; extra == "all"
+Requires-Dist: semver; extra == "all"
+Requires-Dist: dnspython[doh]; extra == "all"
+Requires-Dist: ollama; extra == "all"
+Requires-Dist: flet-video; extra == "all"
+Requires-Dist: google-api-python-client; extra == "all"
+Requires-Dist: httpx; extra == "all"
+Requires-Dist: httpx_retries; extra == "all"
+Requires-Dist: sre_yield; extra == "all"
+Requires-Dist: bokeh; extra == "all"
+Requires-Dist: pymupdf4llm; extra == "all"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

fmtr_tools-1.3.14/fmtr/tools/version

@@ -1 +0,0 @@
-1.3.14

{fmtr_tools-1.3.14 → fmtr_tools-1.3.16}/fmtr.tools.egg-info/requires.txt

@@ -14,61 +14,61 @@ pydantic-ai[logfire,openai]
 ollama
 
 [all]
-deepmerge
-google-auth-httplib2
-fastapi
-bokeh
-pydantic
+filetype
+pyyaml
+huggingface_hub
+tinynetrc
+flet[all]
 tabulate
-dask[bag]
+pydantic-settings
+flet-webview
+docker
+Unidecode
 openpyxl
-filetype
+sentence_transformers
+deepmerge
 pymupdf
-docker
-flet-webview
-dnspython[doh]
+google-auth-oauthlib
+torchaudio
 logfire
-contexttimer
-torchvision
-huggingface_hub
-transformers[sentencepiece]
+diskcache
 cachetools
-httpx
-semver
-sre_yield
-html2text
-pyyaml
-google-auth
-regex
-httpx_retries
+setuptools
+distributed
 tokenizers
-sentence_transformers
-flet-video
-pymupdf4llm
-google-auth-oauthlib
-pytest-cov
-logfire[httpx]
-ollama
 faker
-torchaudio
-pandas
-google-api-python-client
-uvicorn[standard]
-appdirs
-flet[all]
-setuptools
 logfire[fastapi]
+google-auth
+fastapi
+peft
+transformers[sentencepiece]
 yamlscript
+contexttimer
+torchvision
+appdirs
+google-auth-httplib2
+logfire[httpx]
+pytest-cov
+regex
+pydantic
 pydevd-pycharm~=251.25410.159
-tinynetrc
-pydantic-ai[logfire,openai]
-Unidecode
-pydantic-settings
-json_repair
-peft
-distributed
 openai
-diskcache
+dask[bag]
+pandas
+json_repair
+uvicorn[standard]
+html2text
+pydantic-ai[logfire,openai]
+semver
+dnspython[doh]
+ollama
+flet-video
+google-api-python-client
+httpx
+httpx_retries
+sre_yield
+bokeh
+pymupdf4llm
 
 [api]
 fastapi