fetchurl-sdk 0.1.4__tar.gz

fetchurl_sdk-0.1.4/.gitignore

@@ -0,0 +1,2 @@
+.venv
+__pycache__

fetchurl_sdk-0.1.4/.python-version

@@ -0,0 +1 @@
+3.13

fetchurl_sdk-0.1.4/PKG-INFO

@@ -0,0 +1,22 @@
+Metadata-Version: 2.4
+Name: fetchurl-sdk
+Version: 0.1.4
+Summary: Protocol-level client SDK for fetchurl content-addressable cache servers
+Project-URL: Homepage, https://github.com/lucasew/fetchurl
+Project-URL: Repository, https://github.com/lucasew/fetchurl
+Project-URL: Bug Tracker, https://github.com/lucasew/fetchurl/issues
+Author: lucasew
+License: MIT
+Keywords: cache,content-addressable,fetchurl,hash,sha256
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+
+# fetchurl
+
+Simple caching server for URLs with a hash.

fetchurl_sdk-0.1.4/README.md

@@ -0,0 +1,3 @@
+# fetchurl
+
+Simple caching server for URLs with a hash.

fetchurl_sdk-0.1.4/fetchurl/__init__.py

@@ -0,0 +1,417 @@
+"""Fetchurl SDK for Python.
+
+Protocol-level client for fetchurl content-addressable cache servers.
+Works with any HTTP library through the Fetcher/AsyncFetcher protocols.
+
+Zero dependencies — uses only the Python standard library.
+
+Three levels of usage:
+
+  # 1. One-liner with stdlib
+  fetchurl.fetch(UrllibFetcher(), "sha256", hash, urls, output)
+
+  # 2. Custom HTTP client — implement the Fetcher protocol
+  class MyFetcher:
+      def get(self, url, headers):
+          resp = requests.get(url, headers=headers, stream=True)
+          return (resp.status_code, resp.raw)
+
+  fetchurl.fetch(MyFetcher(), "sha256", hash, urls, output)
+
+  # 3. Low-level — drive the state machine yourself
+  session = FetchSession("sha256", hash, urls)
+  while attempt := session.next_attempt():
+      # make HTTP request with whatever library you want
+      ...
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import random
+import re
+from collections.abc import AsyncIterator
+from dataclasses import dataclass, field
+from typing import BinaryIO, Protocol, runtime_checkable
+
+
+# --- Errors ---
+
+
+class FetchUrlError(Exception):
+    """Base exception for fetchurl SDK."""
+
+
+class UnsupportedAlgorithmError(FetchUrlError):
+    """The requested hash algorithm is not supported."""
+
+    def __init__(self, algo: str):
+        self.algo = algo
+        super().__init__(f"unsupported algorithm: {algo}")
+
+
+class HashMismatchError(FetchUrlError):
+    """The content hash does not match the expected hash."""
+
+    def __init__(self, expected: str, actual: str):
+        self.expected = expected
+        self.actual = actual
+        super().__init__(f"hash mismatch: expected {expected}, got {actual}")
+
+
+class AllSourcesFailedError(FetchUrlError):
+    """All servers and sources failed to provide the content."""
+
+    def __init__(self, last_error: Exception | None = None):
+        self.last_error = last_error
+        super().__init__("all sources failed")
+
+
+class PartialWriteError(FetchUrlError):
+    """Bytes were written before failure; output is tainted."""
+
+    def __init__(self, cause: Exception):
+        self.cause = cause
+        super().__init__(f"partial write: {cause}")
+
+
+# --- Algorithm helpers ---
+
+_SUPPORTED_ALGOS = {"sha1", "sha256", "sha512"}
+
+
+def normalize_algo(name: str) -> str:
+    """Normalize algorithm name per spec: lowercase, only [a-z0-9]."""
+    return re.sub(r"[^a-z0-9]", "", name.lower())
+
+
+def is_supported(algo: str) -> bool:
+    """Check if a hash algorithm is supported."""
+    return normalize_algo(algo) in _SUPPORTED_ALGOS
+
+
+# --- SFV helpers (RFC 8941 string lists) ---
+
+
+def encode_source_urls(urls: list[str]) -> str:
+    """Encode URLs as an RFC 8941 string list for X-Source-Urls header."""
+    return ", ".join(
+        '"' + url.replace("\\", "\\\\").replace('"', '\\"') + '"' for url in urls
+    )
+
+
+def parse_fetchurl_server(value: str) -> list[str]:
+    """Parse FETCHURL_SERVER env var (RFC 8941 string list)."""
+    results: list[str] = []
+    i = 0
+    while i < len(value):
+        while i < len(value) and value[i] in " \t":
+            i += 1
+        if i >= len(value):
+            break
+        if value[i] != '"':
+            while i < len(value) and value[i] != ",":
+                i += 1
+            if i < len(value):
+                i += 1
+            continue
+        i += 1
+        s: list[str] = []
+        while i < len(value):
+            if value[i] == "\\" and i + 1 < len(value):
+                s.append(value[i + 1])
+                i += 2
+            elif value[i] == '"':
+                i += 1
+                break
+            else:
+                s.append(value[i])
+                i += 1
+        results.append("".join(s))
+        while i < len(value) and value[i] != ",":
+            i += 1
+        if i < len(value):
+            i += 1
+    return results
+
+
+# --- FetchAttempt ---
+
+
+@dataclass(frozen=True)
+class FetchAttempt:
+    """A single fetch attempt with URL and headers."""
+
+    url: str
+    headers: dict[str, str] = field(default_factory=dict)
+
+
+# --- HashVerifier ---
+
+
+class HashVerifier:
+    """Wraps a binary writer, computes hash, verifies on finish().
+
+    Usage::
+
+        verifier = session.verifier(output_file)
+        while chunk := body.read(65536):
+            verifier.write(chunk)
+        verifier.finish()  # raises HashMismatchError on failure
+    """
+
+    def __init__(self, algo: str, expected_hash: str, writer: BinaryIO):
+        self._writer = writer
+        self._hasher = hashlib.new(normalize_algo(algo))
+        self._expected = expected_hash
+        self._bytes_written = 0
+
+    @property
+    def bytes_written(self) -> int:
+        return self._bytes_written
+
+    def write(self, data: bytes) -> int:
+        n = self._writer.write(data)
+        if n is None:
+            n = len(data)
+        self._hasher.update(data[:n])
+        self._bytes_written += n
+        return n
+
+    def finish(self) -> None:
+        """Verify hash. Raises HashMismatchError on failure."""
+        actual = self._hasher.hexdigest()
+        if actual != self._expected:
+            raise HashMismatchError(self._expected, actual)
+
+
+# --- FetchSession ---
+
+
+class FetchSession:
+    """State machine driving the fetchurl client protocol.
+
+    Servers are tried first (with X-Source-Urls header forwarded),
+    then direct source URLs in random order per spec.
+
+    The caller iterates through attempts, makes HTTP requests
+    with their preferred library, and reports results back::
+
+        session = FetchSession(servers, "sha256", hash, source_urls)
+        while attempt := session.next_attempt():
+            # attempt.url and attempt.headers tell you what to request
+            ...
+            session.report_success()  # or report_partial()
+    """
+
+    def __init__(
+        self,
+        algo: str,
+        hash: str,
+        source_urls: list[str],
+    ):
+        servers = parse_fetchurl_server(os.environ.get("FETCHURL_SERVER", ""))
+        algo = normalize_algo(algo)
+        if not is_supported(algo):
+            raise UnsupportedAlgorithmError(algo)
+
+        self._algo = algo
+        self._hash = hash
+        self._done = False
+        self._success = False
+        self._attempts: list[FetchAttempt] = []
+        self._current = 0
+
+        source_header = encode_source_urls(source_urls) if source_urls else None
+
+        for server in servers:
+            base = server.rstrip("/")
+            url = f"{base}/api/fetchurl/{algo}/{hash}"
+            headers: dict[str, str] = {}
+            if source_header:
+                headers["X-Source-Urls"] = source_header
+            self._attempts.append(FetchAttempt(url=url, headers=headers))
+
+        direct = list(source_urls)
+        random.shuffle(direct)
+        for url in direct:
+            self._attempts.append(FetchAttempt(url=url))
+
+    def next_attempt(self) -> FetchAttempt | None:
+        """Get the next attempt, or None if session is finished.
+
+        If an attempt fails without writing bytes, just call next_attempt() again.
+        """
+        if self._done or self._current >= len(self._attempts):
+            return None
+        attempt = self._attempts[self._current]
+        self._current += 1
+        return attempt
+
+    def report_success(self) -> None:
+        """Mark the session as successful. Stops further attempts."""
+        self._done = True
+        self._success = True
+
+    def report_partial(self) -> None:
+        """Mark that bytes were written before failure. Stops further attempts."""
+        self._done = True
+
+    def succeeded(self) -> bool:
+        return self._success
+
+    def verifier(self, writer: BinaryIO) -> HashVerifier:
+        """Create a HashVerifier for this session's algo and expected hash."""
+        return HashVerifier(self._algo, self._hash, writer)
+
+
+# --- Fetcher protocols ---
+
+
+@runtime_checkable
+class Fetcher(Protocol):
+    """Sync HTTP client protocol.
+
+    Implement this to plug in any HTTP library.
+
+    Example with requests::
+
+        class RequestsFetcher:
+            def get(self, url, headers):
+                resp = requests.get(url, headers=headers, stream=True)
+                return (resp.status_code, resp.raw)
+    """
+
+    def get(self, url: str, headers: dict[str, str]) -> tuple[int, BinaryIO]:
+        """Make a GET request. Returns (status_code, readable_body)."""
+        ...
+
+
+@runtime_checkable
+class AsyncFetcher(Protocol):
+    """Async HTTP client protocol.
+
+    Implement this to plug in any async HTTP library.
+
+    Example with aiohttp::
+
+        class AiohttpFetcher:
+            def __init__(self):
+                self._session = aiohttp.ClientSession()
+
+            async def get(self, url, headers):
+                resp = await self._session.get(url, headers=headers)
+                return (resp.status, resp.content.iter_chunked(65536))
+    """
+
+    async def get(
+        self, url: str, headers: dict[str, str]
+    ) -> tuple[int, AsyncIterator[bytes]]:
+        """Make a GET request. Returns (status_code, async_body_chunks)."""
+        ...
+
+
+# --- UrllibFetcher (stdlib, zero deps) ---
+
+
+class UrllibFetcher:
+    """Fetcher using urllib.request (stdlib, zero dependencies)."""
+
+    def get(self, url: str, headers: dict[str, str]) -> tuple[int, BinaryIO]:
+        import urllib.error
+        import urllib.request
+
+        req = urllib.request.Request(url, headers=headers)
+        try:
+            resp = urllib.request.urlopen(req)
+            return (resp.status, resp)
+        except urllib.error.HTTPError as e:
+            return (e.code, e)
+
+
+# --- Convenience functions ---
+
+_CHUNK_SIZE = 64 * 1024
+
+
+def fetch(
+    fetcher: Fetcher,
+    algo: str,
+    hash: str,
+    source_urls: list[str],
+    out: BinaryIO,
+) -> None:
+    """High-level sync fetch. Handles the full protocol loop.
+
+    Raises AllSourcesFailedError or PartialWriteError on failure.
+    """
+    session = FetchSession(algo, hash, source_urls)
+    last_error: Exception | None = None
+
+    while attempt := session.next_attempt():
+        try:
+            status, body = fetcher.get(attempt.url, dict(attempt.headers))
+        except Exception as e:
+            last_error = e
+            continue
+
+        if status != 200:
+            last_error = Exception(f"unexpected status {status}")
+            continue
+
+        verifier = session.verifier(out)
+        try:
+            while chunk := body.read(_CHUNK_SIZE):
+                verifier.write(chunk)
+            verifier.finish()
+            session.report_success()
+            return
+        except Exception as e:
+            last_error = e
+            if verifier.bytes_written > 0:
+                session.report_partial()
+                raise PartialWriteError(e) from e
+
+    raise AllSourcesFailedError(last_error)
+
+
+async def async_fetch(
+    fetcher: AsyncFetcher,
+    algo: str,
+    hash: str,
+    source_urls: list[str],
+    out: BinaryIO,
+) -> None:
+    """High-level async fetch. Handles the full protocol loop.
+
+    Raises AllSourcesFailedError or PartialWriteError on failure.
+    """
+    session = FetchSession(algo, hash, source_urls)
+    last_error: Exception | None = None
+
+    while attempt := session.next_attempt():
+        try:
+            status, chunks = await fetcher.get(attempt.url, dict(attempt.headers))
+        except Exception as e:
+            last_error = e
+            continue
+
+        if status != 200:
+            last_error = Exception(f"unexpected status {status}")
+            continue
+
+        verifier = session.verifier(out)
+        try:
+            async for chunk in chunks:
+                verifier.write(chunk)
+            verifier.finish()
+            session.report_success()
+            return
+        except Exception as e:
+            last_error = e
+            if verifier.bytes_written > 0:
+                session.report_partial()
+                raise PartialWriteError(e) from e
+
+    raise AllSourcesFailedError(last_error)

fetchurl_sdk-0.1.4/mise.toml

@@ -0,0 +1,3 @@
+[tools]
+ruff = "0.14.13"
+uv = "0.9.28"

fetchurl_sdk-0.1.4/pyproject.toml

@@ -0,0 +1,30 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "fetchurl-sdk"
+version = "0.1.4"
+description = "Protocol-level client SDK for fetchurl content-addressable cache servers"
+readme = "README.md"
+requires-python = ">=3.8"
+dependencies = []
+license = { text = "MIT" }
+authors = [{ name = "lucasew" }]
+keywords = ["fetchurl", "cache", "content-addressable", "hash", "sha256"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3",
+  "Topic :: Internet :: WWW/HTTP",
+  "Topic :: Software Development :: Libraries :: Python Modules",
+]
+
+[project.urls]
+Homepage = "https://github.com/lucasew/fetchurl"
+Repository = "https://github.com/lucasew/fetchurl"
+"Bug Tracker" = "https://github.com/lucasew/fetchurl/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["fetchurl"]

fetchurl_sdk-0.1.4/test_fetchurl.py

@@ -0,0 +1,245 @@
+"""Tests for fetchurl SDK."""
+
+import hashlib
+import io
+import os
+import unittest
+from http.server import HTTPServer, BaseHTTPRequestHandler
+from threading import Thread
+from unittest.mock import patch
+
+import fetchurl
+
+
+def sha256hex(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+class TestNormalizeAlgo(unittest.TestCase):
+    def test_lowercase(self):
+        self.assertEqual(fetchurl.normalize_algo("SHA-256"), "sha256")
+
+    def test_already_normalized(self):
+        self.assertEqual(fetchurl.normalize_algo("sha256"), "sha256")
+
+    def test_strips_non_alnum(self):
+        self.assertEqual(fetchurl.normalize_algo("SHA_512"), "sha512")
+
+
+class TestIsSupported(unittest.TestCase):
+    def test_supported(self):
+        self.assertTrue(fetchurl.is_supported("sha256"))
+        self.assertTrue(fetchurl.is_supported("SHA-256"))
+        self.assertTrue(fetchurl.is_supported("sha1"))
+        self.assertTrue(fetchurl.is_supported("sha512"))
+
+    def test_unsupported(self):
+        self.assertFalse(fetchurl.is_supported("md5"))
+
+
+class TestSFV(unittest.TestCase):
+    def test_encode(self):
+        self.assertEqual(
+            fetchurl.encode_source_urls(["https://a.com", "https://b.com"]),
+            '"https://a.com", "https://b.com"',
+        )
+
+    def test_parse(self):
+        parsed = fetchurl.parse_fetchurl_server('"https://a.com", "https://b.com"')
+        self.assertEqual(parsed, ["https://a.com", "https://b.com"])
+
+    def test_roundtrip(self):
+        urls = ["https://cdn.example.com/f.tar.gz", "https://mirror.org/a.tgz"]
+        encoded = fetchurl.encode_source_urls(urls)
+        decoded = fetchurl.parse_fetchurl_server(encoded)
+        self.assertEqual(decoded, urls)
+
+    def test_parse_with_params(self):
+        parsed = fetchurl.parse_fetchurl_server('"https://a.com";q=0.9, "https://b.com"')
+        self.assertEqual(parsed, ["https://a.com", "https://b.com"])
+
+    def test_empty(self):
+        self.assertEqual(fetchurl.parse_fetchurl_server(""), [])
+
+
+class TestHashVerifier(unittest.TestCase):
+    def test_success(self):
+        data = b"hello world"
+        h = sha256hex(data)
+        out = io.BytesIO()
+        v = fetchurl.HashVerifier("sha256", h, out)
+        v.write(data)
+        self.assertEqual(v.bytes_written, len(data))
+        v.finish()
+        self.assertEqual(out.getvalue(), data)
+
+    def test_mismatch(self):
+        data = b"hello world"
+        wrong_hash = sha256hex(b"wrong")
+        out = io.BytesIO()
+        v = fetchurl.HashVerifier("sha256", wrong_hash, out)
+        v.write(data)
+        with self.assertRaises(fetchurl.HashMismatchError) as ctx:
+            v.finish()
+        self.assertEqual(ctx.exception.expected, wrong_hash)
+
+
+class TestFetchSession(unittest.TestCase):
+    def test_unsupported_algo(self):
+        with self.assertRaises(fetchurl.UnsupportedAlgorithmError):
+            fetchurl.FetchSession("md5", "abc", ["http://src"])
+
+    @patch.dict(os.environ, {"FETCHURL_SERVER": '"http://cache1", "http://cache2"'})
+    def test_attempt_ordering(self):
+        h = sha256hex(b"test")
+        session = fetchurl.FetchSession(
+            "sha256", h, ["http://src1"]
+        )
+
+        a1 = session.next_attempt()
+        self.assertIsNotNone(a1)
+        self.assertTrue(a1.url.startswith("http://cache1/api/fetchurl/sha256/"))
+        self.assertIn("X-Source-Urls", a1.headers)
+
+        a2 = session.next_attempt()
+        self.assertTrue(a2.url.startswith("http://cache2/api/fetchurl/sha256/"))
+
+        a3 = session.next_attempt()
+        self.assertEqual(a3.url, "http://src1")
+        self.assertEqual(a3.headers, {})
+
+        self.assertIsNone(session.next_attempt())
+        self.assertFalse(session.succeeded())
+
+    @patch.dict(os.environ, {"FETCHURL_SERVER": '"http://cache"'})
+    def test_success_stops(self):
+        h = sha256hex(b"test")
+        session = fetchurl.FetchSession("sha256", h, ["http://src"])
+        session.next_attempt()
+        session.report_success()
+        self.assertTrue(session.succeeded())
+        self.assertIsNone(session.next_attempt())
+
+    @patch.dict(os.environ, {"FETCHURL_SERVER": '"http://cache"'})
+    def test_partial_stops(self):
+        h = sha256hex(b"test")
+        session = fetchurl.FetchSession("sha256", h, ["http://src"])
+        session.next_attempt()
+        session.report_partial()
+        self.assertFalse(session.succeeded())
+        self.assertIsNone(session.next_attempt())
+
+
+class TestFetch(unittest.TestCase):
+    """Integration tests using a real HTTP server."""
+
+    @staticmethod
+    def _start_server(handler_class) -> tuple[HTTPServer, str]:
+        server = HTTPServer(("127.0.0.1", 0), handler_class)
+        port = server.server_address[1]
+        thread = Thread(target=server.serve_forever, daemon=True)
+        thread.start()
+        return server, f"http://127.0.0.1:{port}"
+
+    def test_direct_download(self):
+        content = b"test content"
+        h = sha256hex(content)
+
+        class Handler(BaseHTTPRequestHandler):
+            def do_GET(self):
+                self.send_response(200)
+                self.end_headers()
+                self.wfile.write(content)
+
+            def log_message(self, *args):
+                pass
+
+        server, url = self._start_server(Handler)
+        try:
+            out = io.BytesIO()
+            # Empty servers (env var not set by default/or empty)
+            with patch.dict(os.environ, {}, clear=True):
+                fetchurl.fetch(fetchurl.UrllibFetcher(), "sha256", h, [url], out)
+            self.assertEqual(out.getvalue(), content)
+        finally:
+            server.shutdown()
+
+    def test_hash_mismatch_raises_partial(self):
+        class Handler(BaseHTTPRequestHandler):
+            def do_GET(self):
+                self.send_response(200)
+                self.end_headers()
+                self.wfile.write(b"wrong content")
+
+            def log_message(self, *args):
+                pass
+
+        server, url = self._start_server(Handler)
+        try:
+            out = io.BytesIO()
+            with self.assertRaises(fetchurl.PartialWriteError):
+                with patch.dict(os.environ, {}, clear=True):
+                    fetchurl.fetch(
+                        fetchurl.UrllibFetcher(), "sha256", sha256hex(b"right"), [url], out
+                    )
+        finally:
+            server.shutdown()
+
+    def test_all_sources_failed(self):
+        class Handler(BaseHTTPRequestHandler):
+            def do_GET(self):
+                self.send_response(404)
+                self.end_headers()
+
+            def log_message(self, *args):
+                pass
+
+        server, url = self._start_server(Handler)
+        try:
+            out = io.BytesIO()
+            with self.assertRaises(fetchurl.AllSourcesFailedError):
+                with patch.dict(os.environ, {}, clear=True):
+                    fetchurl.fetch(
+                        fetchurl.UrllibFetcher(), "sha256", sha256hex(b"x"), [url], out
+                    )
+        finally:
+            server.shutdown()
+
+    def test_server_fallback_to_direct(self):
+        content = b"fallback content"
+        h = sha256hex(content)
+
+        class BadServer(BaseHTTPRequestHandler):
+            def do_GET(self):
+                self.send_response(500)
+                self.end_headers()
+
+            def log_message(self, *args):
+                pass
+
+        class GoodSource(BaseHTTPRequestHandler):
+            def do_GET(self):
+                self.send_response(200)
+                self.end_headers()
+                self.wfile.write(content)
+
+            def log_message(self, *args):
+                pass
+
+        bad, bad_url = self._start_server(BadServer)
+        good, good_url = self._start_server(GoodSource)
+        try:
+            out = io.BytesIO()
+            # Set server via env var
+            with patch.dict(os.environ, {"FETCHURL_SERVER": f'"{bad_url}"'}):
+                fetchurl.fetch(
+                    fetchurl.UrllibFetcher(), "sha256", h, [good_url], out
+                )
+            self.assertEqual(out.getvalue(), content)
+        finally:
+            bad.shutdown()
+            good.shutdown()
+
+
+if __name__ == "__main__":
+    unittest.main()

fetchurl_sdk-0.1.4/uv.lock

@@ -0,0 +1,8 @@
+version = 1
+revision = 3
+requires-python = ">=3.13"
+
+[[package]]
+name = "fetchurl-sdk"
+version = "0.1.0"
+source = { virtual = "." }