memuron 0.1.1__py3-none-any.whl

memuron/documents/storage.py

@@ -0,0 +1,151 @@
+"""Durable object storage for original document uploads."""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from dataclasses import dataclass
+from typing import Any
+
+from memuron.application.config import settings
+
+
+def _safe_segment(value: str, *, fallback: str) -> str:
+    cleaned = re.sub(r"[^A-Za-z0-9._-]+", "-", value).strip(".-")
+    return cleaned or fallback
+
+
+def _is_configured() -> bool:
+    return bool(
+        settings.object_storage_bucket
+        and settings.object_storage_access_key_id
+        and settings.object_storage_secret_access_key
+    )
+
+
+@dataclass(frozen=True)
+class SourceObject:
+    provider: str
+    bucket: str
+    key: str
+    content_type: str
+    file_name: str
+    size_bytes: int
+    sha256: str
+    public_url: str | None = None
+
+    def as_payload(self) -> dict[str, Any]:
+        payload: dict[str, Any] = {
+            "provider": self.provider,
+            "bucket": self.bucket,
+            "key": self.key,
+            "content_type": self.content_type,
+            "file_name": self.file_name,
+            "size_bytes": self.size_bytes,
+            "sha256": self.sha256,
+        }
+        if self.public_url:
+            payload["public_url"] = self.public_url
+        return payload
+
+
+class DocumentStorage:
+    """S3-compatible storage facade for original source files."""
+
+    def __init__(self) -> None:
+        self.bucket = settings.object_storage_bucket
+
+    @property
+    def configured(self) -> bool:
+        return _is_configured()
+
+    def _client(self) -> Any:
+        import boto3
+        from botocore.config import Config
+
+        return boto3.client(
+            "s3",
+            endpoint_url=settings.object_storage_endpoint_url or None,
+            region_name=settings.object_storage_region or None,
+            aws_access_key_id=settings.object_storage_access_key_id,
+            aws_secret_access_key=settings.object_storage_secret_access_key,
+            config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
+        )
+
+    def put_source(
+        self,
+        *,
+        org_id: str | None,
+        document_key: str,
+        file_name: str,
+        content_type: str | None,
+        file_bytes: bytes,
+    ) -> dict[str, Any] | None:
+        if not self.configured:
+            return None
+        safe_org = _safe_segment(org_id or "no-org", fallback="no-org")
+        safe_name = _safe_segment(file_name, fallback="document")
+        key = f"org/{safe_org}/documents/{document_key}/source/{safe_name}"
+        media_type = content_type or "application/octet-stream"
+        digest = hashlib.sha256(file_bytes).hexdigest()
+        self._client().put_object(
+            Bucket=self.bucket,
+            Key=key,
+            Body=file_bytes,
+            ContentType=media_type,
+            Metadata={
+                "document-key": document_key,
+                "sha256": digest,
+                "original-file-name": safe_name,
+            },
+        )
+        public_url = None
+        if settings.object_storage_public_base_url:
+            public_url = f"{settings.object_storage_public_base_url}/{key}"
+        return SourceObject(
+            provider="s3",
+            bucket=self.bucket,
+            key=key,
+            content_type=media_type,
+            file_name=file_name,
+            size_bytes=len(file_bytes),
+            sha256=digest,
+            public_url=public_url,
+        ).as_payload()
+
+    def presign_download(self, source_object: dict[str, Any]) -> str | None:
+        public_url = source_object.get("public_url")
+        if isinstance(public_url, str) and public_url:
+            return public_url
+        if not self.configured:
+            return None
+        bucket = str(source_object.get("bucket") or self.bucket)
+        key = str(source_object.get("key") or "")
+        if not bucket or not key:
+            return None
+        return self._client().generate_presigned_url(
+            "get_object",
+            Params={"Bucket": bucket, "Key": key},
+            ExpiresIn=settings.object_storage_presign_seconds,
+        )
+
+
+def maybe_store_source_file(
+    *,
+    org_id: str | None,
+    document_key: str,
+    file_name: str,
+    content_type: str | None,
+    file_bytes: bytes,
+) -> dict[str, Any] | None:
+    return DocumentStorage().put_source(
+        org_id=org_id,
+        document_key=document_key,
+        file_name=file_name,
+        content_type=content_type,
+        file_bytes=file_bytes,
+    )
+
+
+def presign_source_object(source_object: dict[str, Any]) -> str | None:
+    return DocumentStorage().presign_download(source_object)

memuron/documents/url_ingest.py

@@ -0,0 +1,375 @@
+"""Safe URL fetching and normalization for document ingest."""
+
+from __future__ import annotations
+
+import ipaddress
+import re
+import socket
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from html.parser import HTMLParser
+from pathlib import Path
+from typing import Any
+from urllib.parse import unquote, urljoin, urlparse
+
+import requests
+
+from memuron.documents.parser import MAX_DOCUMENT_UPLOAD_BYTES
+
+
+DEFAULT_URL_FETCH_TIMEOUT_SECONDS = 15
+MAX_URL_REDIRECTS = 5
+URL_FETCH_USER_AGENT = "MemuronURLIngest/1.0"
+
+
+class UrlIngestError(ValueError):
+    """Raised when a URL cannot be safely fetched for document ingest."""
+
+
+@dataclass(frozen=True)
+class FetchedUrlSource:
+    file_name: str
+    content_type: str | None
+    file_bytes: bytes
+    metadata: dict[str, Any]
+
+
+def fetch_url_source(
+    url: str,
+    *,
+    timeout_seconds: int = DEFAULT_URL_FETCH_TIMEOUT_SECONDS,
+    max_bytes: int = MAX_DOCUMENT_UPLOAD_BYTES,
+    session: requests.Session | None = None,
+) -> FetchedUrlSource:
+    """Fetch and normalize a remote URL into a source document payload."""
+
+    current_url = _validate_fetch_url(url)
+    client = session or requests.Session()
+    fetched_at = datetime.now(UTC).isoformat()
+    response = None
+    for _redirect in range(MAX_URL_REDIRECTS + 1):
+        try:
+            response = client.get(
+                current_url,
+                headers={"User-Agent": URL_FETCH_USER_AGENT},
+                timeout=timeout_seconds,
+                stream=True,
+                allow_redirects=False,
+            )
+        except requests.Timeout as exc:
+            raise UrlIngestError("URL fetch timed out") from exc
+        except requests.RequestException as exc:
+            raise UrlIngestError(f"URL fetch failed: {exc}") from exc
+
+        if 300 <= response.status_code < 400:
+            location = response.headers.get("Location")
+            if not location:
+                raise UrlIngestError("URL redirect is missing a Location header")
+            current_url = _validate_fetch_url(urljoin(current_url, location))
+            continue
+        break
+    else:  # pragma: no cover - for-loop always breaks before this branch
+        raise UrlIngestError("URL has too many redirects")
+
+    assert response is not None
+    final_url = _validate_fetch_url(getattr(response, "url", current_url) or current_url)
+    if response.status_code != 200:
+        raise UrlIngestError(f"URL returned HTTP {response.status_code}")
+
+    raw_content_type = response.headers.get("Content-Type")
+    content_type = _content_type(raw_content_type)
+    raw_bytes = _read_limited_response(response, max_bytes=max_bytes)
+    if _is_html(content_type, final_url):
+        markdown, title = html_to_markdown(raw_bytes, content_type=raw_content_type)
+        file_name = _url_file_name(final_url, content_type="text/markdown", title=title)
+        metadata_content_type = content_type or "text/html"
+        return FetchedUrlSource(
+            file_name=file_name,
+            content_type="text/markdown",
+            file_bytes=markdown.encode("utf-8"),
+            metadata=_fetch_metadata(
+                source_url=url,
+                final_url=final_url,
+                content_type=metadata_content_type,
+                fetched_at=fetched_at,
+                title=title,
+                normalized_content_type="text/markdown",
+                size_bytes=len(raw_bytes),
+            ),
+        )
+
+    file_name = _url_file_name(final_url, content_type=content_type)
+    return FetchedUrlSource(
+        file_name=file_name,
+        content_type=content_type,
+        file_bytes=raw_bytes,
+        metadata=_fetch_metadata(
+            source_url=url,
+            final_url=final_url,
+            content_type=content_type,
+            fetched_at=fetched_at,
+            title=None,
+            normalized_content_type=None,
+            size_bytes=len(raw_bytes),
+        ),
+    )
+
+
+def html_to_markdown(file_bytes: bytes, *, content_type: str | None = None) -> tuple[str, str | None]:
+    """Extract readable markdown-ish text from HTML without heavy dependencies."""
+
+    parser = _ReadableHTMLParser()
+    parser.feed(_decode_html(file_bytes, content_type=content_type))
+    parser.close()
+    title = _normalize_whitespace(parser.title).strip() or None
+    body = _clean_markdown_lines("".join(parser.parts))
+    if title and not body.lstrip().startswith("#"):
+        body = f"# {title}\n\n{body}".strip()
+    if not body.strip():
+        raise UrlIngestError("HTML page did not contain readable text")
+    return body.strip() + "\n", title
+
+
+class _ReadableHTMLParser(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.parts: list[str] = []
+        self._title_parts: list[str] = []
+        self._skip_depth = 0
+        self._in_title = False
+
+    @property
+    def title(self) -> str:
+        return "".join(self._title_parts)
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        tag = tag.lower()
+        if tag in {"script", "style", "noscript", "svg", "template"}:
+            self._skip_depth += 1
+            return
+        if tag == "title":
+            self._in_title = True
+            return
+        if self._skip_depth:
+            return
+        if tag in {"h1", "h2", "h3", "h4", "h5", "h6"}:
+            level = int(tag[1])
+            self.parts.append("\n\n" + "#" * level + " ")
+        elif tag in {"p", "div", "section", "article", "main", "header", "footer", "tr"}:
+            self.parts.append("\n\n")
+        elif tag == "li":
+            self.parts.append("\n- ")
+        elif tag == "br":
+            self.parts.append("\n")
+        elif tag in {"td", "th"}:
+            self.parts.append(" | ")
+
+    def handle_endtag(self, tag: str) -> None:
+        tag = tag.lower()
+        if tag in {"script", "style", "noscript", "svg", "template"} and self._skip_depth:
+            self._skip_depth -= 1
+            return
+        if tag == "title":
+            self._in_title = False
+            return
+        if self._skip_depth:
+            return
+        if tag in {
+            "h1",
+            "h2",
+            "h3",
+            "h4",
+            "h5",
+            "h6",
+            "p",
+            "div",
+            "section",
+            "article",
+            "main",
+            "li",
+            "tr",
+        }:
+            self.parts.append("\n")
+
+    def handle_data(self, data: str) -> None:
+        if self._in_title:
+            self._title_parts.append(data)
+            return
+        if self._skip_depth:
+            return
+        text = _normalize_whitespace(data)
+        if text:
+            self.parts.append(text)
+
+
+def _validate_fetch_url(url: str) -> str:
+    raw = str(url or "").strip()
+    parsed = urlparse(raw)
+    if parsed.scheme.lower() not in {"http", "https"}:
+        raise UrlIngestError("URL ingest only supports http and https URLs")
+    if not parsed.hostname:
+        raise UrlIngestError("URL must include a host")
+    host = parsed.hostname.strip().lower()
+    if host in {"localhost", "localhost.localdomain"} or host.endswith(".localhost"):
+        raise UrlIngestError("URL host is not allowed")
+    _reject_private_host(host)
+    return raw
+
+
+def _reject_private_host(host: str) -> None:
+    try:
+        addresses = [ipaddress.ip_address(host)]
+    except ValueError:
+        try:
+            infos = socket.getaddrinfo(host, None, type=socket.SOCK_STREAM)
+        except socket.gaierror as exc:
+            raise UrlIngestError(f"Could not resolve URL host: {host}") from exc
+        addresses = []
+        for info in infos:
+            raw_address = info[4][0]
+            try:
+                addresses.append(ipaddress.ip_address(raw_address))
+            except ValueError:
+                continue
+    if not addresses:
+        raise UrlIngestError(f"Could not resolve URL host: {host}")
+    for address in addresses:
+        if (
+            address.is_private
+            or address.is_loopback
+            or address.is_link_local
+            or address.is_multicast
+            or address.is_reserved
+            or address.is_unspecified
+        ):
+            raise UrlIngestError("URL host resolves to a private or local address")
+
+
+def _read_limited_response(response: requests.Response, *, max_bytes: int) -> bytes:
+    content_length = response.headers.get("Content-Length")
+    if content_length:
+        try:
+            if int(content_length) > max_bytes:
+                raise UrlIngestError(
+                    f"URL response is too large; max is {max_bytes // (1024 * 1024)} MB"
+                )
+        except ValueError:
+            if content_length.strip().isdigit():
+                raise
+    chunks: list[bytes] = []
+    total = 0
+    for chunk in response.iter_content(chunk_size=64 * 1024):
+        if not chunk:
+            continue
+        total += len(chunk)
+        if total > max_bytes:
+            raise UrlIngestError(
+                f"URL response is too large; max is {max_bytes // (1024 * 1024)} MB"
+            )
+        chunks.append(chunk)
+    return b"".join(chunks)
+
+
+def _content_type(value: str | None) -> str | None:
+    if not value:
+        return None
+    return value.split(";", 1)[0].strip().lower() or None
+
+
+def _is_html(content_type: str | None, url: str) -> bool:
+    path = urlparse(url).path.lower()
+    return content_type in {"text/html", "application/xhtml+xml"} or path.endswith((".html", ".htm"))
+
+
+def _decode_html(file_bytes: bytes, *, content_type: str | None) -> str:
+    charset = None
+    if content_type:
+        match = re.search(r"charset=([A-Za-z0-9._-]+)", content_type, flags=re.IGNORECASE)
+        if match:
+            charset = match.group(1)
+    for encoding in [charset, "utf-8", "latin-1"]:
+        if not encoding:
+            continue
+        try:
+            return file_bytes.decode(encoding)
+        except (LookupError, UnicodeDecodeError):
+            continue
+    return file_bytes.decode("utf-8", errors="replace")
+
+
+def _clean_markdown_lines(text: str) -> str:
+    lines = [_normalize_whitespace(line).strip() for line in text.splitlines()]
+    output: list[str] = []
+    blank = False
+    for line in lines:
+        if not line:
+            if output and not blank:
+                output.append("")
+            blank = True
+            continue
+        output.append(line)
+        blank = False
+    return "\n".join(output).strip()
+
+
+def _normalize_whitespace(text: str) -> str:
+    return re.sub(r"[ \t\r\f\v]+", " ", text.replace("\xa0", " "))
+
+
+def _url_file_name(url: str, *, content_type: str | None, title: str | None = None) -> str:
+    if title:
+        stem = re.sub(r"[^A-Za-z0-9._-]+", "-", title).strip(".-").lower()
+        if stem:
+            return f"{stem[:80]}.md"
+    path_name = Path(unquote(urlparse(url).path or "")).name
+    if path_name and "." in path_name:
+        return re.sub(r"[^A-Za-z0-9._-]+", "-", path_name).strip(".-") or "url-document"
+    stem = re.sub(r"[^A-Za-z0-9._-]+", "-", path_name or urlparse(url).hostname or "url-document")
+    stem = stem.strip(".-") or "url-document"
+    return f"{stem[:80]}{_extension_for_content_type(content_type)}"
+
+
+def _extension_for_content_type(content_type: str | None) -> str:
+    return {
+        "application/pdf": ".pdf",
+        "application/json": ".json",
+        "application/ld+json": ".json",
+        "application/xml": ".xml",
+        "text/xml": ".xml",
+        "text/csv": ".csv",
+        "text/tab-separated-values": ".tsv",
+        "text/markdown": ".md",
+        "text/x-markdown": ".md",
+        "text/html": ".html",
+        "application/xhtml+xml": ".html",
+        "text/plain": ".txt",
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
+        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
+        "application/vnd.ms-excel": ".xls",
+        "application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
+        "application/vnd.ms-powerpoint": ".ppt",
+    }.get(content_type or "", ".txt")
+
+
+def _fetch_metadata(
+    *,
+    source_url: str,
+    final_url: str,
+    content_type: str | None,
+    fetched_at: str,
+    title: str | None,
+    normalized_content_type: str | None,
+    size_bytes: int,
+) -> dict[str, Any]:
+    metadata: dict[str, Any] = {
+        "source_url": source_url,
+        "fetched_url": final_url,
+        "content_type": content_type,
+        "fetched_at": fetched_at,
+        "size_bytes": size_bytes,
+    }
+    if title:
+        metadata["title"] = title
+    if normalized_content_type:
+        metadata["normalized_content_type"] = normalized_content_type
+    return metadata

memuron/domain/__init__.py

@@ -0,0 +1 @@
+"""Shared domain models, schemas, encoders, and validation helpers."""

memuron/domain/decoders.py

@@ -0,0 +1 @@
+"""Product-specific decoders live here."""