velocity-python 0.1.24__tar.gz → 0.1.26__tar.gz

{velocity_python-0.1.24/src/velocity_python.egg-info → velocity_python-0.1.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: velocity-python
-Version: 0.1.24
+Version: 0.1.26
 Summary: A rapid application development library for interfacing with data storage
 Author-email: Velocity Team <info@codeclubs.org>
 License-Expression: MIT

{velocity_python-0.1.24 → velocity_python-0.1.26}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "velocity-python"
-version = "0.1.24"
+version = "0.1.26"
 authors = [
   { name="Velocity Team", email="info@codeclubs.org" },
 ]

{velocity_python-0.1.24 → velocity_python-0.1.26}/src/velocity/__init__.py

@@ -1,4 +1,4 @@
-__version__ = version = "0.1.24"
+__version__ = version = "0.1.26"
 
 import importlib as _importlib
 

{velocity_python-0.1.24 → velocity_python-0.1.26}/src/velocity/aws/__init__.py

@@ -38,6 +38,26 @@ _LAZY_ATTRS = {
     "BACKOFFICE_BUCKET": "velocity.aws.s3",
     "PRIVATE_BUCKETS": "velocity.aws.s3",
     "ALLOWED_UPLOAD_BUCKETS": "velocity.aws.s3",
+    # Asset helpers
+    "AssetReference": "velocity.aws.assets",
+    "AssetUsage": "velocity.aws.assets",
+    "AssetUsageReconciliation": "velocity.aws.assets",
+    "AssetBackfillSummary": "velocity.aws.assets",
+    "AssetRecord": "velocity.aws.assets",
+    "AssetUploadResult": "velocity.aws.assets",
+    "backfill_asset_records": "velocity.aws.assets",
+    "build_asset_key": "velocity.aws.assets",
+    "build_asset_usages": "velocity.aws.assets",
+    "build_backfill_asset_data": "velocity.aws.assets",
+    "find_existing_asset": "velocity.aws.assets",
+    "extract_asset_references_from_html": "velocity.aws.assets",
+    "extract_asset_references_from_record": "velocity.aws.assets",
+    "extract_asset_references_from_value": "velocity.aws.assets",
+    "iter_s3_objects": "velocity.aws.assets",
+    "list_assets": "velocity.aws.assets",
+    "normalize_asset_record": "velocity.aws.assets",
+    "reconcile_asset_usages": "velocity.aws.assets",
+    "upload_image_asset": "velocity.aws.assets",
 }
 
 
@@ -67,4 +87,23 @@ __all__ = [
     "BACKOFFICE_BUCKET",
     "PRIVATE_BUCKETS",
     "ALLOWED_UPLOAD_BUCKETS",
+    "AssetBackfillSummary",
+    "AssetReference",
+    "AssetRecord",
+    "AssetUploadResult",
+    "AssetUsage",
+    "AssetUsageReconciliation",
+    "backfill_asset_records",
+    "build_asset_key",
+    "build_asset_usages",
+    "build_backfill_asset_data",
+    "extract_asset_references_from_html",
+    "extract_asset_references_from_record",
+    "extract_asset_references_from_value",
+    "find_existing_asset",
+    "iter_s3_objects",
+    "list_assets",
+    "normalize_asset_record",
+    "reconcile_asset_usages",
+    "upload_image_asset",
 ]

velocity_python-0.1.26/src/velocity/aws/assets/__init__.py

@@ -0,0 +1,57 @@
+from velocity.aws.assets.references import (
+    AssetReference,
+    extract_asset_references_from_html,
+    extract_asset_references_from_record,
+    extract_asset_references_from_value,
+)
+from velocity.aws.assets.backfill import (
+    AssetBackfillSummary,
+    backfill_asset_records,
+    build_backfill_asset_data,
+    iter_s3_objects,
+)
+from velocity.aws.assets.service import (
+    AssetRecord,
+    AssetUploadResult,
+    build_asset_key,
+    find_existing_asset,
+    list_assets,
+    normalize_asset_record,
+    upload_image_asset,
+)
+from velocity.aws.assets.usage_index import (
+    AssetUsage,
+    AssetUsageReconciliation,
+    build_asset_usages,
+    reconcile_asset_usages,
+)
+from velocity.aws.assets.indexing import (
+    FORM_BUILDER_TEMPLATE_HTML_FIELDS,
+    FORM_BUILDER_TEMPLATE_IMAGE_FIELDS,
+    reconcile_form_builder_template_asset_usages,
+)
+
+__all__ = [
+    "AssetReference",
+    "AssetBackfillSummary",
+    "AssetRecord",
+    "AssetUploadResult",
+    "AssetUsage",
+    "AssetUsageReconciliation",
+    "backfill_asset_records",
+    "build_asset_key",
+    "build_asset_usages",
+    "build_backfill_asset_data",
+    "extract_asset_references_from_html",
+    "extract_asset_references_from_record",
+    "extract_asset_references_from_value",
+    "find_existing_asset",
+    "FORM_BUILDER_TEMPLATE_HTML_FIELDS",
+    "FORM_BUILDER_TEMPLATE_IMAGE_FIELDS",
+    "iter_s3_objects",
+    "list_assets",
+    "normalize_asset_record",
+    "reconcile_form_builder_template_asset_usages",
+    "reconcile_asset_usages",
+    "upload_image_asset",
+]

velocity_python-0.1.26/src/velocity/aws/assets/backfill.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+import os
+from typing import Any, Callable, Iterable, Mapping
+
+from velocity.aws.assets.service import AssetRecord, normalize_asset_record
+from velocity.aws.s3 import PRIVATE_BUCKETS
+
+
+HeadLoader = Callable[[str, str], Mapping[str, Any] | None]
+
+
+@dataclass(frozen=True)
+class AssetBackfillSummary:
+    scanned: int
+    inserted: int
+    skipped_existing: int
+    skipped_unsupported: int
+
+
+def iter_s3_objects(
+    bucket: str,
+    *,
+    prefix: str = "",
+    s3_client=None,
+) -> Iterable[dict[str, Any]]:
+    if s3_client is None:
+        import boto3
+
+        s3_client = boto3.client("s3")
+    paginator = s3_client.get_paginator("list_objects_v2")
+    for page in paginator.paginate(Bucket=bucket, Prefix=prefix):
+        for row in page.get("Contents", []):
+            yield row
+
+
+def build_backfill_asset_data(
+    bucket: str,
+    object_info: Mapping[str, Any],
+    *,
+    head_info: Mapping[str, Any] | None = None,
+) -> dict[str, Any] | None:
+    key = str(object_info.get("Key") or "").strip()
+    if not key or _is_thumbnail_key(key) or key.endswith("/"):
+        return None
+    metadata = dict((head_info or {}).get("Metadata") or {})
+    filename = os.path.basename(key)
+    file_ext = os.path.splitext(filename)[1][1:].lower() or None
+    row = {
+        "bucket": bucket,
+        "key": key,
+        "url": f"https://s3.amazonaws.com/{bucket}/{key}",
+        "public": bucket not in PRIVATE_BUCKETS,
+        "name": filename,
+        "type": file_ext,
+        "etag": _clean_etag(object_info.get("ETag") or (head_info or {}).get("ETag")),
+        "filesize": _coerce_int(object_info.get("Size") or (head_info or {}).get("ContentLength")),
+        "hash": metadata.get("hash"),
+        "thumbnail_key": metadata.get("thumbnail_key"),
+        "client": None if key.startswith("site/uploads") else key.split("/", 1)[0],
+    }
+    row.update(_copy_if_present(metadata, ["format", "width", "height", "is_animated"]))
+    return row
+
+
+def backfill_asset_records(
+    tx,
+    *,
+    bucket: str,
+    objects: Iterable[Mapping[str, Any]],
+    head_loader: HeadLoader | None = None,
+) -> tuple[list[AssetRecord], AssetBackfillSummary]:
+    inserted_assets: list[AssetRecord] = []
+    scanned = 0
+    inserted = 0
+    skipped_existing = 0
+    skipped_unsupported = 0
+
+    for object_info in objects:
+        scanned += 1
+        key = str(object_info.get("Key") or "").strip()
+        if not key or _is_thumbnail_key(key) or key.endswith("/"):
+            skipped_unsupported += 1
+            continue
+        if tx.table("images").find({"key": key}):
+            skipped_existing += 1
+            continue
+        head_info = head_loader(bucket, key) if head_loader else None
+        row = build_backfill_asset_data(bucket, object_info, head_info=head_info)
+        if not row:
+            skipped_unsupported += 1
+            continue
+        tx.table("images").upsert(row, {"key": row["key"]})
+        inserted += 1
+        stored = tx.table("images").find({"key": row["key"]})
+        inserted_assets.append(normalize_asset_record(stored.to_dict() if stored else row))
+
+    return inserted_assets, AssetBackfillSummary(
+        scanned=scanned,
+        inserted=inserted,
+        skipped_existing=skipped_existing,
+        skipped_unsupported=skipped_unsupported,
+    )
+
+
+def _is_thumbnail_key(key: str) -> bool:
+    stem, ext = os.path.splitext(key)
+    return stem.endswith("-thumb") and ext.lower() == ".png"
+
+
+def _clean_etag(value: Any) -> str | None:
+    if value is None:
+        return None
+    text = str(value).strip().strip('"')
+    return text or None
+
+
+def _coerce_int(value: Any) -> int | None:
+    if value in (None, ""):
+        return None
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _copy_if_present(source: Mapping[str, Any], keys: list[str]) -> dict[str, Any]:
+    result: dict[str, Any] = {}
+    for key in keys:
+        if key not in source:
+            continue
+        value = source[key]
+        if key in {"width", "height"}:
+            result[key] = _coerce_int(value)
+        elif key == "is_animated":
+            result[key] = str(value).lower() in {"1", "true", "yes", "y"}
+        else:
+            result[key] = value
+    return result

velocity_python-0.1.26/src/velocity/aws/assets/indexing.py

@@ -0,0 +1,217 @@
+from __future__ import annotations
+
+from typing import Any, Mapping
+
+from velocity.aws.assets.references import AssetReference, extract_asset_references_from_record
+from velocity.aws.assets.usage_index import (
+    AssetUsage,
+    AssetUsageReconciliation,
+    build_asset_usages,
+    reconcile_asset_usages,
+)
+
+
+FORM_BUILDER_TEMPLATE_HTML_FIELDS = (
+    "body_message_text",
+    "form_instructions",
+    "confirmation_body_text",
+    "receipt_body_text",
+    "rally_receipt_body_text",
+    "pii_instructions",
+    "metric_section_instructions",
+    "donation_section_instructions",
+    *tuple(f"donation_{index}_html" for index in range(1, 10)),
+)
+
+FORM_BUILDER_TEMPLATE_IMAGE_FIELDS = (
+    "receipt_header_image",
+    "secure_image_url",
+    "header_banner_image_url",
+    "background_image_url",
+    "main_image_url",
+    "body_header_url",
+    "spot_image",
+    "confirmation_header_image",
+    "confirmation_body_image",
+    *tuple(f"donation_{index}_image" for index in range(1, 10)),
+    *tuple(f"metric_{index}_image" for index in range(1, 10)),
+)
+
+
+def reconcile_form_builder_template_asset_usages(
+    tx,
+    row: Mapping[str, Any] | Any,
+    *,
+    html_fields=FORM_BUILDER_TEMPLATE_HTML_FIELDS,
+    image_fields=FORM_BUILDER_TEMPLATE_IMAGE_FIELDS,
+    last_indexed_by: str = "velocity.aws.dirty_pipeline",
+) -> AssetUsageReconciliation:
+    source_row = _row_to_dict(row)
+    source_sys_id = _as_string(source_row.get("sys_id"))
+    if not source_sys_id:
+        raise ValueError("form_builder_template row must include sys_id")
+
+    references = extract_asset_references_from_record(
+        source_row,
+        html_fields=html_fields,
+        image_fields=image_fields,
+    )
+    desired = build_asset_usages(
+        references,
+        source_table="form_builder_template",
+        source_sys_id=source_sys_id,
+        source_record_label=_build_source_record_label(source_row),
+        last_source_modified=_as_string(source_row.get("sys_modified")),
+        last_indexed_by=last_indexed_by,
+        asset_id_resolver=lambda reference: resolve_asset_id_for_reference(tx, reference),
+    )
+    existing = load_asset_usages(
+        tx,
+        source_table="form_builder_template",
+        source_sys_id=source_sys_id,
+    )
+    reconciliation = reconcile_asset_usages(existing, desired)
+    apply_asset_usage_reconciliation(tx, reconciliation)
+    return reconciliation
+
+
+def load_asset_usages(tx, *, source_table: str, source_sys_id: str) -> list[AssetUsage]:
+    rows = tx.table("asset_usage_index").select(
+        where={
+            "source_table": source_table,
+            "source_sys_id": source_sys_id,
+        }
+    ).all()
+    return [_row_to_asset_usage(row) for row in rows]
+
+
+def apply_asset_usage_reconciliation(tx, reconciliation: AssetUsageReconciliation) -> None:
+    table = tx.table("asset_usage_index")
+    for usage in reconciliation.to_create + reconciliation.to_refresh:
+        row = _asset_usage_to_row(usage)
+        table.upsert(
+            row,
+            {
+                "asset_id": row["asset_id"],
+                "source_table": row["source_table"],
+                "source_sys_id": row["source_sys_id"],
+                "source_field": row["source_field"],
+            },
+        )
+
+    for usage in reconciliation.to_delete:
+        table.delete(
+            where={
+                "asset_id": usage.asset_id,
+                "source_table": usage.source_table,
+                "source_sys_id": usage.source_sys_id,
+                "source_field": usage.source_field,
+            }
+        )
+
+
+def resolve_asset_id_for_reference(tx, reference: AssetReference) -> str | None:
+    if reference.asset_id:
+        return reference.asset_id
+
+    images = tx.table("images")
+    for candidate_key in _candidate_storage_keys(reference.storage_key):
+        row = images.find({"key": candidate_key})
+        asset_id = _extract_sys_id(row)
+        if asset_id:
+            return asset_id
+
+    if reference.url:
+        row = images.find({"url": reference.url})
+        asset_id = _extract_sys_id(row)
+        if asset_id:
+            return asset_id
+
+    return None
+
+
+def _candidate_storage_keys(storage_key: str | None) -> list[str]:
+    if not storage_key:
+        return []
+
+    raw_value = storage_key.strip()
+    normalized_value = raw_value.lstrip("/")
+    candidates = [
+        raw_value,
+        normalized_value,
+        f"/{normalized_value}",
+        f"//{normalized_value}",
+    ]
+    deduped: list[str] = []
+    for candidate in candidates:
+        if candidate and candidate not in deduped:
+            deduped.append(candidate)
+    return deduped
+
+
+def _row_to_asset_usage(row: Mapping[str, Any] | Any) -> AssetUsage:
+    data = _row_to_dict(row)
+    return AssetUsage(
+        asset_id=_as_string(data.get("asset_id")) or "",
+        source_table=_as_string(data.get("source_table")) or "",
+        source_sys_id=_as_string(data.get("source_sys_id")) or "",
+        source_field=_as_string(data.get("source_field")) or "",
+        source_field_type=_as_string(data.get("source_field_type")) or "",
+        asset_reference_value=_as_string(data.get("asset_reference_value")),
+        asset_storage_key_snapshot=_as_string(data.get("asset_storage_key_snapshot")),
+        asset_public_url_snapshot=_as_string(data.get("asset_public_url_snapshot")),
+        source_record_label=_as_string(data.get("source_record_label")),
+        last_source_modified=_as_string(data.get("last_source_modified")),
+        last_indexed_by=_as_string(data.get("last_indexed_by")),
+    )
+
+
+def _asset_usage_to_row(usage: AssetUsage) -> dict[str, Any]:
+    return {
+        "asset_id": usage.asset_id,
+        "source_table": usage.source_table,
+        "source_sys_id": usage.source_sys_id,
+        "source_field": usage.source_field,
+        "source_field_type": usage.source_field_type,
+        "asset_reference_value": usage.asset_reference_value,
+        "asset_storage_key_snapshot": usage.asset_storage_key_snapshot,
+        "asset_public_url_snapshot": usage.asset_public_url_snapshot,
+        "source_record_label": usage.source_record_label,
+        "last_source_modified": usage.last_source_modified,
+        "last_indexed_by": usage.last_indexed_by,
+    }
+
+
+def _build_source_record_label(row: Mapping[str, Any]) -> str | None:
+    for key in ("description", "page_title", "campaign"):
+        label = _as_string(row.get(key))
+        if label:
+            return label
+
+    client = _as_string(row.get("client"))
+    campaign = _as_string(row.get("campaign"))
+    if client and campaign:
+        return f"{client} - {campaign}"
+    return client or campaign
+
+
+def _extract_sys_id(row: Mapping[str, Any] | Any) -> str | None:
+    data = _row_to_dict(row)
+    return _as_string(data.get("sys_id"))
+
+
+def _row_to_dict(row: Mapping[str, Any] | Any) -> dict[str, Any]:
+    if row is None:
+        return {}
+    if hasattr(row, "to_dict"):
+        return row.to_dict()
+    if isinstance(row, Mapping):
+        return dict(row)
+    raise TypeError(f"Unsupported row type: {type(row)!r}")
+
+
+def _as_string(value: Any) -> str | None:
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text or None