ezMM 0.3.5__py3-none-any.whl

ezmm/__init__.py

@@ -0,0 +1,2 @@
+from ezmm.common import (MultimodalSequence, Item, Image, Video, download_image, download_item,
+                         download_video)  # TODO: Add more

ezmm/common/__init__.py

@@ -0,0 +1,21 @@
+import sys
+from pathlib import Path
+
+from ezmm.common.items import *
+from ezmm.common.multimodal_sequence import MultimodalSequence
+from ezmm.common.registry import ItemRegistry
+import logging
+
+APP_NAME = "ezMM"
+PROJECT_ROOT = Path(__file__).parent.parent.parent
+
+# Set up logger
+logger = logging.getLogger(APP_NAME)
+logger.setLevel(logging.DEBUG)
+
+# Only add handler if none exists (avoid duplicate logs on rerun)
+if not logger.hasHandlers():
+    handler = logging.StreamHandler(sys.stdout)
+    formatter = logging.Formatter('[%(levelname)s]: %(message)s')
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)

ezmm/common/items/__init__.py

@@ -0,0 +1,15 @@
+from ezmm.common.items.item import Item, REF, download_item
+from ezmm.common.items.image import Image, download_image
+from ezmm.common.items.video import Video, download_video
+from ezmm.common.items.audio import Audio
+
+ITEM_CLASSES = [Image, Video, Audio]
+KIND2ITEM = {item.kind: item for item in ITEM_CLASSES}
+KINDS = [item.kind for item in ITEM_CLASSES]
+
+# Regex patterns
+KINDS_ALTERNATIVES = "|".join(KINDS)
+ITEM_REF_REGEX = "(" + REF.format(kind=f"(?:{KINDS_ALTERNATIVES})", id="[0-9]+") + ")"  # Captures full ref
+ITEM_ID_REGEX = REF.format(kind=f"(?:{KINDS_ALTERNATIVES})", id="([0-9]+)")  # Captures only ID
+ITEM_KIND_REGEX = REF.format(kind=f"({KINDS_ALTERNATIVES})", id="[0-9]+")  # Captures only kind
+ITEM_KIND_ID_REGEX = REF.format(kind=f"({KINDS_ALTERNATIVES})", id="([0-9]+)")  # Captures kind and ID

ezmm/common/items/audio.py

@@ -0,0 +1,6 @@
+from ezmm.common.items import Item
+
+
+class Audio(Item):
+    kind = "audio"
+    # TODO: Implement

ezmm/common/items/image.py

@@ -0,0 +1,127 @@
+import logging
+from io import BytesIO
+from pathlib import Path
+from typing import Optional
+
+import aiohttp
+import pillow_avif  # Keep this import for AVIF support
+from PIL.Image import Image as PillowImage, open as pillow_open, new as pillow_new, Resampling
+
+from ezmm.common.items.item import Item
+from ezmm.request import request_static
+from ezmm.util import to_base64
+
+logger = logging.getLogger("ezMM")
+logger.debug(f"`pillow_avif` v{pillow_avif.__version__} loaded for AVIF image support.")
+
+
+class Image(Item):
+    kind = "image"
+    _image: Optional[PillowImage] = None
+
+    def __init__(self, file_path: str | Path | None = None,
+                 pillow_image: PillowImage | None = None,
+                 binary_data: bytes | None = None,
+                 source_url: str | None = None,
+                 reference: str | None = None):
+        assert file_path or pillow_image or binary_data or reference
+
+        if hasattr(self, "id"):
+            # The image is already initialized (existing instance returned via __new__())
+            return
+
+        if binary_data is not None:
+            pillow_image = pillow_open(BytesIO(binary_data))
+
+        if pillow_image is not None:
+            pillow_image = _ensure_rgb_mode(pillow_image)
+            # Save the image in a temporary folder
+            file_path = self._temp_file_path(suffix=".jpg")
+            file_path.parent.mkdir(parents=True, exist_ok=True)
+            pillow_image.save(file_path)
+            self._image = pillow_image
+
+        # free memory and file handle
+        self.close()
+
+        super().__init__(file_path,
+                         source_url=source_url,
+                         reference=reference)
+
+    @property
+    def image(self) -> PillowImage:
+        """Lazy-loads the PIL image of this Image item."""
+        if not self._image:
+            with pillow_open(self.file_path) as img:
+                self._image = _ensure_rgb_mode(img).copy()
+        return self._image
+
+    def get_base64_encoded(self) -> str:
+        return to_base64(self.image)
+
+    @property
+    def width(self) -> int:
+        return self.image.width
+
+    @property
+    def height(self) -> int:
+        return self.image.height
+
+    def _same(self, other):
+        return (
+                self.image.mode == other.image.mode and
+                self.image.size == other.image.size and
+                self.image.tobytes() == other.image.tobytes()
+        )
+
+    def as_html(self, path_relative_to: str | Path = None) -> str:
+        if path_relative_to is None:
+            path_relative_to = Path.cwd()
+        path = self.file_path.relative_to(path_relative_to).as_posix()
+        img = f'<img src="/{path}" alt="{self.reference}">'
+        if self.source_url:
+            return f'<a href="{self.source_url}">{img}</a>'
+        else:
+            return img
+
+    def close(self):
+        if self._image:
+            self._image.close()
+            self._image = None
+
+
+def _ensure_rgb_mode(pillow_image: PillowImage) -> PillowImage:
+    """Turns any kind of image (incl. PNGs) into RGB mode to make it JPEG-saveable."""
+    if pillow_image.mode in ["RGBA", "P"]:
+        pillow_image = pillow_image.convert('RGBA')
+        converted = pillow_new("RGB", pillow_image.size, (255, 255, 255))
+        converted.paste(pillow_image, mask=pillow_image.split()[3])  # 3 is the alpha channel
+        return converted
+    if pillow_image.mode != "RGB":
+        return pillow_image.convert('RGB')
+    else:
+        return pillow_image
+
+
+async def download_image(
+        image_url: str,
+        session: aiohttp.ClientSession,
+        ignore_small_images: bool = True,
+        max_size: tuple[int, int] = (2048, 2048)
+) -> Optional[Image]:
+    """Download an image from a URL and return it as an Image object."""
+    # TODO: Handle very large images like: https://eoimages.gsfc.nasa.gov/images/imagerecords/144000/144225/campfire_oli_2018312_lrg.jpg
+    content = await request_static(image_url, session, get_text=False)
+    # TODO: Request page dynamically (better move all request-related functions to ScrapeMM)
+    if content:
+        pillow_img = pillow_open(BytesIO(content))
+
+        if pillow_img:
+            if pillow_img.width > max_size[0] or pillow_img.height > max_size[1]:
+                pillow_img.thumbnail(max_size, Resampling.LANCZOS)  # Preserves aspect ratio
+
+            if not ignore_small_images or (pillow_img.width > 256 and pillow_img.height > 256):
+                # TODO: Check for duplicates, i.e., reuse an existing image if it already exists in the registry
+                image = Image(pillow_image=pillow_img, source_url=image_url)
+                image.relocate(move_not_copy=True)  # Ensure the image is in the temp dir + follows simple naming
+                return image

ezmm/common/items/item.py

@@ -0,0 +1,171 @@
+import asyncio
+import re
+from abc import ABC
+from datetime import datetime
+from pathlib import Path
+from shutil import copyfile, move
+from typing import Sequence, Optional
+
+import aiohttp
+
+from ezmm.config import temp_dir
+from ezmm.request import is_maybe_image_url, is_maybe_video_url
+from ezmm.util import is_item_ref, normalize_path
+
+REF = "<{kind}:{id}>"  # General reference template, defining the reference syntax
+
+
+class Item(ABC):
+    """An element of MultimodalSequences. The data of each item is saved in an individual file."""
+    kind: str  # Specifies the type of the item (image, video, ...)
+    id: int  # Unique identifier of this item within its kind
+    file_path: Path  # The (absolute) path to the file where the data of this item is stored
+    source_url: str  # The (web or file) URL pointing at the Item data's origin
+
+    def __new__(cls, file_path: Path | str = None, source_url: str = None, reference: str = None, **kwargs):
+        """Checks if there already exists an instance of the item with the given reference.
+        If yes, returns the existing reference. Otherwise, instantiates a new one."""
+
+        if file_path or reference:
+            # Look up an existing instance instead of creating a new one
+            from ezmm.common.registry import item_registry
+            item = item_registry.get_cached(reference=reference, kind=cls.kind, file_path=file_path)
+            if item:
+                item.source_url = source_url or item.source_url
+                return item
+            elif reference:
+                raise ValueError(f"No item with reference '{reference}'.")
+
+        return super().__new__(cls)
+
+    def __init__(self, file_path: Path | str, source_url: str = None, reference: str = None):
+        if hasattr(self, "id"):
+            # The item is already initialized (existing instance returned via __new__())
+            return
+        self.file_path = normalize_path(file_path)
+        self.source_url = source_url or self.file_path.absolute().as_uri()
+        from ezmm.common.registry import item_registry
+        item_registry.add_and_assign_id(self)  # Ensure the item is registered and get an ID assigned
+
+    @property
+    def reference(self) -> str:
+        return REF.format(kind=self.kind, id=self.id)
+
+    def _same(self, other) -> bool:
+        """Compares the content data with the other item for equality."""
+        raise NotImplementedError
+
+    @staticmethod
+    def from_reference(reference: str) -> Optional["Item"]:
+        from ezmm.common.registry import item_registry
+        return item_registry.get(reference)
+
+    def close(self):
+        """Closes any resources held by this item."""
+        pass
+
+    def as_html(self, path_relative_to: str | Path = None) -> str:
+        """Returns the item as HTML code."""
+        return f"<p>Item {self.reference} does not support HTML yet.</p>"
+
+    def relocate(self, move_not_copy=False):
+        """Copies the item's file to the temp_dir if not
+        located there already. Moves it instead if move=True."""
+        new_path = self._default_file_path(suffix=self.file_path.suffix)
+        if self.file_path != new_path:
+            # Ensure the target directory exists
+            new_path.parent.mkdir(parents=True, exist_ok=True)
+
+            # Move/copy file to target directory
+            self.close()
+            move(self.file_path, new_path) if move_not_copy else copyfile(self.file_path, new_path)
+
+            # Update the file path to the new location
+            self.file_path = new_path
+            from ezmm.common.registry import item_registry
+            item_registry.update_file_path(self)
+
+    def get_size(self) -> int:
+        """Returns the size of the item in bytes."""
+        return self.file_path.stat().st_size
+
+    def _temp_file_path(self, suffix: str = "") -> Path:
+        """Used when the item's ID is not set yet."""
+        filename = datetime.now().strftime("%Y-%m-%d_%H-%M-%S-%f") + suffix
+        return normalize_path(temp_dir / "items" / filename)
+
+    def _default_file_path(self, suffix: str = "") -> Path:
+        """Only usable after item initialization."""
+        default_filename = str(self.id) + suffix
+        return normalize_path(temp_dir / self.kind / default_filename)
+
+    def __eq__(self, other):
+        return (self is other or
+                isinstance(other, Item) and (
+                        self.kind == other.kind and self.id == other.id or  # Should never trigger
+                        self._same(other)
+                ))
+
+    def __hash__(self):
+        # TODO: Make hash content-dependent => identify known items by hash
+        return hash((self.kind, self.id))
+
+
+def resolve_references_from_sequence(seq: Sequence[str | Item]) -> list[str | Item]:
+    """Identifies all item references within the sequence and replaces them with
+    an instance of the referenced item. Returns the (interleaved) list of
+    strings and items."""
+    processed = []
+    for item in seq:
+        if isinstance(item, str):
+            if item.strip(" "):  # Drop excess whitespaces
+                resolved = resolve_references_from_string(item)
+                processed.extend(resolved)
+        elif item:  # Drop Nones
+            processed.append(item)
+    return processed
+
+
+def resolve_references_from_string(string: str) -> list[str | Item]:
+    """Identifies all item references within the string and replaces them with
+    an instance of the referenced item. Returns the (interleaved) list of
+    strings and items."""
+    from ezmm.common.registry import item_registry
+    from ezmm.common.items import ITEM_REF_REGEX
+    ref_regex = rf"\s?{ITEM_REF_REGEX}\s?"  # Extend to optional whitespaces before and after the ref
+    split = re.split(ref_regex, string)
+    # Replace each reference with its actual item object
+    for i in range(len(split)):
+        substr = split[i]
+        if is_item_ref(substr):
+            item = item_registry.get(substr)
+            if item is not None:
+                split[i] = item
+    return split
+
+
+async def download_item(url: str,
+                        session: Optional[aiohttp.ClientSession] = None,
+                        ignore_small_images: bool = True) -> Optional[Item]:
+    """Downloads the item from the given URL and returns an instance of the
+    corresponding item class. Reuses a session if provided."""
+
+    own_session = session is None
+    if own_session:
+        session = aiohttp.ClientSession()
+
+    try:
+        if await is_maybe_image_url(url, session):
+            from ezmm.common.items.image import download_image
+            return await download_image(url, ignore_small_images=ignore_small_images, session=session)
+        if await is_maybe_video_url(url, session):
+            from ezmm.common.items.video import download_video
+            return await download_video(url, session)
+        # TODO: Handle audios
+    finally:
+        if own_session:
+            await session.close()
+
+if __name__ == "__main__":
+    result = asyncio.run(download_item("https://media.cnn.com/api/v1/images/stellar/prod/02-overview-of-kursk-training-area-15april2025-wv2.jpg?q=w_1110,c_fill"))
+    print(result)

ezmm/common/items/video.py

@@ -0,0 +1,253 @@
+import base64
+import logging
+from pathlib import Path
+from typing import Optional
+from urllib.parse import urljoin
+
+import aiohttp
+import cv2
+import m3u8
+import numpy as np
+
+from ezmm.common.items.item import Item
+from ezmm.request import fetch_headers, HEADERS
+from ezmm.util import ts_to_mp4, to_base64
+
+logger = logging.getLogger("ezMM")
+
+
+class Video(Item):
+    kind = "video"
+    _video: Optional[cv2.VideoCapture] = None
+
+    def __init__(self, file_path: str | Path = None,
+                 binary_data: bytes = None,
+                 source_url: str = None,
+                 reference: str = None):
+        assert file_path or binary_data or reference
+
+        if hasattr(self, "id"):
+            return
+
+        if binary_data:
+            # Save binary data to temporary file
+            file_path = self._temp_file_path(suffix=".mp4")
+            file_path.parent.mkdir(parents=True, exist_ok=True)
+            with open(file_path, 'wb') as f:
+                f.write(binary_data)
+
+        super().__init__(file_path,
+                         source_url=source_url,
+                         reference=reference)
+
+    @property
+    def video(self) -> cv2.VideoCapture:
+        """Lazy-loads the video capture of this Video item."""
+        if not self._video:
+            self._open_video()
+        return self._video
+
+    def _open_video(self):
+        """Opens the video file for reading."""
+        self._video = cv2.VideoCapture(str(self.file_path))
+
+    @property
+    def width(self) -> int:
+        return int(self.video.get(cv2.CAP_PROP_FRAME_WIDTH))
+
+    @property
+    def height(self) -> int:
+        return int(self.video.get(cv2.CAP_PROP_FRAME_HEIGHT))
+
+    @property
+    def frame_count(self) -> int:
+        return int(self.video.get(cv2.CAP_PROP_FRAME_COUNT))
+
+    @property
+    def fps(self) -> float:
+        return self.video.get(cv2.CAP_PROP_FPS)
+
+    @property
+    def duration(self) -> float:
+        """Returns the duration of the video in seconds."""
+        return self.frame_count / self.fps
+
+    @property
+    def bytes(self) -> bytes:
+        """Returns the video as bytes."""
+        return self.file_path.read_bytes()
+
+    def _get_frames(self) -> list[bytes]:
+        """Returns the list of all frames from the video."""
+        frames = []
+        if not self.video.isOpened():
+            self._open_video()
+        while self.video.isOpened():
+            success, frame = self.video.read()
+            if not success:
+                logger.warning(f"Failed to read frame from video {self.file_path}")
+                break
+            _, frame = cv2.imencode(".jpeg", frame)
+            frames.append(frame)
+        self.video.release()
+        return frames
+
+    def sample_frames(self, n_frames: int = 5) -> list[bytes]:
+        """Returns the specified number of frames sampled evenly spaced from the video.
+        Always includes the first frame. Includes the last frame if n_frames > 1."""
+        assert n_frames > 0, "Number of frames must be greater than 0."
+        frames = self._get_frames()
+        n_frames = min(n_frames, len(frames))
+        frame_ids = np.linspace(0, len(frames) - 1, n_frames, dtype=int)
+        sampled = [frames[i] for i in frame_ids]
+        return sampled
+
+    def get_base64_encoded(self, n_frames: int = 5) -> list[str]:
+        """Returns base64-encoded frames, evenly sampled from the video."""
+        frames = self.sample_frames(n_frames)
+        frames_encoded = [base64.b64encode(frame).decode("utf-8") for frame in frames]
+        return frames_encoded
+
+    def _same(self, other):
+        return (
+                self.width == other.width and
+                self.height == other.height and
+                self.frame_count == other.frame_count and
+                self.file_path.read_bytes() == other.file_path.read_bytes()
+        )
+
+    def as_html(self, path_relative_to: str | Path = None) -> str:
+        if path_relative_to is None:
+            path_relative_to = Path.cwd()
+        path = self.file_path.relative_to(path_relative_to).as_posix()
+        return f'<video controls src="/{path}"></video>'
+
+    def close(self):
+        if self._video:
+            self._video.release()
+            self._video = None
+
+
+async def download_video(
+        video_url: str,
+        session: aiohttp.ClientSession
+) -> Optional[Video]:
+    """Downloads the linked video (stream) and returns it as a Video object."""
+
+    try:
+        headers = await fetch_headers(video_url, session, timeout=3)
+        content_type = headers.get('Content-Type') or headers.get('content-type')
+        if content_type.startswith("video/"):
+            return await download_video_file(video_url, session)
+        elif content_type == "application/vnd.apple.mpegurl":
+            return await download_hls_video(video_url, session)
+        else:
+            logger.warning(f"Cannot download video! Unable to handle content type: {content_type}.")
+
+    except Exception as e:
+        logger.debug(f"Error downloading video from {video_url}"
+                     f"\n{type(e).__name__}: {e}")
+
+
+async def download_video_file(
+        video_url: str,
+        session: aiohttp.ClientSession
+) -> Optional[Video]:
+    """Download a single video file from a URL and return it as a Video object."""
+    try:
+        async with session.get(video_url, headers=HEADERS) as response:
+            if response.status == 200:
+                content = await response.read()
+                video = Video(binary_data=content, source_url=video_url)
+                video.relocate(move_not_copy=True)
+                return video
+    except Exception as e:
+        logger.debug(f"Error downloading video file from {video_url}"
+                     f"\n{type(e).__name__}: {e}")
+
+
+async def download_hls_video(
+        playlist_url: str,
+        session: aiohttp.ClientSession
+) -> Optional[Video]:
+    """Download an HTTP Live Streaming (HLS) video from a playlist URL and return it as a Video object."""
+    try:
+        # Download the m3u8 playlist file
+        async with session.get(playlist_url, headers=HEADERS) as response:
+            if response.status != 200:
+                logger.debug(f"Failed to download playlist: {response.status}")
+                return None
+            playlist_content = await response.text()
+
+        playlist = m3u8.loads(playlist_content)
+        base_url = playlist_url.rsplit('/', 1)[0] + '/'
+
+        # Check if this is a master playlist (contains variant playlists)
+        if playlist.is_variant:
+            # Choose the highest quality variant
+            best_playlist = playlist.playlists[-1]  # Usually the last one is of highest quality
+
+            # Manually construct the absolute URL for the variant playlist
+            variant_url = urljoin(base_url, best_playlist.uri)
+
+            # Download the variant playlist
+            async with session.get(variant_url, headers=HEADERS) as var_response:
+                if var_response.status != 200:
+                    logger.error(f"Failed to download variant playlist: {var_response.status}")
+                    return None
+                variant_content = await var_response.text()
+
+            # Parse the variant playlist
+            variant_playlist = m3u8.loads(variant_content)
+            playlist = variant_playlist  # Use this for segment downloads
+
+            # Update base_url for segment downloads
+            base_url = variant_url.rsplit('/', 1)[0] + '/'
+
+        # Download all segments
+        video_segments = []
+
+        for i, segment in enumerate(playlist.segments):
+            # Construct full URL for the segment
+            if segment.uri.startswith('http'):
+                segment_url = segment.uri
+            else:
+                segment_url = urljoin(base_url, segment.uri)
+
+            # Download the segment with SSL disabled
+            try:
+                async with session.get(segment_url, headers=HEADERS) as seg_response:
+                    if seg_response.status == 200:
+                        segment_data = await seg_response.read()
+                        video_segments.append(segment_data)
+            except Exception as e:
+                logger.debug(f"Failed to download segment {i} from {segment_url}: {e}")
+
+        # Combine all segments
+        if video_segments:
+            ts_bytes = b''.join(video_segments)
+            mp4_bytes = ts_to_mp4(ts_bytes)
+
+            # Create Video object with MP4 content
+            video = Video(binary_data=mp4_bytes, source_url=playlist_url)
+            video.relocate(move_not_copy=True)
+            return video
+
+    except Exception as e:
+        logger.debug(f"Error downloading HLS video from {playlist_url}"
+                     f"\n{type(e).__name__}: {e}")
+
+    return None
+
+
+async def download_vid(url):
+    async with aiohttp.ClientSession() as session:
+        return await download_video(url, session)
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    video = asyncio.run(
+        download_vid("https://upload.wikimedia.org/wikipedia/commons/transcoded/a/a7/How_to_make_video.webm/How_to_make_video.webm.1080p.vp9.webm"))
+    print(video)