memories-classify 1.2.0__py3-none-any.whl

classify/__init__.py

@@ -0,0 +1 @@
+"""Init python."""

classify/__main__.py

@@ -0,0 +1,6 @@
+"""Main python."""
+
+from .main import main
+
+if __name__ == "__main__":
+    main()

classify/classify.py

@@ -0,0 +1,70 @@
+"""Main function to classify pictures and videos."""
+
+import logging
+import os
+from collections.abc import Callable
+
+from tqdm import tqdm
+from tqdm.contrib.logging import logging_redirect_tqdm
+
+from .processors.files import FileProcessor
+from .processors.image import ImageProcessor
+from .processors.video import VideoProcessor
+from .settings import ClassifySettings
+
+_LOGGER = logging.getLogger("classify")
+
+
+class Classify:
+    """Classify global class."""
+
+    def __init__(self, settings: ClassifySettings):
+        """Initialize the class."""
+        self.settings = settings
+        self.fp = FileProcessor(settings=settings)
+        self.ip = ImageProcessor(settings=settings, file_processor=self.fp)
+        self.vp = VideoProcessor(settings=settings, file_processor=self.fp)
+
+    def run(self) -> None:
+        """Classify pictures and videos."""
+
+        if not self.fp.pictures and not self.fp.videos:
+            _LOGGER.info("No pictures or videos found")
+            return
+
+        _LOGGER.info(
+            "##### Clean Android Google Photo trashed and pending pictures uploaded #####"
+        )
+        self.fp.delete_android_trash_files()
+
+        self.process_files("picture", self.fp.pictures, self.ip.process)
+        self.process_files("video", self.fp.videos, self.vp.process)
+
+        _LOGGER.info("")
+
+    def process_files(
+        self, kind: str, paths: list[str], process: Callable[[str], None]
+    ) -> None:
+        """Process files of a kind, logging errors without stopping."""
+        if not paths:
+            return
+
+        _LOGGER.info("")
+        _LOGGER.info("##### %ss #####", kind.capitalize())
+        # Iterate on a copy, processing may update the file lists
+        with (
+            logging_redirect_tqdm(loggers=[_LOGGER]),
+            tqdm(list(paths), desc=f"{kind.capitalize()}s", unit=kind) as progress,
+        ):
+            for path in progress:
+                _LOGGER.debug(
+                    "Process %s %s (%s GB)",
+                    kind,
+                    path,
+                    round(os.path.getsize(path) / 1e9, 3),
+                )
+
+                try:
+                    process(path)
+                except Exception as exc:  # noqa: BLE001
+                    _LOGGER.error("Error processing %s %s: %s", kind, path, exc)

classify/const.py

@@ -0,0 +1,23 @@
+"""Constants."""
+
+PICTURE_EXTENSIONS = [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff"]
+VIDEO_EXTENSIONS = [".mp4", ".mov", ".avi", ".mkv", ".webm"]
+
+VIDEO_CODEC = "hevc"
+
+FILENAME_DATE_FORMATS = {
+    r"(\d{8}_\d{9})": "%Y%m%d_%H%M%S%f",
+    r"(\d{8}-\d{9})": "%Y%m%d-%H%M%S%f",
+    r"(\d{14})": "%Y%m%d%H%M%S",
+}
+
+DEFAULT_NAME_FORMAT = "%Y-%m-%d-%Hh%Mm%S"
+
+DEFAULT_COMMENT_MESSAGE = "Processed by memories-classify"
+
+DEFAULT_VIDEO_BITRATE_MBPS_LIMIT = 30
+
+DEFAULT_FFMPEG_PATH = "ffmpeg"
+DEFAULT_FFMPEG_INPUT_EXTRA_ARGS = ""
+DEFAULT_FFMPEG_OUTPUT_EXTRA_ARGS = ""
+DEFAULT_FFPROBE_PATH = "ffprobe"

classify/exception.py

@@ -0,0 +1,9 @@
+"""Exceptions for the package."""
+
+
+class ClassifyException(Exception):
+    """Classify exception."""
+
+
+class ClassifyEncodingException(Exception):
+    """Exception raised when encoding fails."""

classify/logger.py

@@ -0,0 +1,30 @@
+"""Custom logger module"""
+
+import logging
+from typing import ClassVar
+
+
+class CustomFormatter(logging.Formatter):
+    """Custom log formatter"""
+
+    grey = "\x1b[90;20m"
+    white = "\x1b[97;20m"
+    yellow = "\x1b[33;20m"
+    red = "\x1b[31;20m"
+    bold_red = "\x1b[31;1m"
+    reset = "\x1b[0m"
+    log_format = "[%(asctime)s][%(levelname)s] %(message)s"
+
+    FORMATS: ClassVar[dict[int, str]] = {
+        logging.DEBUG: grey + log_format + reset,
+        logging.INFO: white + log_format + reset,
+        logging.WARNING: yellow + log_format + reset,
+        logging.ERROR: red + log_format + reset,
+        logging.CRITICAL: bold_red + log_format + reset,
+    }
+
+    def format(self, record: logging.LogRecord) -> str:
+        """Format log record"""
+        log_fmt = self.FORMATS.get(record.levelno)
+        formatter = logging.Formatter(log_fmt)
+        return formatter.format(record)

classify/main.py

@@ -0,0 +1,70 @@
+"""Classify."""
+
+import logging
+import os
+import sys
+from subprocess import DEVNULL, CalledProcessError, check_call
+
+from .classify import Classify
+from .exception import ClassifyException
+from .logger import CustomFormatter
+from .settings import ClassifySettings, parse_args
+
+_LOGGER = logging.getLogger("classify")
+
+
+def is_tool_available(path: str) -> bool:
+    """Check that an ffmpeg tool can be executed."""
+    try:
+        check_call([path, "-version"], stdout=DEVNULL, stderr=DEVNULL)
+    except OSError, CalledProcessError:
+        return False
+    return True
+
+
+def main(arg_list: list[str] | None = None):
+    """Call from cli."""
+
+    args = parse_args(arg_list)
+
+    if not _LOGGER.handlers:
+        handler = logging.StreamHandler()
+        handler.setFormatter(CustomFormatter())
+        _LOGGER.addHandler(handler)
+    _LOGGER.setLevel(logging.DEBUG if args.verbose else logging.INFO)
+    _LOGGER.info("Classify pictures and videos tool")
+
+    try:
+        settings = ClassifySettings(args=args)
+    except ClassifyException as exc:
+        _LOGGER.error("%s", exc)
+        sys.exit(1)
+
+    if settings.dry_run:
+        _LOGGER.warning("Dry run mode activated")
+
+    if not os.path.isdir(settings.directory):
+        _LOGGER.error("Directory %s does not exist", settings.directory)
+        sys.exit(1)
+
+    _LOGGER.info("Process directory: %s", settings.directory)
+
+    for tool_path, option in (
+        (settings.ffmpeg_path, "--ffmpeg-path"),
+        (settings.ffprobe_path, "--ffprobe-path"),
+    ):
+        if not is_tool_available(tool_path):
+            _LOGGER.error(
+                "%s not found (install ffmpeg or set path with %s)", tool_path, option
+            )
+            sys.exit(1)
+
+    try:
+        Classify(settings).run()
+        _LOGGER.info("End")
+    except ClassifyException as exc:
+        _LOGGER.error("End with error: %s", exc)
+        sys.exit(1)
+    except KeyboardInterrupt:
+        _LOGGER.warning("Interrupted")
+        sys.exit(130)

classify/processors/files.py

@@ -0,0 +1,200 @@
+"""Files management for Classify."""
+
+import filecmp
+import logging
+import os
+import re
+import shutil
+import string
+from collections.abc import Callable, Iterator
+from datetime import UTC, datetime
+
+from classify.settings import ClassifySettings
+
+from ..const import FILENAME_DATE_FORMATS, PICTURE_EXTENSIONS, VIDEO_EXTENSIONS
+from ..exception import ClassifyException
+
+_LOGGER = logging.getLogger("classify")
+
+
+class FileProcessor:
+    """Files processor for Classify."""
+
+    pictures: list[str]
+    videos: list[str]
+
+    def __init__(self, settings: ClassifySettings) -> None:
+        """Init."""
+        self.settings = settings
+        self.pictures = []
+        self.videos = []
+
+        if not os.path.exists(self.settings.output):
+            _LOGGER.info("Create missing output directory %s", self.settings.output)
+            if not self.settings.dry_run:
+                os.makedirs(self.settings.output)
+
+        self.reload()
+
+        _LOGGER.info(
+            "Found %d pictures and %d videos",
+            len(self.pictures),
+            len(self.videos),
+        )
+
+    def reload(self) -> None:
+        """Reload files from a directory."""
+        output_path = os.path.abspath(self.settings.output)
+        for root, dirs, files in os.walk(self.settings.directory):
+            # Do not process again files classified in an output inside the input
+            dirs[:] = [
+                directory
+                for directory in dirs
+                if os.path.abspath(os.path.join(root, directory)) != output_path
+            ]
+            for file in files:
+                file_path = os.path.join(root, file)
+                file_relpath = os.path.relpath(file_path, self.settings.directory)
+                if any(
+                    re.match(pattern, file_relpath) for pattern in self.settings.exclude
+                ):
+                    _LOGGER.info("Exclude %s because of exclude pattern", file_relpath)
+                    continue
+                file_extension = os.path.splitext(file)[1].lower()
+                if file_extension in PICTURE_EXTENSIONS:
+                    self.pictures.append(file_path)
+                elif file_extension in VIDEO_EXTENSIONS:
+                    self.videos.append(file_path)
+
+    def remove_file(self, file_path: str) -> None:
+        """Delete a file from disk and drop it from the lists to process."""
+        if not self.settings.dry_run and os.path.exists(file_path):
+            _LOGGER.debug("Remove %s", file_path)
+            os.remove(file_path)
+        if file_path in self.pictures:
+            self.pictures.remove(file_path)
+        elif file_path in self.videos:
+            self.videos.remove(file_path)
+        else:
+            raise ClassifyException(f"File {file_path} not found in the list")
+
+    def get_output_path(self, file: str) -> str:
+        """Get the output path for a file."""
+        relpath = os.path.dirname(os.path.relpath(file, self.settings.directory))
+        return os.path.join(self.settings.output, relpath)
+
+    def is_date_named(self, file_path: str) -> bool:
+        """Check if a file is named from a date, with an optional duplicate suffix."""
+        base_name = os.path.splitext(os.path.basename(file_path))[0]
+        for name in (base_name, base_name[:-1]):
+            try:
+                datetime.strptime(name, self.settings.name_format)
+            except ValueError:
+                continue
+            return name == base_name or base_name[-1] in string.ascii_lowercase
+        return False
+
+    def iter_filepaths_from_date(
+        self, dest_dir: str, date_taken: datetime, extension: str
+    ) -> Iterator[str]:
+        """Yield the file paths named from a date, then with a letter suffix."""
+        base_name = date_taken.strftime(self.settings.name_format)
+        for suffix in ["", *string.ascii_lowercase]:
+            yield os.path.join(dest_dir, f"{base_name}{suffix}{extension}")
+
+    def get_available_filepath_from_date(
+        self,
+        dest_dir: str,
+        date_taken: datetime,
+        extension: str,
+        source_file: str | None = None,
+    ) -> str:
+        """Get an available file path named from a date.
+
+        A letter is appended on name conflicts. The source file path is returned
+        when it already has one of the candidate names.
+        """
+        for file_path in self.iter_filepaths_from_date(dest_dir, date_taken, extension):
+            if source_file is not None and os.path.abspath(
+                file_path
+            ) == os.path.abspath(source_file):
+                return file_path
+            if not os.path.exists(file_path):
+                return file_path
+        raise ClassifyException(
+            f"No available file name for {date_taken} in {dest_dir}"
+        )
+
+    def find_existing_copy(
+        self,
+        dest_dir: str,
+        date_taken: datetime,
+        extension: str,
+        is_copy: Callable[[str], bool],
+    ) -> str | None:
+        """Find a file named from a date that is already a copy of the source."""
+        for file_path in self.iter_filepaths_from_date(dest_dir, date_taken, extension):
+            if os.path.exists(file_path) and is_copy(file_path):
+                return file_path
+        return None
+
+    def place_in_output(self, path: str, date_taken: datetime, extension: str) -> None:
+        """Copy or move a file to its date name in the output directory."""
+        dest_dir_path = self.get_output_path(path)
+        if self.settings.keep_original and (
+            existing_copy := self.find_existing_copy(
+                dest_dir=dest_dir_path,
+                date_taken=date_taken,
+                extension=extension,
+                is_copy=lambda file_path: (
+                    os.path.abspath(file_path) != os.path.abspath(path)
+                    and filecmp.cmp(file_path, path, shallow=False)
+                ),
+            )
+        ):
+            _LOGGER.debug("%s already copied to %s", path, existing_copy)
+            return
+
+        new_path = self.get_available_filepath_from_date(
+            dest_dir=dest_dir_path,
+            date_taken=date_taken,
+            extension=extension,
+            source_file=path,
+        )
+        if os.path.abspath(new_path) == os.path.abspath(path):
+            _LOGGER.debug("Already named correctly")
+            return
+
+        if self.settings.keep_original:
+            _LOGGER.info("Copy %s to %s", path, new_path)
+        else:
+            _LOGGER.info("Rename %s to %s", path, new_path)
+        if self.settings.dry_run:
+            return
+        os.makedirs(dest_dir_path, exist_ok=True)
+        if self.settings.keep_original:
+            shutil.copy2(path, new_path)
+        else:
+            # Unlike os.rename, works when the output is on another file system
+            shutil.move(path, new_path)
+
+    def get_date_from_file_name(self, file_path: str) -> datetime | None:
+        """Get the date from a file name, like PXL_20241010_174118780.mp4."""
+        file_name = os.path.basename(file_path)
+        for regex, date_format in FILENAME_DATE_FORMATS.items():
+            if date_match := re.search(regex, file_name):
+                _LOGGER.debug("Date taken from filename: %s", date_match.group(0))
+                # Phone file names hold the UTC time
+                date_src = datetime.strptime(date_match.group(0), date_format).replace(
+                    tzinfo=UTC
+                )
+                return date_src.astimezone(self.settings.user_timezone)
+        return None
+
+    def delete_android_trash_files(self) -> None:
+        """Delete Android trash files."""
+        for file_path in self.pictures + self.videos:
+            file_name = os.path.basename(file_path)
+            if file_name.startswith((".trashed", ".pending")):
+                _LOGGER.info("Delete %s", file_name)
+                self.remove_file(file_path)

classify/processors/image.py

@@ -0,0 +1,79 @@
+"""Image processor."""
+
+import logging
+import os
+from datetime import datetime
+
+from PIL import Image
+from PIL.ExifTags import Base as ExifBase
+
+from ..settings import ClassifySettings
+from .files import FileProcessor
+
+_LOGGER = logging.getLogger("classify")
+
+
+class ImageProcessor:
+    """Image processor class"""
+
+    def __init__(
+        self, settings: ClassifySettings, file_processor: FileProcessor
+    ) -> None:
+        """Initialize the class"""
+        self.settings = settings
+        self.fp = file_processor
+
+    def get_date_taken(self, path: str) -> datetime | None:
+        """Get the date taken from the exif or the file name of a picture"""
+        if date_taken := self.get_exif_date_taken(path):
+            return date_taken
+        return self.fp.get_date_from_file_name(path)
+
+    def get_exif_date_taken(self, path: str) -> datetime | None:
+        """Get the date taken from the exif of a picture"""
+        with Image.open(path) as img:
+            exif = img.getexif()
+        if not exif:
+            return None
+
+        date_taken = None
+
+        # First check EXIF IFD for DateTimeOriginal
+        try:
+            exif_ifd = exif.get_ifd(0x8769)
+            if int(ExifBase.DateTimeOriginal) in exif_ifd:
+                date_taken = exif_ifd[int(ExifBase.DateTimeOriginal)]
+        except Exception:
+            _LOGGER.debug("Cannot read EXIF IFD of %s", path, exc_info=True)
+
+        # Fallback to main EXIF for DateTime
+        if not date_taken and int(ExifBase.DateTime) in exif:
+            date_taken = exif[int(ExifBase.DateTime)]
+
+        if not date_taken:
+            return None
+
+        try:
+            return datetime.strptime(date_taken, "%Y:%m:%d %H:%M:%S")
+        except ValueError:
+            _LOGGER.warning("Invalid EXIF date %s in %s", date_taken, path)
+            return None
+
+    def rename_from_date_taken(self, path: str) -> None:
+        """Rename a picture from date taken"""
+        picture_date_taken = self.get_date_taken(path)
+        if not picture_date_taken:
+            _LOGGER.warning("Cannot get date from picture %s", path)
+            return
+
+        _LOGGER.debug(
+            "Picture %s taken on %s", os.path.basename(path), picture_date_taken
+        )
+        extension = os.path.splitext(path)[1].lower()
+        if extension == ".jpeg":
+            extension = ".jpg"
+        self.fp.place_in_output(path, picture_date_taken, extension)
+
+    def process(self, path: str) -> None:
+        """Process a picture"""
+        self.rename_from_date_taken(path)