library 3.2.2__tar.gz → 3.2.4__tar.gz

{library-3.2.2 → library-3.2.4}/.github/README.md

@@ -99,7 +99,7 @@ To stop playing press Ctrl+C in either the terminal or mpv
 <details><summary>List all subcommands</summary>
 
     $ library
-    library (v3.2.002; 103 subcommands)
+    library (v3.2.004; 102 subcommands)
 
     Create database subcommands:
     ╭─────────────────┬──────────────────────────────────────────╮
@@ -197,8 +197,6 @@ To stop playing press Ctrl+C in either the terminal or mpv
     │ filesystem     │ Find files by mimetype and size                     │
     ├────────────────┼─────────────────────────────────────────────────────┤
     │ similar-files  │ Find similar files based on filename and size       │
-    ├────────────────┼─────────────────────────────────────────────────────┤
-    │ llm-map        │ Run LLMs across multiple files                      │
     ╰────────────────┴─────────────────────────────────────────────────────╯
 
     Tabular data subcommands:
@@ -1614,11 +1612,11 @@ BTW, for some cols like time_deleted you'll need to specify a where clause so th
 <details><summary>Find similar folders based on folder name, size, and count</summary>
 
     $ library similar-folders -h
-    usage: library similar-folders PATH ...
+    usage: library similar-folders (--filter-names | --filter-counts | --filter-sizes | --filter-durations) PATH ...
 
     Find similar folders based on foldernames, similar size, and similar number of files
 
-        library similar-folders ~/d/
+        library similar-folders --filter-names --filter-counts --filter-sizes ~/d/
 
         group /home/xk/d/dump/datasets/*vector          total_size    median_size      files
         ----------------------------------------------  ------------  -------------  -------
@@ -1730,11 +1728,11 @@ BTW, for some cols like time_deleted you'll need to specify a where clause so th
 <details><summary>Find similar files based on filename and size</summary>
 
     $ library similar-files -h
-    usage: library similar-files PATH ...
+    usage: library similar-files (--filter-names | --filter-sizes | --filter-durations) PATH ...
 
     Find similar files using filenames and size
 
-        library similar-files ~/d/
+        library similar-files --filter-names --filter-sizes ~/d/
 
     Find similar files based on ONLY foldernames, using the full path
 
@@ -1752,34 +1750,6 @@ BTW, for some cols like time_deleted you'll need to specify a where clause so th
         library similar-files --filter-names --filter-durations --estimated-duplicates 3 .
 
 
-</details>
-
-###### llm-map
-
-<details><summary>Run LLMs across multiple files</summary>
-
-    $ library llm-map -h
-    usage: library llm-map LLAMA_FILE [paths ...] [--llama-args LLAMA_ARGS] [--prompt STR] [--text [INT]] [--rename]
-
-    Run a llamafile with a prompt including path names and file contents
-
-    Rename files based on file contents
-
-        library llm-map ./gemma2.llamafile ~/Downloads/booka.pdf --rename --text
-
-        cat llm_map_renames.csv
-        Path,Output
-        /home/xk/Downloads/booka.pdf,/home/xk/Downloads/Mining_Massive_Datasets.pdf
-
-    Using GGUF files
-
-        wget https://github.com/Mozilla-Ocho/llamafile/releases/download/0.8.9/llamafile-0.8.9
-        chmod +x ~/Downloads/llamafile-0.8.9
-        mv ~/Downloads/llamafile-0.8.9 ~/.local/bin/llamafile  # move it somewhere in your $PATH
-
-        library llm-map --model ~/Downloads/llava-v1.5-7b-Q4_K.gguf --image-model ~/Downloads/llava-v1.5-7b-mmproj-Q4_0.gguf --prompt 'what do you see?' ~/Downloads/comp_*.jpg
-
-
 </details>
 
 ### Tabular data subcommands
@@ -2487,9 +2457,10 @@ Inspired somewhat by https://nikkhokkho.sourceforge.io/?page=FileOptimizer
 <details><summary>Download media</summary>
 
     $ library download -h
-    usage: library download DATABASE [--prefix /mnt/d/] --video [--subs] [--auto-subs] [--small] | --audio | --photos [--safe]
+    usage: library download DATABASE [--prefix /mnt/d/] [--video (default) | --audio | --image | --filesystem] [--safe]
 
     Files will be saved to <prefix>/<extractor>/. The default prefix is the current working directory.
+    The default download profile is video. `--safe` supports audio, video, and image profiles only.
 
     By default things will download in a random order
 

{library-3.2.2 → library-3.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: library
-Version: 3.2.2
+Version: 3.2.4
 Summary: xk media library
 Author-Email: Jacob Chapman <7908073+chapmanjacobd@users.noreply.github.com>
 License: BSD 3-Clause License
@@ -207,7 +207,7 @@ To stop playing press Ctrl+C in either the terminal or mpv
 <details><summary>List all subcommands</summary>
 
     $ library
-    library (v3.2.002; 103 subcommands)
+    library (v3.2.004; 102 subcommands)
 
     Create database subcommands:
     ╭─────────────────┬──────────────────────────────────────────╮
@@ -305,8 +305,6 @@ To stop playing press Ctrl+C in either the terminal or mpv
     │ filesystem     │ Find files by mimetype and size                     │
     ├────────────────┼─────────────────────────────────────────────────────┤
     │ similar-files  │ Find similar files based on filename and size       │
-    ├────────────────┼─────────────────────────────────────────────────────┤
-    │ llm-map        │ Run LLMs across multiple files                      │
     ╰────────────────┴─────────────────────────────────────────────────────╯
 
     Tabular data subcommands:
@@ -1722,11 +1720,11 @@ BTW, for some cols like time_deleted you'll need to specify a where clause so th
 <details><summary>Find similar folders based on folder name, size, and count</summary>
 
     $ library similar-folders -h
-    usage: library similar-folders PATH ...
+    usage: library similar-folders (--filter-names | --filter-counts | --filter-sizes | --filter-durations) PATH ...
 
     Find similar folders based on foldernames, similar size, and similar number of files
 
-        library similar-folders ~/d/
+        library similar-folders --filter-names --filter-counts --filter-sizes ~/d/
 
         group /home/xk/d/dump/datasets/*vector          total_size    median_size      files
         ----------------------------------------------  ------------  -------------  -------
@@ -1838,11 +1836,11 @@ BTW, for some cols like time_deleted you'll need to specify a where clause so th
 <details><summary>Find similar files based on filename and size</summary>
 
     $ library similar-files -h
-    usage: library similar-files PATH ...
+    usage: library similar-files (--filter-names | --filter-sizes | --filter-durations) PATH ...
 
     Find similar files using filenames and size
 
-        library similar-files ~/d/
+        library similar-files --filter-names --filter-sizes ~/d/
 
     Find similar files based on ONLY foldernames, using the full path
 
@@ -1860,34 +1858,6 @@ BTW, for some cols like time_deleted you'll need to specify a where clause so th
         library similar-files --filter-names --filter-durations --estimated-duplicates 3 .
 
 
-</details>
-
-###### llm-map
-
-<details><summary>Run LLMs across multiple files</summary>
-
-    $ library llm-map -h
-    usage: library llm-map LLAMA_FILE [paths ...] [--llama-args LLAMA_ARGS] [--prompt STR] [--text [INT]] [--rename]
-
-    Run a llamafile with a prompt including path names and file contents
-
-    Rename files based on file contents
-
-        library llm-map ./gemma2.llamafile ~/Downloads/booka.pdf --rename --text
-
-        cat llm_map_renames.csv
-        Path,Output
-        /home/xk/Downloads/booka.pdf,/home/xk/Downloads/Mining_Massive_Datasets.pdf
-
-    Using GGUF files
-
-        wget https://github.com/Mozilla-Ocho/llamafile/releases/download/0.8.9/llamafile-0.8.9
-        chmod +x ~/Downloads/llamafile-0.8.9
-        mv ~/Downloads/llamafile-0.8.9 ~/.local/bin/llamafile  # move it somewhere in your $PATH
-
-        library llm-map --model ~/Downloads/llava-v1.5-7b-Q4_K.gguf --image-model ~/Downloads/llava-v1.5-7b-mmproj-Q4_0.gguf --prompt 'what do you see?' ~/Downloads/comp_*.jpg
-
-
 </details>
 
 ### Tabular data subcommands
@@ -2595,9 +2565,10 @@ Inspired somewhat by https://nikkhokkho.sourceforge.io/?page=FileOptimizer
 <details><summary>Download media</summary>
 
     $ library download -h
-    usage: library download DATABASE [--prefix /mnt/d/] --video [--subs] [--auto-subs] [--small] | --audio | --photos [--safe]
+    usage: library download DATABASE [--prefix /mnt/d/] [--video (default) | --audio | --image | --filesystem] [--safe]
 
     Files will be saved to <prefix>/<extractor>/. The default prefix is the current working directory.
+    The default download profile is video. `--safe` supports audio, video, and image profiles only.
 
     By default things will download in a random order
 

{library-3.2.2 → library-3.2.4}/library/__main__.py

@@ -5,7 +5,7 @@ from tabulate import tabulate
 from library.utils import argparse_utils, iterables
 from library.utils.log_utils import log
 
-__version__ = "3.2.002"
+__version__ = "3.2.004"
 
 progs = {
     "Create database subcommands": {
@@ -58,7 +58,6 @@ progs = {
         "sample_compare": "Compare files using sample-hash and other shortcuts",
         "filesystem": "Find files by mimetype and size",
         "similar_files": "Find similar files based on filename and size",
-        "llm_map": "Run LLMs across multiple files",
     },
     "Tabular data subcommands": {
         "eda": "Exploratory Data Analysis on table-like files",
@@ -197,7 +196,6 @@ modules = {
     "library.files.sample_compare.sample_compare": ["cmp"],
     "library.files.sample_hash.sample_hash": ["hash", "hash-file"],
     "library.files.similar_files.similar_files": [],
-    "library.files.llm_map.llm_map": [],
     "library.folders.move_list.move_list": ["mv-list"],
     "library.folders.merge_mv.merge_mv": ["mv"],
     "library.folders.merge_mv.merge_cp": ["cp"],

{library-3.2.2 → library-3.2.4}/library/createdb/fs_add.py

@@ -264,7 +264,7 @@ def scan_path(args, path_str: str) -> int:
             while m is None:
                 m = extract_metadata(args, new_files.pop())
 
-            extract_chunk(args, [m])
+            extract_chunk(args, iterables.conform(m))
             db_utils.optimize(args)
             del args.playlist_path
 
@@ -301,7 +301,7 @@ def scan_path(args, path_str: str) -> int:
                     playlist_path=path, **{k: v for k, v in args.__dict__.items() if k not in {"db"}}
                 )
                 metadata = parallel.map(partial(extract_metadata, mp_args), chunk_paths)
-                metadata = list(filter(None, metadata))
+                metadata = iterables.conform(metadata)
                 extract_chunk(args, metadata)
             print()
 

{library-3.2.2 → library-3.2.4}/library/createdb/fs_add_metadata.py

@@ -1,4 +1,4 @@
-import os, re
+import argparse, os, re
 from multiprocessing import TimeoutError as mp_TimeoutError
 from pathlib import Path
 from timeit import default_timer as timer
@@ -39,7 +39,7 @@ munge_book_tags_slow = processes.with_timeout(350)(munge_book_tags)
 munge_book_tags_fast = processes.with_timeout(70)(munge_book_tags)
 
 
-def extract_metadata(mp_args, path) -> dict[str, str | int | None] | None:
+def extract_metadata(mp_args, path) -> dict[str, str | int | None] | list[dict[str, str | int | None]] | None:
     try:
         path.encode()
     except UnicodeEncodeError:
@@ -127,11 +127,17 @@ def extract_metadata(mp_args, path) -> dict[str, str | int | None] | None:
             )
             if result is None:
                 return None
+            if isinstance(result, list):
+                processed_args = argparse.Namespace(**(vars(mp_args) | {"process": False}))
+                return [m for output_path in result if (m := extract_metadata(processed_args, output_path))]
             path = m["path"] = str(result)
         elif objects.is_profile(mp_args, DBType.video) and Path(path).suffix not in [".av1.mkv"]:
             result = process_ffmpeg.process_path(mp_args, path)
             if result is None:
                 return None
+            if isinstance(result, list):
+                processed_args = argparse.Namespace(**(vars(mp_args) | {"process": False}))
+                return [m for output_path in result if (m := extract_metadata(processed_args, output_path))]
             path = m["path"] = str(result)
         elif objects.is_profile(mp_args, DBType.image) and Path(path).suffix not in [".avif", ".avifs"]:
             result = process_image.process_path(mp_args, path)

{library-3.2.2 → library-3.2.4}/library/createdb/getty_add.py

@@ -2,10 +2,13 @@ import sqlite3
 
 from library import usage
 from library.data.http_errors import HTTPStatus
-from library.utils import arggroups, argparse_utils, iterables, web
+from library.mediadb import db_media, db_playlists
+from library.utils import arggroups, argparse_utils, consts, iterables, web
 from library.utils.log_utils import log
 from library.utils.objects import traverse_obj
 
+GETTY_COLLECTION_URL = "https://data.getty.edu/museum/collection/"
+
 
 def parse_args():
     parser = argparse_utils.ArgumentParser(usage=usage.getty_add)
@@ -15,6 +18,7 @@ def parse_args():
     arggroups.database(parser)
     args = parser.parse_args()
     arggroups.args_post(args, parser, create_db=True)
+    args.profile = consts.DBType.image
 
     web.requests_session(args)  # prepare requests session
 
@@ -163,7 +167,7 @@ def objects_extract(args, j):
     )
 
     d = {
-        "path": image_path or None,
+        "path": image_path or j["id"],
         "title": j["_label"],
         "types": "; ".join(set(d["_label"] for d in j["classified_as"]) - ignore_types),
         "description": description,
@@ -199,6 +203,12 @@ def update_objects(args):
         ]
 
     print("Fetching", len(unknown_objects), "unknown objects")
+    playlists_id = db_playlists.add(
+        args,
+        GETTY_COLLECTION_URL,
+        {"title": "Getty Museum collection"},
+        extractor_key="Getty",
+    )
 
     for unknown_object in unknown_objects:
         log.debug("Fetching %s...", unknown_object)
@@ -206,14 +216,26 @@ def update_objects(args):
         page_data = getty_fetch(unknown_object)
         if page_data:
             images = objects_extract(args, page_data)
-            args.db["media"].insert_all(images, alter=True, pk="id")
+            for image in images:
+                db_media.add(args, {"playlists_id": playlists_id, **image})
         else:
-            args.db["media"].insert({"title": "404 Not Found", "object_path": unknown_object}, alter=True, pk="id")
+            db_media.add(
+                args,
+                {
+                    "playlists_id": playlists_id,
+                    "path": unknown_object,
+                    "title": "404 Not Found",
+                    "object_path": unknown_object,
+                },
+            )
 
 
 def getty_add():
     args = parse_args()
 
+    db_playlists.create(args)
+    db_media.create(args)
+
     update_activity_stream(args)
 
     """

{library-3.2.2 → library-3.2.4}/library/createdb/hn_add.py

@@ -1,4 +1,4 @@
-import argparse, asyncio, queue, sqlite3, threading
+import argparse, asyncio, importlib, queue, sqlite3, threading
 
 from library import usage
 from library.utils import arggroups, argparse_utils, db_utils, objects, web
@@ -103,7 +103,7 @@ async def run(args, db_queue):
 def hacker_news_add() -> None:
     args = parse_args(usage=usage.hn_add)
     try:
-        import aiohttp
+        importlib.import_module("aiohttp")
     except ModuleNotFoundError:
         log.error("aiohttp is required for hn_extract. Install with pip install aiohttp or pip install library[deluxe]")
         raise

{library-3.2.2 → library-3.2.4}/library/editdb/dedupe_db.py

@@ -36,11 +36,31 @@ def dedupe_rows(args, tablename, primary_keys, business_keys):
     with args.db.conn:
         args.db.conn.execute(
             f"""
-            DELETE FROM {tablename}
-            WHERE ({','.join(primary_keys)}) NOT IN (
-                SELECT {','.join(f"MIN({col}) AS {col}" for col in primary_keys)}
+            WITH ranked AS (
+                SELECT
+                    rowid
+                    , {','.join(business_keys)}
+                    , ROW_NUMBER() OVER (
+                        PARTITION BY {','.join(business_keys)}
+                        ORDER BY {','.join(primary_keys)}, rowid
+                    ) AS duplicate_number
+                    , COUNT(*) OVER (
+                        PARTITION BY {','.join(business_keys)}, {','.join(primary_keys)}
+                    ) AS primary_key_count
                 FROM {tablename}
-                GROUP BY {','.join(business_keys)}
+            ), candidates AS (
+                SELECT
+                    rowid
+                    , duplicate_number
+                    , MAX(primary_key_count) OVER (
+                        PARTITION BY {','.join(business_keys)}
+                    ) AS max_primary_key_count
+                FROM ranked
+            )
+            DELETE FROM {tablename} WHERE rowid IN (
+                SELECT rowid
+                FROM candidates
+                WHERE duplicate_number > 1 AND max_primary_key_count = 1
             )
             """,
         )

{library-3.2.2 → library-3.2.4}/library/editdb/dedupe_media.py

@@ -1,4 +1,4 @@
-import argparse, difflib, os, re, shlex, tempfile
+import argparse, difflib, os, shlex, tempfile
 from collections import defaultdict
 from concurrent.futures import ThreadPoolExecutor
 from copy import deepcopy
@@ -395,11 +395,6 @@ def get_fs_duplicates(args) -> list[dict]:
     return dup_media
 
 
-def filter_split_files(paths):
-    pattern = r"\.\d{3,5}\."
-    return filter(lambda x: not re.search(pattern, x), paths)
-
-
 def dedupe_media() -> None:
     args = parse_args()
 

{library-3.2.2 → library-3.2.4}/library/files/similar_files.py

@@ -27,13 +27,13 @@ def parse_args():
 
     arggroups.paths_or_stdin(parser)
     args = parser.parse_intermixed_args()
-    arggroups.args_post(args, parser)
 
+    if not args.filter_names and not args.filter_sizes and not args.filter_durations:
+        parser.error("specify at least one filter mode: --filter-names, --filter-sizes, or --filter-durations")
+
+    arggroups.args_post(args, parser)
     arggroups.files_post(args)
     arggroups.similar_files_post(args)
-    if not args.filter_names and not args.filter_sizes and not args.filter_durations:
-        print("Nothing to do")
-        raise NotImplementedError
 
     return args
 

{library-3.2.2 → library-3.2.4}/library/folders/big_dirs.py

@@ -91,6 +91,7 @@ def group_files_by_parent(args, media) -> list[dict]:
     for parent, media in list(p_media.items()):
         d[parent] = {
             "total": len(media),
+            "folders": 0,
             "duration": sum(m.get("duration") or 0 for m in media if not bool(m.get("time_deleted"))),
             "median_duration": nums.safe_median(m.get("duration") for m in media if not bool(m.get("time_deleted"))),
             "size": sum(m.get("size") or 0 for m in media if not bool(m.get("time_deleted"))),

{library-3.2.2 → library-3.2.4}/library/folders/merge_mv.py

@@ -217,7 +217,8 @@ def gen_src_dest(args, sources: Iterable[str], destination: str, shortcut_allowe
                 log.debug("rglob-file relpath %s", relpath)
                 if args.modify_depth:
                     rel_p = Path(relpath)
-                    parts = rel_p.parent.parts[args.modify_depth]
+                    parts = rel_p.parent.parts
+                    parts = parts[args.modify_depth]
                     relpath = os.path.join(*parts, rel_p.name)
                     log.debug("rglob-file modify_depth %s %s", parts, relpath)
 

{library-3.2.2 → library-3.2.4}/library/folders/similar_folders.py

@@ -48,12 +48,13 @@ def parse_args():
 
     arggroups.paths_or_stdin(parser)
     args = parser.parse_intermixed_args()
-    arggroups.args_post(args, parser)
 
     if not args.filter_names and not args.filter_counts and not args.filter_sizes and not args.filter_durations:
-        print("Nothing to do")
-        raise NotImplementedError
+        parser.error(
+            "specify at least one filter mode: --filter-names, --filter-counts, --filter-sizes, or --filter-durations"
+        )
 
+    arggroups.args_post(args, parser)
     if args.filter_counts and not any([args.folder_counts, args.file_counts, args.folder_sizes]):
         args.file_counts = ["+2"]
 

{library-3.2.2 → library-3.2.4}/library/mediadb/download.py

@@ -61,15 +61,15 @@ def parse_args():
 
     parser.set_defaults(fts=False)
     args, unk = parser.parse_known_intermixed_args()
-    arggroups.args_post(args, parser, create_db=args.database and args.database.endswith(consts.SQLITE_EXTENSIONS))
 
     if unk and not args.profile in (DBType.video, DBType.audio):
         parser.error(f"unrecognized arguments: {' '.join(unk)}")
-    args.unk = unk
 
-    if not args.profile and not args.print:
-        log.error("Download profile must be specified. Use one of: --video OR --audio OR --image OR --filesystem")
-        raise SystemExit(1)
+    if args.safe and args.profile == DBType.filesystem:
+        parser.error("--safe is supported only with --audio, --video, or --image")
+
+    arggroups.args_post(args, parser, create_db=args.database and args.database.endswith(consts.SQLITE_EXTENSIONS))
+    args.unk = unk
 
     arggroups.sql_fs_post(args)
     arggroups.filter_links_post(args)
@@ -212,6 +212,7 @@ def download(args=None) -> None:
                         local_path = None
                         error = str(excinfo)
 
+                    local_paths = [local_path]
                     if local_path and args.process:
                         extension = local_path.rsplit(".", 1)[-1].lower()
                         if extension in consts.AUDIO_ONLY_EXTENSIONS | consts.VIDEO_EXTENSIONS:
@@ -220,23 +221,26 @@ def download(args=None) -> None:
                             result = process_image.process_path(args, local_path)
 
                         if result is not None:
-                            local_path = str(result)
+                            local_paths = process_ffmpeg.result_paths(result)
 
                     is_not_found = error is not None and "HTTPNotFound" in error
                     if error is not None and "HTTPNotFound" not in error:
                         any_error = True
 
-                    db_media.download_add(
-                        args,
-                        webpath=original_path,
-                        info=m,
-                        local_path=local_path,
-                        error=error,
-                        mark_deleted=is_not_found,
-                        delete_webpath_entry=(
-                            not any_error if i == len(dl_paths) - 1 else False
-                        ),  # only check after last download link was saved
-                    )
+                    for output_index, local_path in enumerate(local_paths):
+                        db_media.download_add(
+                            args,
+                            webpath=original_path,
+                            info=m,
+                            local_path=local_path,
+                            error=error,
+                            mark_deleted=is_not_found,
+                            delete_webpath_entry=(
+                                not any_error
+                                if i == len(dl_paths) - 1 and output_index == len(local_paths) - 1
+                                else False
+                            ),  # only check after last downloaded output was saved
+                        )
             else:
                 raise NotImplementedError
 

{library-3.2.2 → library-3.2.4}/library/mediadb/download_status.py

@@ -44,13 +44,14 @@ def download_status() -> None:
 
     for m in media:
         extractor_key = m.get("extractor_key", "Playlist-less media")
+        time_downloaded = m.get("time_downloaded") or 0
 
         if "download_attempts" in m and (m["download_attempts"] or 0) >= args.download_retries:
             extractor_stats[extractor_key]["retries_exceeded"] += 1
-        elif (m.get("time_downloaded") or 0) > 0 or (
+        elif time_downloaded > 0 or (
             not m["path"].startswith("http") and (m.get("webpath") or "").startswith("http")
         ):
-            if (m["time_downloaded"] + retry_delay) >= consts.APPLICATION_START:
+            if (time_downloaded + retry_delay) >= consts.APPLICATION_START:
                 extractor_stats[extractor_key]["downloaded_recently"] += 1
         elif m["path"].startswith("http"):
             if "time_modified" in m: