promnesia 1.2.20240810__py3-none-any.whl → 1.4.20250909__py3-none-any.whl

promnesia/compat.py

@@ -1,12 +1,12 @@
-## we used to have compat fixes here for these for python3.7
-## keeping in case any sources depended on compat functions
-from subprocess import PIPE, run, check_call, check_output, Popen
-from typing import Protocol, Literal
-## 
+from typing import TYPE_CHECKING
 
+if not TYPE_CHECKING:
+    ## we used to have compat fixes here for these for python3.7
+    ## keeping in case any sources depended on compat functions
+    from subprocess import PIPE, Popen, check_call, check_output, run  # noqa: F401
+    from typing import Literal, Protocol  # noqa: F401
+    ##
 
-# can remove after python3.9
-def removeprefix(text: str, prefix: str) -> str:
-    if text.startswith(prefix):
-        return text[len(prefix):]
-    return text
+    # todo deprecate properly
+    def removeprefix(text: str, prefix: str) -> str:
+        return text.removeprefix(prefix)

promnesia/config.py

@@ -1,55 +1,53 @@
-from pathlib import Path
-import os
-from types import ModuleType
-from typing import List, Optional, Union, NamedTuple, Iterable, Callable
+from __future__ import annotations
+
 import importlib
 import importlib.util
+import os
 import warnings
+from collections.abc import Callable, Iterable
+from pathlib import Path
+from types import ModuleType
+from typing import NamedTuple
 
-from .common import PathIsh, default_output_dir, default_cache_dir
-from .common import Res, Source, DbVisit
-
+from .common import DbVisit, PathIsh, Res, Source, default_cache_dir, default_output_dir
 
 HookT = Callable[[Res[DbVisit]], Iterable[Res[DbVisit]]]
 
 
-from typing import Any
-
-
 ModuleName = str
 
 # something that can be converted into a proper Source
-ConfigSource = Union[Source, ModuleName, ModuleType]
+ConfigSource = Source | ModuleName | ModuleType
 
 
 class Config(NamedTuple):
     # TODO remove default from sources once migrated
-    SOURCES: List[ConfigSource] = []
+    SOURCES: list[ConfigSource] = []  # noqa: RUF012
 
     # if not specified, uses user data dir
-    OUTPUT_DIR: Optional[PathIsh] = None
+    OUTPUT_DIR: PathIsh | None = None
 
-    CACHE_DIR: Optional[PathIsh] = ''
-    FILTERS: List[str] = []
+    CACHE_DIR: PathIsh | None = ''
+    FILTERS: list[str] = []  # noqa: RUF012
 
-    HOOK: Optional[HookT] = None
+    HOOK: HookT | None = None
 
     #
     # NOTE: INDEXERS is deprecated, use SOURCES instead
-    INDEXERS: List[ConfigSource] = []
-    #MIME_HANDLER: Optional[str] = None # TODO
+    INDEXERS: list[ConfigSource] = []  # noqa: RUF012
+    # MIME_HANDLER: Optional[str] = None # TODO
 
     @property
     def sources(self) -> Iterable[Res[Source]]:
-        idx = self.INDEXERS
-
         if len(self.INDEXERS) > 0:
             warnings.warn("'INDEXERS' is deprecated. Please use 'SOURCES'!", DeprecationWarning)
 
         raw = self.SOURCES + self.INDEXERS
 
         if len(raw) == 0:
-            raise RuntimeError("Please specify SOURCES in the config! See https://github.com/karlicoss/promnesia#setup for more information")
+            raise RuntimeError(
+                "Please specify SOURCES in the config! See https://github.com/karlicoss/promnesia#setup for more information"
+            )
 
         for r in raw:
             if isinstance(r, ModuleName):
@@ -68,14 +66,14 @@ class Config(NamedTuple):
                 yield Source(r)
 
     @property
-    def cache_dir(self) -> Optional[Path]:
+    def cache_dir(self) -> Path | None:
         # TODO we used to use this for cachew, but it's best to rely on HPI modules etc to cofigure this
         # keeping just in case for now
         cd = self.CACHE_DIR
-        cpath: Optional[Path]
+        cpath: Path | None
         if cd is None:
-            cpath = None # means 'disabled' in cachew
-        elif cd == '': # meh.. but need to make it None friendly..
+            cpath = None  # means 'disabled' in cachew
+        elif cd == '':  # meh.. but need to make it None friendly..
             cpath = default_cache_dir()
         else:
             cpath = Path(cd)
@@ -96,15 +94,17 @@ class Config(NamedTuple):
         return self.output_dir / 'promnesia.sqlite'
 
     @property
-    def hook(self) -> Optional[HookT]:
+    def hook(self) -> HookT | None:
         return self.HOOK
 
-instance: Optional[Config] = None
+
+instance: Config | None = None
 
 
 def has() -> bool:
     return instance is not None
 
+
 def get() -> Config:
     assert instance is not None, "Expected config to be set, but it's not"
     return instance
@@ -126,9 +126,12 @@ def import_config(config_file: PathIsh) -> Config:
 
     # todo just exec??
     name = p.stem
-    spec = importlib.util.spec_from_file_location(name, p); assert spec is not None
-    mod = importlib.util.module_from_spec(spec); assert mod is not None
-    loader = spec.loader; assert loader is not None
+    spec = importlib.util.spec_from_file_location(name, p)
+    assert spec is not None
+    mod = importlib.util.module_from_spec(spec)
+    assert mod is not None
+    loader = spec.loader
+    assert loader is not None
     loader.exec_module(mod)
 
     d = {}
@@ -139,7 +142,7 @@ def import_config(config_file: PathIsh) -> Config:
 
 
 # TODO: ugh. this causes warnings to be repeated multiple times... need to reuse the pool or something..
-def use_cores() -> Optional[int]:
+def use_cores() -> int | None:
     '''
     Somewhat experimental.
     For now only used in sources.auto, perhaps later will be shared among the other indexers.
@@ -150,15 +153,15 @@ def use_cores() -> Optional[int]:
         return None
     try:
         return int(cs)
-    except ValueError: # any other value means 'use all
+    except ValueError:  # any other value means 'use all
         return 0
 
 
-def extra_fd_args() -> List[str]:
+def extra_fd_args() -> list[str]:
     '''
     Not sure where it belongs yet... so via env variable for now
     Can be used to pass --ignore-file parameter
     '''
     v = os.environ.get('PROMNESIA_FD_EXTRA_ARGS', '')
-    extra = v.split() # eh, hopefully splitting that way is ok...
+    extra = v.split()  # eh, hopefully splitting that way is ok...
     return extra

promnesia/database/common.py

@@ -1,10 +1,11 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
 from datetime import datetime
-from typing import Sequence, Tuple
 
 from sqlalchemy import (
     Column,
     Integer,
-    Row,
     String,
 )
 
@@ -30,7 +31,7 @@ def get_columns() -> Sequence[Column]:
     return res
 
 
-def db_visit_to_row(v: DbVisit) -> Tuple:
+def db_visit_to_row(v: DbVisit) -> tuple:
     # ugh, very hacky...
     # we want to make sure the resulting tuple only consists of simple types
     # so we can use dbengine directly

promnesia/database/dump.py

@@ -1,9 +1,10 @@
-from pathlib import Path
+from __future__ import annotations
+
 import sqlite3
-from typing import Dict, Iterable, List, Optional, Set
+from collections.abc import Iterable
+from pathlib import Path
 
 from more_itertools import chunked
-
 from sqlalchemy import (
     Engine,
     MetaData,
@@ -16,6 +17,7 @@ from sqlalchemy import (
 )
 from sqlalchemy.dialects import sqlite as dialect_sqlite
 
+from .. import config
 from ..common import (
     DbVisit,
     Loc,
@@ -24,9 +26,7 @@ from ..common import (
     get_logger,
     now_tz,
 )
-from .common import get_columns, db_visit_to_row
-from .. import config
-
+from .common import db_visit_to_row, get_columns
 
 # NOTE: I guess the main performance benefit from this is not creating too many tmp lists and avoiding overhead
 # since as far as sql is concerned it should all be in the same transaction. only a guess
@@ -50,7 +50,7 @@ def begin_immediate_transaction(conn):
     conn.exec_driver_sql('BEGIN IMMEDIATE')
 
 
-Stats = Dict[Optional[SourceName], int]
+Stats = dict[SourceName | None, int]
 
 
 # returns critical warnings
@@ -58,8 +58,8 @@ def visits_to_sqlite(
     vit: Iterable[Res[DbVisit]],
     *,
     overwrite_db: bool,
-    _db_path: Optional[Path] = None,  # only used in tests
-) -> List[Exception]:
+    _db_path: Path | None = None,  # only used in tests
+) -> list[Exception]:
     if _db_path is None:
         db_path = config.get().db
     else:
@@ -95,7 +95,7 @@ def visits_to_sqlite(
 
     def query_total_stats(conn) -> Stats:
         query = select(table.c.src, func.count(table.c.src)).select_from(table).group_by(table.c.src)
-        return {src: cnt for (src, cnt) in conn.execute(query).all()}
+        return dict(conn.execute(query).all())
 
     def get_engine(*args, **kwargs) -> Engine:
         # kwargs['echo'] = True  # useful for debugging
@@ -122,7 +122,7 @@ def visits_to_sqlite(
     # (note that this also requires WAL mode)
     engine = get_engine(f'sqlite:///{db_path}', connect_args={'timeout': _CONNECTION_TIMEOUT_SECONDS})
 
-    cleared: Set[str] = set()
+    cleared: set[str] = set()
 
     # by default, sqlalchemy does some sort of BEGIN (implicit) transaction, which doesn't provide proper isolation??
     # see https://docs.sqlalchemy.org/en/20/dialects/sqlite.html#serializable-isolation-savepoints-transactional-ddl
@@ -144,7 +144,7 @@ def visits_to_sqlite(
         insert_stmt_raw = str(insert_stmt.compile(dialect=dialect_sqlite.dialect(paramstyle='qmark')))
 
         for chunk in chunked(vit_ok(), n=_CHUNK_BY):
-            srcs = set(v.src or '' for v in chunk)
+            srcs = {v.src or '' for v in chunk}
             new = srcs.difference(cleared)
 
             for src in new:
@@ -181,7 +181,7 @@ def visits_to_sqlite(
         for k, v in stats_changes.items():
             logger.info(f'database stats changes: {k} {v}')
 
-    res: List[Exception] = []
+    res: list[Exception] = []
     if total_ok == 0:
         res.append(RuntimeError('No visits were indexed, something is probably wrong!'))
     return res

promnesia/database/load.py

@@ -1,19 +1,19 @@
+from __future__ import annotations
+
 from pathlib import Path
-from typing import Tuple, List
 
 from sqlalchemy import (
-    create_engine,
-    exc,
     Engine,
-    MetaData,
     Index,
+    MetaData,
     Table,
+    create_engine,
+    exc,
 )
 
 from .common import DbVisit, get_columns, row_to_db_visit
 
-
-DbStuff = Tuple[Engine, Table]
+DbStuff = tuple[Engine, Table]
 
 
 def get_db_stuff(db_path: Path) -> DbStuff:
@@ -39,7 +39,7 @@ def get_db_stuff(db_path: Path) -> DbStuff:
     return engine, table
 
 
-def get_all_db_visits(db_path: Path) -> List[DbVisit]:
+def get_all_db_visits(db_path: Path) -> list[DbVisit]:
     # NOTE: this is pretty inefficient if the DB is huge
     # mostly intended for tests
     engine, table = get_db_stuff(db_path)

promnesia/extract.py

@@ -1,20 +1,22 @@
-from functools import lru_cache
+from __future__ import annotations
+
 import re
-import traceback
-from typing import Set, Iterable, Sequence, Union
+from collections.abc import Iterable, Sequence
+from functools import lru_cache
 
 from .cannon import CanonifyException
 from .common import (
-    logger,
-    DbVisit, Visit,
-    Res,
-    SourceName, Source,
+    DbVisit,
     Filter,
+    Res,
+    Results,
+    Source,
+    SourceName,
     Url,
-    Results, Extractor,
+    Visit,
+    logger,
 )
 
-
 DEFAULT_FILTERS = (
     r'^chrome-\w+://',
     r'chrome://newtab',
@@ -23,18 +25,17 @@ DEFAULT_FILTERS = (
     r'^about:',
     r'^blob:',
     r'^view-source:',
-
     r'^content:',
 )
 
 
 # TODO maybe move these to configs?
-@lru_cache(1) #meh, not sure what would happen under tests?
+@lru_cache(1)  # meh, not sure what would happen under tests?
 def filters() -> Sequence[Filter]:
     from . import config
 
     flt = list(DEFAULT_FILTERS)
-    if config.has(): # meeeh...
+    if config.has():  # meeeh...
         cfg = config.get()
         flt.extend(cfg.FILTERS)
     return tuple(make_filter(f) for f in flt)
@@ -53,7 +54,7 @@ def extract_visits(source: Source, *, src: SourceName) -> Iterable[Res[DbVisit]]
         yield e
         return
 
-    handled: Set[Visit] = set()
+    handled: set[Visit] = set()
     try:
         for p in vit:
             if isinstance(p, Exception):
@@ -65,7 +66,7 @@ def extract_visits(source: Source, *, src: SourceName) -> Iterable[Res[DbVisit]]
                 yield p
                 continue
 
-            if p in handled: # no need to emit duplicates
+            if p in handled:  # no need to emit duplicates
                 continue
             handled.add(p)
 
@@ -75,7 +76,6 @@ def extract_visits(source: Source, *, src: SourceName) -> Iterable[Res[DbVisit]]
         logger.exception(e)
         yield e
 
-
     logger.info('extracting via %s: got %d visits', source.description, len(handled))
 
 
@@ -94,11 +94,13 @@ def filtered(url: Url) -> bool:
     return any(f(url) for f in filters())
 
 
-def make_filter(thing: Union[str, Filter]) -> Filter:
+def make_filter(thing: str | Filter) -> Filter:
     if isinstance(thing, str):
         rc = re.compile(thing)
+
         def filter_(u: str) -> bool:
             return rc.search(u) is not None
+
         return filter_
-    else: # must be predicate
+    else:  # must be predicate
         return thing

promnesia/logging.py

@@ -3,20 +3,25 @@
 Default logger is a bit meh, see 'test'/run this file for a demo
 '''
 
+
 def test() -> None:
     import logging
     import sys
-    from typing import Callable
+    from collections.abc import Callable
 
     M: Callable[[str], None] = lambda s: print(s, file=sys.stderr)
 
     M("   Logging module's defaults are not great...'")
     l = logging.getLogger('test_logger')
-    l.error("For example, this should be logged as error. But it's not even formatted properly, doesn't have logger name or level")
+    l.error(
+        "For example, this should be logged as error. But it's not even formatted properly, doesn't have logger name or level"
+    )
 
     M("   The reason is that you need to remember to call basicConfig() first")
     logging.basicConfig()
-    l.error("OK, this is better. But the default format kinda sucks, I prefer having timestamps and the file/line number")
+    l.error(
+        "OK, this is better. But the default format kinda sucks, I prefer having timestamps and the file/line number"
+    )
 
     M("")
     M("    With LazyLogger you get a reasonable logging format, colours and other neat things")
@@ -29,12 +34,12 @@ def test() -> None:
 
 
 import logging
-from typing import Union, Optional, cast
 import os
 import warnings
+from typing import cast
 
 Level = int
-LevelIsh = Optional[Union[Level, str]]
+LevelIsh = Level | str | None
 
 
 def mklevel(level: LevelIsh) -> Level:
@@ -50,18 +55,22 @@ def mklevel(level: LevelIsh) -> Level:
 
 
 FORMAT = '{start}[%(levelname)-7s %(asctime)s %(name)s %(filename)s:%(lineno)d]{end} %(message)s'
+# fmt: off
 FORMAT_COLOR   = FORMAT.format(start='%(color)s', end='%(end_color)s')
-FORMAT_NOCOLOR = FORMAT.format(start='', end='')
+FORMAT_NOCOLOR = FORMAT.format(start=''         , end='')
+# fmt: on
 DATEFMT = '%Y-%m-%d %H:%M:%S'
 
-COLLAPSE_DEBUG_LOGS = os.environ.get('COLLAPSE_DEBUG_LOGS', False)
+COLLAPSE_DEBUG_LOGS = os.environ.get('COLLAPSE_DEBUG_LOGS', False)  # noqa: PLW1508
 
 _init_done = 'lazylogger_init_done'
 
+
 def setup_logger(logger: logging.Logger, level: LevelIsh) -> None:
     lvl = mklevel(level)
     try:
-        import logzero  # type: ignore[import-not-found]
+        import logzero  # type: ignore[import-not-found,import-untyped,unused-ignore]
+
         formatter = logzero.LogFormatter(
             fmt=FORMAT_COLOR,
             datefmt=DATEFMT,
@@ -73,7 +82,7 @@ def setup_logger(logger: logging.Logger, level: LevelIsh) -> None:
         use_logzero = False
 
     logger.addFilter(AddExceptionTraceback())
-    if use_logzero and not COLLAPSE_DEBUG_LOGS: # all set, nothing to do
+    if use_logzero and not COLLAPSE_DEBUG_LOGS:  # all set, nothing to do
         # 'simple' setup
         logzero.setup_logger(logger.name, level=lvl, formatter=formatter)  # type: ignore[possibly-undefined]
         return
@@ -91,16 +100,17 @@ class LazyLogger(logging.Logger):
         logger = logging.getLogger(name)
 
         # this is called prior to all _log calls so makes sense to do it here?
-        def isEnabledFor_lazyinit(*args, logger=logger, orig=logger.isEnabledFor, **kwargs) -> bool:
+        def isEnabledFor_lazyinit(*args, logger: logging.Logger = logger, orig=logger.isEnabledFor, **kwargs) -> bool:
             if not getattr(logger, _init_done, False):  # init once, if necessary
                 setup_logger(logger, level=level)
                 setattr(logger, _init_done, True)
-                logger.isEnabledFor = orig # restore the callback
-            return orig(*args, **kwargs)
+                # restore the callback
+                logger.isEnabledFor = orig  # type: ignore[method-assign]  # ty: ignore[invalid-assignment]
+            return orig(*args, **kwargs)  # ty: ignore[missing-argument]
 
         # oh god.. otherwise might go into an inf loop
         if not hasattr(logger, _init_done):
-            setattr(logger, _init_done, False) # will setup on the first call
+            setattr(logger, _init_done, False)  # will setup on the first call
             logger.isEnabledFor = isEnabledFor_lazyinit  # type: ignore[method-assign]
         return cast(LazyLogger, logger)
 
@@ -129,6 +139,7 @@ class CollapseDebugHandler(logging.StreamHandler):
     Collapses subsequent debug log lines and redraws on the same line.
     Hopefully this gives both a sense of progress and doesn't clutter the terminal as much?
     '''
+
     last = False
 
     def emit(self, record: logging.LogRecord) -> None:
@@ -137,12 +148,13 @@ class CollapseDebugHandler(logging.StreamHandler):
             cur = record.levelno == logging.DEBUG and '\n' not in msg
             if cur:
                 if self.last:
-                    self.stream.write('\033[K' + '\r') # clear line + return carriage
+                    self.stream.write('\033[K' + '\r')  # clear line + return carriage
             else:
                 if self.last:
-                    self.stream.write('\n') # clean up after the last debug line
+                    self.stream.write('\n')  # clean up after the last debug line
             self.last = cur
             import os
+
             columns, _ = os.get_terminal_size(0)
             # ugh. the columns thing is meh. dunno I guess ultimately need curses for that
             # TODO also would be cool to have a terminal post-processor? kinda like tail but aware of logging keywords (INFO/DEBUG/etc)

promnesia/misc/install_server.py

@@ -1,15 +1,12 @@
-#!/usr/bin/env python3
 from __future__ import annotations
 
 import argparse
 import os
+import platform
 import sys
 import time
 from pathlib import Path
-import platform
-import shutil
 from subprocess import check_call, run
-from typing import List
 
 SYSTEM = platform.system()
 UNSUPPORTED_SYSTEM = RuntimeError(f'Platform {SYSTEM} is not supported yet!')
@@ -54,51 +51,57 @@ LAUNCHD_TEMPLATE = '''
 
 
 def systemd(*args: str | Path, method=check_call) -> None:
-    method([
-        'systemctl', '--no-pager', '--user', *args,
-    ])
+    method(['systemctl', '--no-pager', '--user', *args])
 
 
-def install_systemd(name: str, out: Path, launcher: str, largs: List[str]) -> None:
+def install_systemd(name: str, out: Path, launcher: str, largs: list[str]) -> None:
     unit_name = name
 
     import shlex
+
     extra_args = ' '.join(shlex.quote(str(a)) for a in largs)
 
-    out.write_text(SYSTEMD_TEMPLATE.format(
-        launcher=launcher,
-        extra_args=extra_args,
-    ))
+    out.write_text(
+        SYSTEMD_TEMPLATE.format(
+            launcher=launcher,
+            extra_args=extra_args,
+        )
+    )
 
     try:
-        systemd('stop' , unit_name, method=run) # ignore errors here if it wasn't running in the first place
+        systemd('stop', unit_name, method=run)  # ignore errors here if it wasn't running in the first place
         systemd('daemon-reload')
         systemd('enable', unit_name)
-        systemd('start' , unit_name)
+        systemd('start', unit_name)
         systemd('status', unit_name)
     except Exception as e:
-        print(f"Something has gone wrong... you might want to use 'journalctl --user -u {unit_name}' to investigate", file=sys.stderr)
+        print(
+            f"Something has gone wrong... you might want to use 'journalctl --user -u {unit_name}' to investigate",
+            file=sys.stderr,
+        )
         raise e
 
 
-def install_launchd(name: str, out: Path, launcher: str, largs: List[str]) -> None:
+def install_launchd(name: str, out: Path, launcher: str, largs: list[str]) -> None:
     service_name = name
     arguments = '\n'.join(f'<string>{a}</string>' for a in [launcher, *largs])
-    out.write_text(LAUNCHD_TEMPLATE.format(
-        service_name=service_name,
-        arguments=arguments,
-    ))
+    out.write_text(
+        LAUNCHD_TEMPLATE.format(
+            service_name=service_name,
+            arguments=arguments,
+        )
+    )
     cmd = ['launchctl', 'load', '-w', str(out)]
     print('Running: ' + ' '.join(cmd), file=sys.stderr)
     check_call(cmd)
 
-    time.sleep(1) # to give it some time? not sure if necessary
+    time.sleep(1)  # to give it some time? not sure if necessary
     check_call(f'launchctl list | grep {name}', shell=True)
 
 
 def install(args: argparse.Namespace) -> None:
     name = args.name
-    # todo use appdirs for config dir detection
+    # todo use platformdirs for config dir detection
     if SYSTEM == 'Linux':
         # Check for existence of systemd
         # https://www.freedesktop.org/software/systemd/man/sd_booted.html
@@ -108,7 +111,7 @@ def install(args: argparse.Namespace) -> None:
         if Path(name).suffix != suf:
             name = name + suf
         out = Path(f'~/.config/systemd/user/{name}')
-    elif SYSTEM == 'Darwin': # osx
+    elif SYSTEM == 'Darwin':  # osx
         out = Path(f'~/Library/LaunchAgents/{name}.plist')
     else:
         raise UNSUPPORTED_SYSTEM
@@ -116,22 +119,24 @@ def install(args: argparse.Namespace) -> None:
     print(f"Writing launch script to {out}", file=sys.stderr)
 
     # ugh. we want to know whether we're invoked 'properly' as an executable or ad-hoc via scripts/promnesia
+    extra_exe: list[str] = []
     if os.environ.get('DIRTY_RUN') is not None:
         launcher = str(root() / 'scripts/promnesia')
     else:
-        exe = shutil.which('promnesia'); assert exe is not None
-        launcher = exe # older systemd wants absolute paths..
+        launcher = sys.executable
+        extra_exe = ['-m', 'promnesia']
 
     db = args.db
     largs = [
+        *extra_exe,
         'serve',
         *([] if db is None else ['--db', str(db)]),
         '--timezone', args.timezone,
         '--host', args.host,
         '--port', args.port,
-    ]
+    ]  # fmt: skip
 
-    out.parent.mkdir(parents=True, exist_ok=True) # sometimes systemd dir doesn't exist
+    out.parent.mkdir(parents=True, exist_ok=True)  # sometimes systemd dir doesn't exist
     if SYSTEM == 'Linux':
         install_systemd(name=name, out=out, launcher=launcher, largs=largs)
     elif SYSTEM == 'Darwin':