promnesia 1.2.20240810__py3-none-any.whl → 1.3.20241021__py3-none-any.whl

promnesia/compat.py

@@ -1,12 +1,12 @@
-## we used to have compat fixes here for these for python3.7
-## keeping in case any sources depended on compat functions
-from subprocess import PIPE, run, check_call, check_output, Popen
-from typing import Protocol, Literal
-## 
+from typing import TYPE_CHECKING
 
+if not TYPE_CHECKING:
+    ## we used to have compat fixes here for these for python3.7
+    ## keeping in case any sources depended on compat functions
+    from subprocess import PIPE, Popen, check_call, check_output, run  # noqa: F401
+    from typing import Literal, Protocol  # noqa: F401
+    ##
 
-# can remove after python3.9
-def removeprefix(text: str, prefix: str) -> str:
-    if text.startswith(prefix):
-        return text[len(prefix):]
-    return text
+    # todo deprecate properly
+    def removeprefix(text: str, prefix: str) -> str:
+        return text.removeprefix(prefix)

promnesia/config.py

@@ -1,21 +1,19 @@
-from pathlib import Path
-import os
-from types import ModuleType
-from typing import List, Optional, Union, NamedTuple, Iterable, Callable
+from __future__ import annotations
+
 import importlib
 import importlib.util
+import os
 import warnings
+from collections.abc import Iterable
+from pathlib import Path
+from types import ModuleType
+from typing import Callable, NamedTuple, Union
 
-from .common import PathIsh, default_output_dir, default_cache_dir
-from .common import Res, Source, DbVisit
-
+from .common import DbVisit, PathIsh, Res, Source, default_cache_dir, default_output_dir
 
 HookT = Callable[[Res[DbVisit]], Iterable[Res[DbVisit]]]
 
 
-from typing import Any
-
-
 ModuleName = str
 
 # something that can be converted into a proper Source
@@ -24,19 +22,19 @@ ConfigSource = Union[Source, ModuleName, ModuleType]
 
 class Config(NamedTuple):
     # TODO remove default from sources once migrated
-    SOURCES: List[ConfigSource] = []
+    SOURCES: list[ConfigSource] = []
 
     # if not specified, uses user data dir
-    OUTPUT_DIR: Optional[PathIsh] = None
+    OUTPUT_DIR: PathIsh | None = None
 
-    CACHE_DIR: Optional[PathIsh] = ''
-    FILTERS: List[str] = []
+    CACHE_DIR: PathIsh | None = ''
+    FILTERS: list[str] = []
 
-    HOOK: Optional[HookT] = None
+    HOOK: HookT | None = None
 
     #
     # NOTE: INDEXERS is deprecated, use SOURCES instead
-    INDEXERS: List[ConfigSource] = []
+    INDEXERS: list[ConfigSource] = []
     #MIME_HANDLER: Optional[str] = None # TODO
 
     @property
@@ -68,11 +66,11 @@ class Config(NamedTuple):
                 yield Source(r)
 
     @property
-    def cache_dir(self) -> Optional[Path]:
+    def cache_dir(self) -> Path | None:
         # TODO we used to use this for cachew, but it's best to rely on HPI modules etc to cofigure this
         # keeping just in case for now
         cd = self.CACHE_DIR
-        cpath: Optional[Path]
+        cpath: Path | None
         if cd is None:
             cpath = None # means 'disabled' in cachew
         elif cd == '': # meh.. but need to make it None friendly..
@@ -96,10 +94,10 @@ class Config(NamedTuple):
         return self.output_dir / 'promnesia.sqlite'
 
     @property
-    def hook(self) -> Optional[HookT]:
+    def hook(self) -> HookT | None:
         return self.HOOK
 
-instance: Optional[Config] = None
+instance: Config | None = None
 
 
 def has() -> bool:
@@ -139,7 +137,7 @@ def import_config(config_file: PathIsh) -> Config:
 
 
 # TODO: ugh. this causes warnings to be repeated multiple times... need to reuse the pool or something..
-def use_cores() -> Optional[int]:
+def use_cores() -> int | None:
     '''
     Somewhat experimental.
     For now only used in sources.auto, perhaps later will be shared among the other indexers.
@@ -154,7 +152,7 @@ def use_cores() -> Optional[int]:
         return 0
 
 
-def extra_fd_args() -> List[str]:
+def extra_fd_args() -> list[str]:
     '''
     Not sure where it belongs yet... so via env variable for now
     Can be used to pass --ignore-file parameter

promnesia/database/common.py

@@ -1,10 +1,11 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
 from datetime import datetime
-from typing import Sequence, Tuple
 
 from sqlalchemy import (
     Column,
     Integer,
-    Row,
     String,
 )
 
@@ -30,7 +31,7 @@ def get_columns() -> Sequence[Column]:
     return res
 
 
-def db_visit_to_row(v: DbVisit) -> Tuple:
+def db_visit_to_row(v: DbVisit) -> tuple:
     # ugh, very hacky...
     # we want to make sure the resulting tuple only consists of simple types
     # so we can use dbengine directly

promnesia/database/dump.py

@@ -1,9 +1,11 @@
-from pathlib import Path
+from __future__ import annotations
+
 import sqlite3
-from typing import Dict, Iterable, List, Optional, Set
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Optional
 
 from more_itertools import chunked
-
 from sqlalchemy import (
     Engine,
     MetaData,
@@ -16,6 +18,7 @@ from sqlalchemy import (
 )
 from sqlalchemy.dialects import sqlite as dialect_sqlite
 
+from .. import config
 from ..common import (
     DbVisit,
     Loc,
@@ -24,9 +27,7 @@ from ..common import (
     get_logger,
     now_tz,
 )
-from .common import get_columns, db_visit_to_row
-from .. import config
-
+from .common import db_visit_to_row, get_columns
 
 # NOTE: I guess the main performance benefit from this is not creating too many tmp lists and avoiding overhead
 # since as far as sql is concerned it should all be in the same transaction. only a guess
@@ -50,7 +51,7 @@ def begin_immediate_transaction(conn):
     conn.exec_driver_sql('BEGIN IMMEDIATE')
 
 
-Stats = Dict[Optional[SourceName], int]
+Stats = dict[Optional[SourceName], int]
 
 
 # returns critical warnings
@@ -58,8 +59,8 @@ def visits_to_sqlite(
     vit: Iterable[Res[DbVisit]],
     *,
     overwrite_db: bool,
-    _db_path: Optional[Path] = None,  # only used in tests
-) -> List[Exception]:
+    _db_path: Path | None = None,  # only used in tests
+) -> list[Exception]:
     if _db_path is None:
         db_path = config.get().db
     else:
@@ -95,7 +96,7 @@ def visits_to_sqlite(
 
     def query_total_stats(conn) -> Stats:
         query = select(table.c.src, func.count(table.c.src)).select_from(table).group_by(table.c.src)
-        return {src: cnt for (src, cnt) in conn.execute(query).all()}
+        return dict(conn.execute(query).all())
 
     def get_engine(*args, **kwargs) -> Engine:
         # kwargs['echo'] = True  # useful for debugging
@@ -122,7 +123,7 @@ def visits_to_sqlite(
     # (note that this also requires WAL mode)
     engine = get_engine(f'sqlite:///{db_path}', connect_args={'timeout': _CONNECTION_TIMEOUT_SECONDS})
 
-    cleared: Set[str] = set()
+    cleared: set[str] = set()
 
     # by default, sqlalchemy does some sort of BEGIN (implicit) transaction, which doesn't provide proper isolation??
     # see https://docs.sqlalchemy.org/en/20/dialects/sqlite.html#serializable-isolation-savepoints-transactional-ddl
@@ -144,7 +145,7 @@ def visits_to_sqlite(
         insert_stmt_raw = str(insert_stmt.compile(dialect=dialect_sqlite.dialect(paramstyle='qmark')))
 
         for chunk in chunked(vit_ok(), n=_CHUNK_BY):
-            srcs = set(v.src or '' for v in chunk)
+            srcs = {v.src or '' for v in chunk}
             new = srcs.difference(cleared)
 
             for src in new:
@@ -181,7 +182,7 @@ def visits_to_sqlite(
         for k, v in stats_changes.items():
             logger.info(f'database stats changes: {k} {v}')
 
-    res: List[Exception] = []
+    res: list[Exception] = []
     if total_ok == 0:
         res.append(RuntimeError('No visits were indexed, something is probably wrong!'))
     return res

promnesia/database/load.py

@@ -1,19 +1,19 @@
+from __future__ import annotations
+
 from pathlib import Path
-from typing import Tuple, List
 
 from sqlalchemy import (
-    create_engine,
-    exc,
     Engine,
-    MetaData,
     Index,
+    MetaData,
     Table,
+    create_engine,
+    exc,
 )
 
 from .common import DbVisit, get_columns, row_to_db_visit
 
-
-DbStuff = Tuple[Engine, Table]
+DbStuff = tuple[Engine, Table]
 
 
 def get_db_stuff(db_path: Path) -> DbStuff:
@@ -39,7 +39,7 @@ def get_db_stuff(db_path: Path) -> DbStuff:
     return engine, table
 
 
-def get_all_db_visits(db_path: Path) -> List[DbVisit]:
+def get_all_db_visits(db_path: Path) -> list[DbVisit]:
     # NOTE: this is pretty inefficient if the DB is huge
     # mostly intended for tests
     engine, table = get_db_stuff(db_path)

promnesia/extract.py

@@ -1,20 +1,22 @@
-from functools import lru_cache
+from __future__ import annotations
+
 import re
-import traceback
-from typing import Set, Iterable, Sequence, Union
+from collections.abc import Iterable, Sequence
+from functools import lru_cache
 
 from .cannon import CanonifyException
 from .common import (
-    logger,
-    DbVisit, Visit,
-    Res,
-    SourceName, Source,
+    DbVisit,
     Filter,
+    Res,
+    Results,
+    Source,
+    SourceName,
     Url,
-    Results, Extractor,
+    Visit,
+    logger,
 )
 
-
 DEFAULT_FILTERS = (
     r'^chrome-\w+://',
     r'chrome://newtab',
@@ -53,7 +55,7 @@ def extract_visits(source: Source, *, src: SourceName) -> Iterable[Res[DbVisit]]
         yield e
         return
 
-    handled: Set[Visit] = set()
+    handled: set[Visit] = set()
     try:
         for p in vit:
             if isinstance(p, Exception):
@@ -94,7 +96,7 @@ def filtered(url: Url) -> bool:
     return any(f(url) for f in filters())
 
 
-def make_filter(thing: Union[str, Filter]) -> Filter:
+def make_filter(thing: str | Filter) -> Filter:
     if isinstance(thing, str):
         rc = re.compile(thing)
         def filter_(u: str) -> bool:

promnesia/kjson.py

@@ -3,19 +3,19 @@ Some experimental ideas on JSON processing.
 This is a bit overengineered and I admit it!
 I'll make it more readable, but in the meantime feel free to open an issue if you're confused about something.
 """
+from __future__ import annotations
 
-from typing import Any, Dict, List, Union, Tuple, cast
+from typing import Any, Union, cast
 
-
-JDict = Dict[str, Any] # TODO not sure if we can do recursive..
-JList = List[Any]
+JDict = dict[str, Any] # TODO not sure if we can do recursive..
+JList = list[Any]
 JPrim = Union[str, int, float] # , type(None)]
 
 Json = Union[JDict, JList, JPrim]
 
-JPathPart = Tuple[Json, Union[str, int]]
+JPathPart = tuple[Json, Union[str, int]]
 
-JPath = Tuple[JPathPart, ...]
+JPath = tuple[JPathPart, ...]
 
 
 class JsonProcessor:
@@ -36,7 +36,7 @@ class JsonProcessor:
         if res is self.SKIP:
             return
         for k, v in js.items():
-            path = cast(JPath, jp + ((js, k), ))
+            path = cast(JPath, jp + ((js, k), ))  # noqa: RUF005
             self._do(v, path)
 
     def do_list(self, js: JList, jp: JPath) -> None:
@@ -45,7 +45,7 @@ class JsonProcessor:
         if res is self.SKIP:
             return
         for i, x in enumerate(js):
-            path = cast(JPath, jp + ((js, i), ))
+            path = cast(JPath, jp + ((js, i), ))  # noqa: RUF005
             self._do(x, path)
 
     def _do(self, js: Json, path: JPath) -> None:
@@ -65,7 +65,7 @@ class JsonProcessor:
         self._do(js, path)
 
     @classmethod
-    def kpath(cls, path: JPath) -> Tuple[JPathPart, ...]:
+    def kpath(cls, path: JPath) -> tuple[JPathPart, ...]:
         return tuple(x[1] for x in path) # type: ignore
 
 # TODO path is a sequence of jsons and keys?
@@ -73,9 +73,10 @@ class JsonProcessor:
 def test_json_processor():
     handled = []
     class Proc(JsonProcessor):
-        def handle_dict(self, value: JDict, path):
+        def handle_dict(self, value: JDict, path):  # noqa: ARG002
             if 'skipme' in self.kpath(path):  # type: ignore[comparison-overlap]
                 return JsonProcessor.SKIP
+            return None
 
         def handle_str(self, value: str, path):
             if 'http' in value:

promnesia/logging.py

@@ -29,9 +29,9 @@ def test() -> None:
 
 
 import logging
-from typing import Union, Optional, cast
 import os
 import warnings
+from typing import Optional, Union, cast
 
 Level = int
 LevelIsh = Optional[Union[Level, str]]

promnesia/misc/install_server.py

@@ -1,15 +1,12 @@
-#!/usr/bin/env python3
 from __future__ import annotations
 
 import argparse
 import os
+import platform
 import sys
 import time
 from pathlib import Path
-import platform
-import shutil
 from subprocess import check_call, run
-from typing import List
 
 SYSTEM = platform.system()
 UNSUPPORTED_SYSTEM = RuntimeError(f'Platform {SYSTEM} is not supported yet!')
@@ -59,7 +56,7 @@ def systemd(*args: str | Path, method=check_call) -> None:
     ])
 
 
-def install_systemd(name: str, out: Path, launcher: str, largs: List[str]) -> None:
+def install_systemd(name: str, out: Path, launcher: str, largs: list[str]) -> None:
     unit_name = name
 
     import shlex
@@ -81,7 +78,7 @@ def install_systemd(name: str, out: Path, launcher: str, largs: List[str]) -> No
         raise e
 
 
-def install_launchd(name: str, out: Path, launcher: str, largs: List[str]) -> None:
+def install_launchd(name: str, out: Path, launcher: str, largs: list[str]) -> None:
     service_name = name
     arguments = '\n'.join(f'<string>{a}</string>' for a in [launcher, *largs])
     out.write_text(LAUNCHD_TEMPLATE.format(
@@ -116,14 +113,16 @@ def install(args: argparse.Namespace) -> None:
     print(f"Writing launch script to {out}", file=sys.stderr)
 
     # ugh. we want to know whether we're invoked 'properly' as an executable or ad-hoc via scripts/promnesia
+    extra_exe: list[str] = []
     if os.environ.get('DIRTY_RUN') is not None:
         launcher = str(root() / 'scripts/promnesia')
     else:
-        exe = shutil.which('promnesia'); assert exe is not None
-        launcher = exe # older systemd wants absolute paths..
+        launcher = sys.executable
+        extra_exe = ['-m', 'promnesia']
 
     db = args.db
     largs = [
+        *extra_exe,
         'serve',
         *([] if db is None else ['--db', str(db)]),
         '--timezone', args.timezone,

promnesia/server.py

@@ -1,35 +1,45 @@
-#!/usr/bin/python3
 from __future__ import annotations
 
 import argparse
-from dataclasses import dataclass
-from datetime import timedelta
-from functools import lru_cache
 import importlib.metadata
 import json
 import logging
 import os
+from dataclasses import dataclass
+from datetime import timedelta
+from functools import lru_cache
 from pathlib import Path
-from typing import List, NamedTuple, Dict, Optional, Any, Tuple, Protocol
-
+from typing import Any, NamedTuple, Optional, Protocol
 
+import fastapi
 import pytz
 from pytz import BaseTzInfo
-
-import fastapi
-
-from sqlalchemy import literal, between, or_, and_, exc, select
-from sqlalchemy import Column, Table, func, types
-from sqlalchemy.sql.elements import ColumnElement
+from sqlalchemy import (
+    Column,
+    Table,
+    and_,
+    between,
+    exc,
+    func,
+    literal,
+    or_,
+    select,
+    types,
+)
 from sqlalchemy.sql import text
+from sqlalchemy.sql.elements import ColumnElement
 
-
-from .common import PathWithMtime, DbVisit, Url, setup_logger, default_output_dir, get_system_tz
 from .cannon import canonify
+from .common import (
+    DbVisit,
+    PathWithMtime,
+    default_output_dir,
+    get_system_tz,
+    setup_logger,
+)
 from .database.load import DbStuff, get_db_stuff, row_to_db_visit
 
-
-Json = Dict[str, Any]
+Json = dict[str, Any]
 
 app = fastapi.FastAPI()
 
@@ -65,7 +75,7 @@ class ServerConfig(NamedTuple):
         })
 
     @classmethod
-    def from_str(cls, cfgs: str) -> 'ServerConfig':
+    def from_str(cls, cfgs: str) -> ServerConfig:
         d = json.loads(cfgs)
         return cls(
             db      =Path         (d['db']),
@@ -111,7 +121,7 @@ def as_json(v: DbVisit) -> Json:
     }
 
 
-def get_db_path(check: bool=True) -> Path:
+def get_db_path(*, check: bool=True) -> Path:
     db = EnvConfig.get().db
     if check:
         assert db.exists(), db
@@ -125,7 +135,7 @@ def _get_stuff(db_path: PathWithMtime) -> DbStuff:
     return get_db_stuff(db_path=db_path.path)
 
 
-def get_stuff(db_path: Optional[Path]=None) -> DbStuff: # TODO better name
+def get_stuff(db_path: Path | None=None) -> DbStuff: # TODO better name
     # ok, it will always load from the same db file; but intermediate would be kinda an optional dump.
     if db_path is None:
         db_path = get_db_path()
@@ -136,7 +146,7 @@ def db_stats(db_path: Path) -> Json:
     engine, table = get_stuff(db_path)
     query = select(func.count()).select_from(table)
     with engine.connect() as conn:
-        total = list(conn.execute(query))[0][0]
+        [(total,)] = conn.execute(query)
     return {
         'total_visits': total,
     }
@@ -172,17 +182,17 @@ def search_common(url: str, where: Where) -> VisitsResponse:
     with engine.connect() as conn:
         try:
             # TODO make more defensive here
-            visits: List[DbVisit] = [row_to_db_visit(row) for row in conn.execute(query)]
+            visits: list[DbVisit] = [row_to_db_visit(row) for row in conn.execute(query)]
         except exc.OperationalError as e:
             if getattr(e, 'msg', None) == 'no such table: visits':
-                logger.warn('you may have to run indexer first!')
+                logger.warning('you may have to run indexer first!')
                 #result['visits'] = [{an error with a msg}] # TODO
                 #return result
             raise
 
     logger.debug('got %d visits from db', len(visits))
 
-    vlist: List[DbVisit] = []
+    vlist: list[DbVisit] = []
     for vis in visits:
         dt = vis.dt
         if dt.tzinfo is None: # FIXME need this for /visits endpoint as well?
@@ -225,7 +235,7 @@ def status() -> Json:
         logger.exception(e)
         stats = {'ERROR': str(e)}
 
-    version: Optional[str]
+    version: str | None
     try:
         version = get_version()
     except Exception as e:
@@ -299,7 +309,7 @@ def search_around(request: SearchAroundRequest) -> VisitsResponse:
 
     return search_common(
         url='http://dummy.org', # NOTE: not used in the where query (below).. perhaps need to get rid of this
-        where=lambda table, url: between(
+        where=lambda table, url: between(  # noqa: ARG005
             func.strftime(
                 '%s', # NOTE: it's tz aware, e.g. would distinguish +05:00 vs -03:00
                 # this is a bit fragile, relies on cachew internal timestamp format, e.g.
@@ -322,25 +332,26 @@ def search_around(request: SearchAroundRequest) -> VisitsResponse:
 _NO_VERSION = (0, 11, 14)
 _LATEST = (9999, 9999, 9999)
 
-def as_version(version: str) -> Tuple[int, int, int]:
+def as_version(version: str) -> tuple[int, int, int]:
     if version == '':
         return _NO_VERSION
     try:
         [v1, v2, v3] = map(int, version.split('.'))
-        return (v1, v2, v3)
     except Exception as e:
         logger = get_logger()
         logger.error('error while parsing version %s', version)
         logger.exception(e)
         return _LATEST
+    else:
+        return (v1, v2, v3)
 
 
 @dataclass
 class VisitedRequest:
-    urls: List[str]
+    urls: list[str]
     client_version: str = ''
 
-VisitedResponse = List[Optional[Json]]
+VisitedResponse = list[Optional[Json]]
 
 @app.get ('/visited', response_model=VisitedResponse)
 @app.post('/visited', response_model=VisitedResponse)
@@ -355,7 +366,7 @@ def visited(request: VisitedRequest) -> VisitedResponse:
     version = as_version(client_version)
 
     nurls = [canonify(u) for u in urls]
-    snurls = list(sorted(set(nurls)))
+    snurls = sorted(set(nurls))
 
     if len(snurls) == 0:
         return []
@@ -388,7 +399,7 @@ SELECT queried, visits.*
     # brings down large queries to 50ms...
     with engine.connect() as conn:
         res = list(conn.execute(query))
-        present: Dict[str, Any] = {row[0]: row_to_db_visit(row[1:]) for row in res}
+        present: dict[str, Any] = {row[0]: row_to_db_visit(row[1:]) for row in res}
     results = []
     for nu in nurls:
         r = present.get(nu, None)