promnesia 1.3.20241021__py3-none-any.whl → 1.4.20250909__py3-none-any.whl

promnesia/extract.py

@@ -25,18 +25,17 @@ DEFAULT_FILTERS = (
     r'^about:',
     r'^blob:',
     r'^view-source:',
-
     r'^content:',
 )
 
 
 # TODO maybe move these to configs?
-@lru_cache(1) #meh, not sure what would happen under tests?
+@lru_cache(1)  # meh, not sure what would happen under tests?
 def filters() -> Sequence[Filter]:
     from . import config
 
     flt = list(DEFAULT_FILTERS)
-    if config.has(): # meeeh...
+    if config.has():  # meeeh...
         cfg = config.get()
         flt.extend(cfg.FILTERS)
     return tuple(make_filter(f) for f in flt)
@@ -67,7 +66,7 @@ def extract_visits(source: Source, *, src: SourceName) -> Iterable[Res[DbVisit]]
                 yield p
                 continue
 
-            if p in handled: # no need to emit duplicates
+            if p in handled:  # no need to emit duplicates
                 continue
             handled.add(p)
 
@@ -77,7 +76,6 @@ def extract_visits(source: Source, *, src: SourceName) -> Iterable[Res[DbVisit]]
         logger.exception(e)
         yield e
 
-
     logger.info('extracting via %s: got %d visits', source.description, len(handled))
 
 
@@ -99,8 +97,10 @@ def filtered(url: Url) -> bool:
 def make_filter(thing: str | Filter) -> Filter:
     if isinstance(thing, str):
         rc = re.compile(thing)
+
         def filter_(u: str) -> bool:
             return rc.search(u) is not None
+
         return filter_
-    else: # must be predicate
+    else:  # must be predicate
         return thing

promnesia/logging.py

@@ -3,20 +3,25 @@
 Default logger is a bit meh, see 'test'/run this file for a demo
 '''
 
+
 def test() -> None:
     import logging
     import sys
-    from typing import Callable
+    from collections.abc import Callable
 
     M: Callable[[str], None] = lambda s: print(s, file=sys.stderr)
 
     M("   Logging module's defaults are not great...'")
     l = logging.getLogger('test_logger')
-    l.error("For example, this should be logged as error. But it's not even formatted properly, doesn't have logger name or level")
+    l.error(
+        "For example, this should be logged as error. But it's not even formatted properly, doesn't have logger name or level"
+    )
 
     M("   The reason is that you need to remember to call basicConfig() first")
     logging.basicConfig()
-    l.error("OK, this is better. But the default format kinda sucks, I prefer having timestamps and the file/line number")
+    l.error(
+        "OK, this is better. But the default format kinda sucks, I prefer having timestamps and the file/line number"
+    )
 
     M("")
     M("    With LazyLogger you get a reasonable logging format, colours and other neat things")
@@ -31,10 +36,10 @@ def test() -> None:
 import logging
 import os
 import warnings
-from typing import Optional, Union, cast
+from typing import cast
 
 Level = int
-LevelIsh = Optional[Union[Level, str]]
+LevelIsh = Level | str | None
 
 
 def mklevel(level: LevelIsh) -> Level:
@@ -50,18 +55,22 @@ def mklevel(level: LevelIsh) -> Level:
 
 
 FORMAT = '{start}[%(levelname)-7s %(asctime)s %(name)s %(filename)s:%(lineno)d]{end} %(message)s'
+# fmt: off
 FORMAT_COLOR   = FORMAT.format(start='%(color)s', end='%(end_color)s')
-FORMAT_NOCOLOR = FORMAT.format(start='', end='')
+FORMAT_NOCOLOR = FORMAT.format(start=''         , end='')
+# fmt: on
 DATEFMT = '%Y-%m-%d %H:%M:%S'
 
-COLLAPSE_DEBUG_LOGS = os.environ.get('COLLAPSE_DEBUG_LOGS', False)
+COLLAPSE_DEBUG_LOGS = os.environ.get('COLLAPSE_DEBUG_LOGS', False)  # noqa: PLW1508
 
 _init_done = 'lazylogger_init_done'
 
+
 def setup_logger(logger: logging.Logger, level: LevelIsh) -> None:
     lvl = mklevel(level)
     try:
-        import logzero  # type: ignore[import-not-found]
+        import logzero  # type: ignore[import-not-found,import-untyped,unused-ignore]
+
         formatter = logzero.LogFormatter(
             fmt=FORMAT_COLOR,
             datefmt=DATEFMT,
@@ -73,7 +82,7 @@ def setup_logger(logger: logging.Logger, level: LevelIsh) -> None:
         use_logzero = False
 
     logger.addFilter(AddExceptionTraceback())
-    if use_logzero and not COLLAPSE_DEBUG_LOGS: # all set, nothing to do
+    if use_logzero and not COLLAPSE_DEBUG_LOGS:  # all set, nothing to do
         # 'simple' setup
         logzero.setup_logger(logger.name, level=lvl, formatter=formatter)  # type: ignore[possibly-undefined]
         return
@@ -91,16 +100,17 @@ class LazyLogger(logging.Logger):
         logger = logging.getLogger(name)
 
         # this is called prior to all _log calls so makes sense to do it here?
-        def isEnabledFor_lazyinit(*args, logger=logger, orig=logger.isEnabledFor, **kwargs) -> bool:
+        def isEnabledFor_lazyinit(*args, logger: logging.Logger = logger, orig=logger.isEnabledFor, **kwargs) -> bool:
             if not getattr(logger, _init_done, False):  # init once, if necessary
                 setup_logger(logger, level=level)
                 setattr(logger, _init_done, True)
-                logger.isEnabledFor = orig # restore the callback
-            return orig(*args, **kwargs)
+                # restore the callback
+                logger.isEnabledFor = orig  # type: ignore[method-assign]  # ty: ignore[invalid-assignment]
+            return orig(*args, **kwargs)  # ty: ignore[missing-argument]
 
         # oh god.. otherwise might go into an inf loop
         if not hasattr(logger, _init_done):
-            setattr(logger, _init_done, False) # will setup on the first call
+            setattr(logger, _init_done, False)  # will setup on the first call
             logger.isEnabledFor = isEnabledFor_lazyinit  # type: ignore[method-assign]
         return cast(LazyLogger, logger)
 
@@ -129,6 +139,7 @@ class CollapseDebugHandler(logging.StreamHandler):
     Collapses subsequent debug log lines and redraws on the same line.
     Hopefully this gives both a sense of progress and doesn't clutter the terminal as much?
     '''
+
     last = False
 
     def emit(self, record: logging.LogRecord) -> None:
@@ -137,12 +148,13 @@ class CollapseDebugHandler(logging.StreamHandler):
             cur = record.levelno == logging.DEBUG and '\n' not in msg
             if cur:
                 if self.last:
-                    self.stream.write('\033[K' + '\r') # clear line + return carriage
+                    self.stream.write('\033[K' + '\r')  # clear line + return carriage
             else:
                 if self.last:
-                    self.stream.write('\n') # clean up after the last debug line
+                    self.stream.write('\n')  # clean up after the last debug line
             self.last = cur
             import os
+
             columns, _ = os.get_terminal_size(0)
             # ugh. the columns thing is meh. dunno I guess ultimately need curses for that
             # TODO also would be cool to have a terminal post-processor? kinda like tail but aware of logging keywords (INFO/DEBUG/etc)

promnesia/misc/install_server.py

@@ -51,51 +51,57 @@ LAUNCHD_TEMPLATE = '''
 
 
 def systemd(*args: str | Path, method=check_call) -> None:
-    method([
-        'systemctl', '--no-pager', '--user', *args,
-    ])
+    method(['systemctl', '--no-pager', '--user', *args])
 
 
 def install_systemd(name: str, out: Path, launcher: str, largs: list[str]) -> None:
     unit_name = name
 
     import shlex
+
     extra_args = ' '.join(shlex.quote(str(a)) for a in largs)
 
-    out.write_text(SYSTEMD_TEMPLATE.format(
-        launcher=launcher,
-        extra_args=extra_args,
-    ))
+    out.write_text(
+        SYSTEMD_TEMPLATE.format(
+            launcher=launcher,
+            extra_args=extra_args,
+        )
+    )
 
     try:
-        systemd('stop' , unit_name, method=run) # ignore errors here if it wasn't running in the first place
+        systemd('stop', unit_name, method=run)  # ignore errors here if it wasn't running in the first place
         systemd('daemon-reload')
         systemd('enable', unit_name)
-        systemd('start' , unit_name)
+        systemd('start', unit_name)
         systemd('status', unit_name)
     except Exception as e:
-        print(f"Something has gone wrong... you might want to use 'journalctl --user -u {unit_name}' to investigate", file=sys.stderr)
+        print(
+            f"Something has gone wrong... you might want to use 'journalctl --user -u {unit_name}' to investigate",
+            file=sys.stderr,
+        )
         raise e
 
 
 def install_launchd(name: str, out: Path, launcher: str, largs: list[str]) -> None:
     service_name = name
     arguments = '\n'.join(f'<string>{a}</string>' for a in [launcher, *largs])
-    out.write_text(LAUNCHD_TEMPLATE.format(
-        service_name=service_name,
-        arguments=arguments,
-    ))
+    out.write_text(
+        LAUNCHD_TEMPLATE.format(
+            service_name=service_name,
+            arguments=arguments,
+        )
+    )
     cmd = ['launchctl', 'load', '-w', str(out)]
     print('Running: ' + ' '.join(cmd), file=sys.stderr)
     check_call(cmd)
 
-    time.sleep(1) # to give it some time? not sure if necessary
+    time.sleep(1)  # to give it some time? not sure if necessary
     check_call(f'launchctl list | grep {name}', shell=True)
 
 
 def install(args: argparse.Namespace) -> None:
     name = args.name
-    # todo use appdirs for config dir detection
+    # todo use platformdirs for config dir detection
     if SYSTEM == 'Linux':
         # Check for existence of systemd
         # https://www.freedesktop.org/software/systemd/man/sd_booted.html
@@ -105,7 +111,7 @@ def install(args: argparse.Namespace) -> None:
         if Path(name).suffix != suf:
             name = name + suf
         out = Path(f'~/.config/systemd/user/{name}')
-    elif SYSTEM == 'Darwin': # osx
+    elif SYSTEM == 'Darwin':  # osx
         out = Path(f'~/Library/LaunchAgents/{name}.plist')
     else:
         raise UNSUPPORTED_SYSTEM
@@ -128,9 +134,9 @@ def install(args: argparse.Namespace) -> None:
         '--timezone', args.timezone,
         '--host', args.host,
         '--port', args.port,
-    ]
+    ]  # fmt: skip
 
-    out.parent.mkdir(parents=True, exist_ok=True) # sometimes systemd dir doesn't exist
+    out.parent.mkdir(parents=True, exist_ok=True)  # sometimes systemd dir doesn't exist
     if SYSTEM == 'Linux':
         install_systemd(name=name, out=out, launcher=launcher, largs=largs)
     elif SYSTEM == 'Darwin':

promnesia/server.py

@@ -9,11 +9,10 @@ from dataclasses import dataclass
 from datetime import timedelta
 from functools import lru_cache
 from pathlib import Path
-from typing import Any, NamedTuple, Optional, Protocol
+from typing import Any, NamedTuple, Protocol
+from zoneinfo import ZoneInfo
 
 import fastapi
-import pytz
-from pytz import BaseTzInfo
 from sqlalchemy import (
     Column,
     Table,
@@ -43,6 +42,7 @@ Json = dict[str, Any]
 
 app = fastapi.FastAPI()
 
+
 # meh. need this since I don't have hooks in hug to initialize logging properly..
 @lru_cache(1)
 def get_logger() -> logging.Logger:
@@ -61,26 +61,26 @@ def get_logger() -> logging.Logger:
 
 
 def get_version() -> str:
+    assert __package__ is not None  # make type checker happy
     return importlib.metadata.version(__package__)
 
 
 class ServerConfig(NamedTuple):
     db: Path
-    timezone: BaseTzInfo
+    timezone: ZoneInfo
 
     def as_str(self) -> str:
-        return json.dumps({
-            'timezone': self.timezone.zone,
-            'db'      : str(self.db),
-        })
+        return json.dumps(
+            {
+                'timezone': self.timezone.key,
+                'db': str(self.db),
+            }
+        )
 
     @classmethod
     def from_str(cls, cfgs: str) -> ServerConfig:
         d = json.loads(cfgs)
-        return cls(
-            db      =Path         (d['db']),
-            timezone=pytz.timezone(d['timezone'])
-        )
+        return cls(db=Path(d['db']), timezone=ZoneInfo(d['timezone']))
 
 
 class EnvConfig:
@@ -98,8 +98,10 @@ class EnvConfig:
     def set(cfg: ServerConfig) -> None:
         os.environ[EnvConfig.KEY] = cfg.as_str()
 
+
 # todo how to return exception in error?
 
+
 def as_json(v: DbVisit) -> Json:
     # yep, this is NOT %Y-%m-%d as is seems to be the only format with timezone that Date.parse in JS accepts. Just forget it.
     dts = v.dt.strftime('%d %b %Y %H:%M:%S %z')
@@ -114,14 +116,14 @@ def as_json(v: DbVisit) -> Json:
         'duration': v.duration,
         'locator': {
             'title': loc.title,
-            'href' : loc.href,
+            'href': loc.href,
         },
-        'original_url'  : v.orig_url,
+        'original_url': v.orig_url,
         'normalised_url': v.norm_url,
     }
 
 
-def get_db_path(*, check: bool=True) -> Path:
+def get_db_path(*, check: bool = True) -> Path:
     db = EnvConfig.get().db
     if check:
         assert db.exists(), db
@@ -135,7 +137,7 @@ def _get_stuff(db_path: PathWithMtime) -> DbStuff:
     return get_db_stuff(db_path=db_path.path)
 
 
-def get_stuff(db_path: Path | None=None) -> DbStuff: # TODO better name
+def get_stuff(db_path: Path | None = None) -> DbStuff:  # TODO better name
     # ok, it will always load from the same db file; but intermediate would be kinda an optional dump.
     if db_path is None:
         db_path = get_db_path()
@@ -153,8 +155,8 @@ def db_stats(db_path: Path) -> Json:
 
 
 class Where(Protocol):
-    def __call__(self, table: Table, url: str) -> ColumnElement[bool]:
-        ...
+    def __call__(self, table: Table, url: str) -> ColumnElement[bool]: ...
+
 
 @dataclass
 class VisitsResponse:
@@ -186,8 +188,8 @@ def search_common(url: str, where: Where) -> VisitsResponse:
         except exc.OperationalError as e:
             if getattr(e, 'msg', None) == 'no such table: visits':
                 logger.warning('you may have to run indexer first!')
-                #result['visits'] = [{an error with a msg}] # TODO
-                #return result
+                # result['visits'] = [{an error with a msg}] # TODO
+                # return result
             raise
 
     logger.debug('got %d visits from db', len(visits))
@@ -195,9 +197,8 @@ def search_common(url: str, where: Where) -> VisitsResponse:
     vlist: list[DbVisit] = []
     for vis in visits:
         dt = vis.dt
-        if dt.tzinfo is None: # FIXME need this for /visits endpoint as well?
-            tz = config.timezone
-            dt = tz.localize(dt)
+        if dt.tzinfo is None:  # FIXME need this for /visits endpoint as well?
+            dt = dt.replace(tzinfo=config.timezone)
             vis = vis._replace(dt=dt)
         vlist.append(vis)
 
@@ -212,8 +213,8 @@ def search_common(url: str, where: Where) -> VisitsResponse:
 
 # TODO hmm, seems that the extension is using post for all requests??
 # perhasp should switch to get for most endpoint
-@app.get ('/status', response_model=Json)
-@app.post('/status', response_model=Json)
+@app.get ('/status', response_model=Json)  # fmt: skip
+@app.post('/status', response_model=Json)  # fmt: skip
 def status() -> Json:
     '''
     Ideally, status will always respond, regardless the internal state of the backend?
@@ -246,15 +247,16 @@ def status() -> Json:
         'version': version,
         'db'     : db_path,
         'stats'  : stats,
-    }
+    }  # fmt: skip
 
 
 @dataclass
 class VisitsRequest:
     url: str
 
-@app.get ('/visits', response_model=VisitsResponse)
-@app.post('/visits', response_model=VisitsResponse)
+
+@app.get ('/visits', response_model=VisitsResponse)  # fmt: skip
+@app.post('/visits', response_model=VisitsResponse)  # fmt: skip
 def visits(request: VisitsRequest) -> VisitsResponse:
     url = request.url
     get_logger().info('/visited %s', url)
@@ -265,7 +267,7 @@ def visits(request: VisitsRequest) -> VisitsResponse:
             # exact match
             table.c.norm_url == url,
             # + child visits, but only 'interesting' ones
-            and_(table.c.context != None, table.c.norm_url.startswith(url, autoescape=True))  # noqa: E711
+            and_(table.c.context != None, table.c.norm_url.startswith(url, autoescape=True)),  # noqa: E711
         ),
     )
 
@@ -274,11 +276,13 @@ def visits(request: VisitsRequest) -> VisitsResponse:
 class SearchRequest:
     url: str
 
-@app.get ('/search', response_model=VisitsResponse)
-@app.post('/search', response_model=VisitsResponse)
+
+@app.get ('/search', response_model=VisitsResponse)  # fmt: skip
+@app.post('/search', response_model=VisitsResponse)  # fmt: skip
 def search(request: SearchRequest) -> VisitsResponse:
     url = request.url
     get_logger().info('/search %s', url)
+    # fmt: off
     return search_common(
         url=url,
         where=lambda table, url: or_(
@@ -289,49 +293,54 @@ def search(request: SearchRequest) -> VisitsResponse:
             table.c.locator_title.contains(url, autoescape=True),
         ),
     )
+    # fmt: on
 
 
 @dataclass
 class SearchAroundRequest:
     timestamp: float
 
-@app.get ('/search_around', response_model=VisitsResponse)
-@app.post('/search_around', response_model=VisitsResponse)
+
+@app.get ('/search_around', response_model=VisitsResponse)  # fmt: skip
+@app.post('/search_around', response_model=VisitsResponse)  # fmt: skip
 def search_around(request: SearchAroundRequest) -> VisitsResponse:
     timestamp = request.timestamp
     get_logger().info('/search_around %s', timestamp)
-    utc_timestamp = timestamp # old 'timestamp' name is legacy
+    utc_timestamp = timestamp  # old 'timestamp' name is legacy
 
     # TODO meh. use count/pagination instead?
-    delta_back  = timedelta(hours=3  ).total_seconds()
+    delta_back = timedelta(hours=3).total_seconds()
     delta_front = timedelta(minutes=2).total_seconds()
     # TODO not sure about delta_front.. but it also serves as quick hack to accommodate for all the truncations etc
 
     return search_common(
-        url='http://dummy.org', # NOTE: not used in the where query (below).. perhaps need to get rid of this
+        url='http://dummy.org',  # NOTE: not used in the where query (below).. perhaps need to get rid of this
         where=lambda table, url: between(  # noqa: ARG005
             func.strftime(
-                '%s', # NOTE: it's tz aware, e.g. would distinguish +05:00 vs -03:00
+                '%s',  # NOTE: it's tz aware, e.g. would distinguish +05:00 vs -03:00
                 # this is a bit fragile, relies on cachew internal timestamp format, e.g.
                 # 2020-11-10T06:13:03.196376+00:00 Europe/London
                 func.substr(
                     table.c.dt,
-                    1, # substr is 1-indexed
+                    1,  # substr is 1-indexed
                     # instr finds the first match, but if not found it defaults to 0.. which we hack by concatting with ' '
                     func.instr(func.cast(table.c.dt, types.Unicode).op('||')(' '), ' ') - 1,
                     # for fucks sake.. seems that cast is necessary otherwise it tries to treat ' ' as datetime???
-                )
-            ) - literal(utc_timestamp),
+                ),
+            )
+            - literal(utc_timestamp),
             literal(-delta_back),
             literal(delta_front),
         ),
     )
 
+
 # before 0.11.14 (including), extension didn't share the version
 # so if it's not shared, assume that version
 _NO_VERSION = (0, 11, 14)
 _LATEST = (9999, 9999, 9999)
 
+
 def as_version(version: str) -> tuple[int, int, int]:
     if version == '':
         return _NO_VERSION
@@ -351,10 +360,12 @@ class VisitedRequest:
     urls: list[str]
     client_version: str = ''
 
-VisitedResponse = list[Optional[Json]]
 
-@app.get ('/visited', response_model=VisitedResponse)
-@app.post('/visited', response_model=VisitedResponse)
+VisitedResponse = list[Json | None]
+
+
+@app.get ('/visited', response_model=VisitedResponse)  # fmt: skip
+@app.post('/visited', response_model=VisitedResponse)  # fmt: skip
 def visited(request: VisitedRequest) -> VisitedResponse:
     # TODO instead switch logging to fastapi
     urls = request.urls
@@ -363,7 +374,7 @@ def visited(request: VisitedRequest) -> VisitedResponse:
     logger = get_logger()
     logger.info('/visited %s %s', urls, client_version)
 
-    version = as_version(client_version)
+    _version = as_version(client_version)  # todo use it?
 
     nurls = [canonify(u) for u in urls]
     snurls = sorted(set(nurls))
@@ -376,10 +387,11 @@ def visited(request: VisitedRequest) -> VisitedResponse:
     # sqlalchemy doesn't seem to support SELECT FROM (VALUES (...)) in its api
     # also doesn't support array binding...
     # https://stackoverflow.com/questions/13190392/how-can-i-bind-a-list-to-a-parameter-in-a-custom-query-in-sqlalchemy
-    bstring = ','.join(f'(:b{i})'   for i, _ in enumerate(snurls))
-    bdict = {            f'b{i}': v for i, v in enumerate(snurls)}
+    bstring = ','.join(f'(:b{i})'   for i, _ in enumerate(snurls))  # fmt: skip
+    bdict = {            f'b{i}': v for i, v in enumerate(snurls)}  # fmt: skip
     # TODO hopefully, visits.* thing only returns one visit??
-    query = text(f"""
+    query = (
+        text(f"""
 WITH cte(queried) AS (SELECT * FROM (values {bstring}))
 SELECT queried, visits.*
     FROM cte JOIN visits
@@ -389,9 +401,12 @@ SELECT queried, visits.*
     but somehow DESC is the one that actually works..
 */
     ORDER BY visits.context IS NULL DESC
-    """).bindparams(**bdict).columns(
-        Column('match', types.Unicode),
-        *table.columns,
+    """)
+        .bindparams(**bdict)
+        .columns(
+            Column('match', types.Unicode),
+            *table.columns,
+        )
     )
     # TODO might be very beneficial for performance to have an intermediate table
     # SELECT visits.* FROM visits GROUP BY visits.norm_url ORDER BY visits.context IS NULL DESC
@@ -402,7 +417,7 @@ SELECT queried, visits.*
         present: dict[str, Any] = {row[0]: row_to_db_visit(row[1:]) for row in res}
     results = []
     for nu in nurls:
-        r = present.get(nu, None)
+        r = present.get(nu)
         results.append(None if r is None else as_json(r))
 
     # no need for it anymore, extension has been updated since
@@ -422,6 +437,7 @@ def _run(*, host: str, port: str, quiet: bool, config: ServerConfig) -> None:
     EnvConfig.set(config)
 
     import uvicorn
+
     uvicorn.run('promnesia.server:app', host=host, port=int(port), log_level='debug')
 
 
@@ -433,7 +449,7 @@ def run(args: argparse.Namespace) -> None:
         config=ServerConfig(
             db=args.db,
             timezone=args.timezone,
-        )
+        ),
     )
 
 
@@ -475,7 +491,7 @@ def setup_parser(p: argparse.ArgumentParser) -> None:
 
     p.add_argument(
         '--timezone',
-        type=pytz.timezone,
+        type=ZoneInfo,
         default=get_system_tz(),
         help='Fallback timezone, defaults to the system timezone if not specified',
     )