promnesia 1.2.20240810__tar.gz → 1.3.20241021__tar.gz

{promnesia-1.2.20240810 → promnesia-1.3.20241021}/.github/workflows/main.yml

@@ -24,22 +24,23 @@ jobs:
       fail-fast: false
       matrix:
         platform: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
         exclude: [
             # windows runners are pretty scarce, so let's only run lowest and highest python version
-            {platform: windows-latest, python-version: '3.9' },
             {platform: windows-latest, python-version: '3.10'},
             {platform: windows-latest, python-version: '3.11'},
+            {platform: windows-latest, python-version: '3.12'},
 
             # same, macos is a bit too slow and ubuntu covers python quirks well
-            {platform: macos-latest  , python-version: '3.9' },
             {platform: macos-latest  , python-version: '3.10'},
             {platform: macos-latest  , python-version: '3.11'},
+            {platform: macos-latest  , python-version: '3.12'},
         ]
 
     runs-on: ${{ matrix.platform }}
 
-    #continue-on-error: ${{ matrix.platform == 'windows-latest' }}
+    # useful for 'optional' pipelines
+    # continue-on-error: ${{ matrix.platform == 'windows-latest' }}
 
     steps:
     # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation
@@ -63,11 +64,13 @@ jobs:
     - if: matrix.platform == 'ubuntu-latest'  # no need to compute coverage for other platforms
       uses: actions/upload-artifact@v4
       with:
+        include-hidden-files: true
         name: .coverage.mypy-core_${{ matrix.platform }}_${{ matrix.python-version }}
         path: .coverage.mypy-core/
     - if: matrix.platform == 'ubuntu-latest'  # no need to compute coverage for other platforms
       uses: actions/upload-artifact@v4
       with:
+        include-hidden-files: true
         name: .coverage.mypy-misc_${{ matrix.platform }}_${{ matrix.python-version }}
         path: .coverage.mypy-misc/
 

{promnesia-1.2.20240810 → promnesia-1.3.20241021}/PKG-INFO

@@ -1,15 +1,16 @@
 Metadata-Version: 2.1
 Name: promnesia
-Version: 1.2.20240810
+Version: 1.3.20241021
 Summary: Enhancement of your browsing history
 Home-page: https://github.com/karlicoss/promnesia
 Author: Dmitrii Gerasimov
 Author-email: karlicoss@gmail.com
-Requires-Python: >=3.8
+Requires-Python: >=3.9
 License-File: LICENSE
 Requires-Dist: appdirs
 Requires-Dist: tzlocal
 Requires-Dist: more_itertools
+Requires-Dist: typing-extensions
 Requires-Dist: pytz
 Requires-Dist: sqlalchemy>=2.0
 Requires-Dist: urlextract

{promnesia-1.2.20240810 → promnesia-1.3.20241021}/mypy.ini

@@ -1,14 +1,13 @@
 [mypy]
-namespace_packages = True
 pretty = True
 show_error_context = True
-show_error_codes = True
 show_column_numbers = True
 show_error_end = True
+warn_redundant_casts = True
 warn_unused_ignores = True
 check_untyped_defs = True
-enable_error_code = possibly-undefined
 strict_equality = True
+enable_error_code = possibly-undefined
 
 # not sure why mypy started discovering it (since 0.800??)
 [mypy-hypothesis]

promnesia-1.3.20241021/ruff.toml

@@ -0,0 +1,147 @@
+target-version = "py39"  # NOTE: inferred from pyproject.toml if present
+
+lint.extend-select = [
+    "F",    # flakes rules -- default, but extend just in case
+    "E",    # pycodestyle  -- default, but extend just in case
+    "W",    # various warnings
+
+    "B",    # 'bugbear' set -- various possible bugs
+    "C4",   # flake8-comprehensions -- unnecessary list/map/dict calls
+    "COM",  # trailing commas
+    "EXE",  # various checks wrt executable files
+    "I",    # sort imports
+    "ICN",  # various import conventions
+    "FBT",  # detect use of boolean arguments
+    "FURB", # various rules
+    "PERF", # various potential performance speedups
+    "PD",   # pandas rules
+    "PIE",  # 'misc' lints
+    "PLC",  # pylint convention rules
+    "PLR",  # pylint refactor rules
+    "PLW",  # pylint warnings
+    "PT",   # pytest stuff
+    "PYI",  # various type hinting rules
+    "RET",  # early returns
+    "RUF",  # various ruff-specific rules
+    "TID",  # various imports suggestions
+    "TRY",  # various exception handling rules
+    "UP",   # detect deprecated python stdlib stuff
+    "FA",   # suggest using from __future__ import annotations
+    "PTH",  # pathlib migration
+    "ARG",  # unused argument checks
+    "A",    # builtin shadowing
+    "G",    # logging stuff
+    # "EM",  # TODO hmm could be helpful to prevent duplicate err msg in traceback.. but kinda annoying
+
+    # "ALL", # uncomment this to check for new rules!
+]
+
+lint.ignore = [
+    "D",     # annoying nags about docstrings
+    "N",     # pep naming
+    "TCH",   # type checking rules, mostly just suggests moving imports under TYPE_CHECKING
+    "S",     # bandit (security checks) -- tends to be not very useful, lots of nitpicks
+    "DTZ",   # datetimes checks -- complaining about missing tz and mostly false positives
+    "FIX",   # complains about fixmes/todos -- annoying
+    "TD",    # complains about todo formatting -- too annoying
+    "ANN",   # missing type annotations? seems way to strict though
+
+### too opinionated style checks
+    "E501",  # too long lines
+    "E702",  # Multiple statements on one line (semicolon)
+    "E731",  # assigning lambda instead of using def
+    "E741",  # Ambiguous variable name: `l`
+    "E742",  # Ambiguous class name: `O
+    "E401",  # Multiple imports on one line
+    "F403",  # import *` used; unable to detect undefined names
+###
+
+###
+    "E722",  # Do not use bare `except` ## Sometimes it's useful for defensive imports and that sort of thing..
+    "F811",  # Redefinition of unused  # this gets in the way of pytest fixtures (e.g. in cachew)
+
+## might be nice .. but later and I don't wanna make it strict
+    "E402",  # Module level import not at top of file
+
+### maybe consider these soon
+    # sometimes it's useful to give a variable a name even if we don't use it as a documentation
+    # on the other hand, often is a sign of error
+    "F841",  # Local variable `count` is assigned to but never used
+###
+
+    "RUF100",  # unused noqa -- handle later
+    "RUF012",  # mutable class attrs should be annotated with ClassVar... ugh pretty annoying for user configs
+
+### these are just nitpicky, we usually know better
+    "PLR0911",  # too many return statements
+    "PLR0912",  # too many branches
+    "PLR0913",  # too many function arguments
+    "PLR0915",  # too many statements
+    "PLR1714",  # consider merging multiple comparisons
+    "PLR2044",  # line with empty comment
+    "PLR5501",  # use elif instead of else if
+    "PLR2004",  # magic value in comparison -- super annoying in tests
+###
+    "PLR0402",  # import X.Y as Y -- TODO maybe consider enabling it, but double check
+
+    "B009",  # calling gettattr with constant attribute -- this is useful to convince mypy
+    "B010",  # same as above, but setattr
+    "B011",  # complains about assert False
+    "B017",  # pytest.raises(Exception)
+    "B023",  # seems to result in false positives?
+    "B028",  # suggest using explicit stacklevel? TODO double check later, but not sure it's useful
+
+    # complains about useless pass, but has sort of a false positive if the function has a docstring?
+    # this is common for click entrypoints (e.g. in __main__), so disable
+    "PIE790",
+
+    # a bit too annoying, offers to convert for loops to list comprehension
+    # , which may heart readability
+    "PERF401",
+
+    # suggests no using exception in for loops
+    # we do use this technique a lot, plus in 3.11 happy path exception handling is "zero-cost"
+    "PERF203",
+
+    "RET504", # unnecessary assignment before returning -- that can be useful for readability
+    "RET505", # unnecessary else after return -- can hurt readability
+
+    "PLW0603",  # global variable update.. we usually know why we are doing this
+    "PLW2901",  # for loop variable overwritten, usually this is intentional
+
+    "PT004",  # deprecated rule, will be removed later
+    "PT011",  # pytest raises should is too broad
+    "PT012",  # pytest raises should contain a single statement
+
+    "COM812",  # trailing comma missing -- mostly just being annoying with long multiline strings
+
+    "PD901",   # generic variable name df
+
+    "TRY003",  # suggests defining exception messages in exception class -- kinda annoying
+    "TRY004",  # prefer TypeError -- don't see the point
+    "TRY201",  # raise without specifying exception name -- sometimes hurts readability
+    "TRY400",  # TODO double check this, might be useful
+    "TRY401",  # redundant exception in logging.exception call? TODO double check, might result in excessive logging
+
+    "PGH",  # TODO force error code in mypy instead
+
+    "TID252",  # Prefer absolute imports over relative imports from parent modules
+
+    "UP038",  # suggests using | (union) in isisntance checks.. but it results in slower code
+
+    ## too annoying
+    "T20",     # just complains about prints and pprints
+    "Q",       # flake quotes, too annoying
+    "C90",     # some complexity checking
+    "G004",    # logging statement uses f string
+    "ERA001",  # commented out code
+    "SLF001",  # private member accessed
+    "BLE001",  # do not catch 'blind' Exception
+    "INP001",  # complains about implicit namespace packages
+    "SIM",     # some if statements crap
+    "RSE102",  # complains about missing parens in exceptions
+    ##
+
+    "ARG001",  # ugh, kinda annoying when using pytest fixtures
+    "RUF001", "RUF002", "RUF003",  # spams about non-latin characters that we do use for testing
+]

{promnesia-1.2.20240810 → promnesia-1.3.20241021}/setup.py

@@ -32,11 +32,12 @@ def main() -> None:
         author_email='karlicoss@gmail.com',
         description='Enhancement of your browsing history',
 
-        python_requires='>=3.8',
+        python_requires='>=3.9',
         install_requires=[
             'appdirs', # for portable user directories detection
             'tzlocal',
             'more_itertools',
+            'typing-extensions',
             'pytz',
             'sqlalchemy>=2.0',  # DB api
 

{promnesia-1.2.20240810 → promnesia-1.3.20241021}/src/promnesia/__init__.py

@@ -1,6 +1,17 @@
-from pathlib import Path
-from .common import PathIsh, Visit, Source, last, Loc, Results, DbVisit, Context, Res
-
 # add deprecation warning so eventually this may converted to a namespace package?
 import warnings
+
+from .common import (  # noqa: F401
+    Context,
+    DbVisit,
+    Loc,
+    PathIsh,
+    Res,
+    Results,
+    Source,
+    Visit,
+    last,
+)
+
+# TODO think again about it -- what are the pros and cons?
 warnings.warn("DEPRECATED! Please import directly from 'promnesia.common', e.g. 'from promnesia.common import Visit, Source, Results'", DeprecationWarning)

{promnesia-1.2.20240810 → promnesia-1.3.20241021}/src/promnesia/__main__.py

@@ -5,24 +5,34 @@ import ast
 import importlib
 import inspect
 import os
-from pathlib import Path
+import shlex
 import shutil
-from subprocess import run, check_call, Popen
 import sys
+from collections.abc import Iterable, Iterator, Sequence
+from pathlib import Path
+from subprocess import Popen, check_call, run
 from tempfile import TemporaryDirectory, gettempdir
-from typing import Callable, Sequence, Iterable, Iterator, Union
-
-
-from . import config
-from . import server
-from .misc import install_server
-from .common import Extractor, PathIsh, logger, get_tmpdir, DbVisit, Res
-from .common import Source, get_system_tz, user_config_file, default_config_path
+from typing import Callable
+
+from . import config, server
+from .common import (
+    DbVisit,
+    Extractor,
+    PathIsh,
+    Res,
+    Source,
+    default_config_path,
+    get_system_tz,
+    get_tmpdir,
+    logger,
+    user_config_file,
+)
 from .database.dump import visits_to_sqlite
 from .extract import extract_visits
+from .misc import install_server
 
 
-def iter_all_visits(sources_subset: Iterable[Union[str, int]]=()) -> Iterator[Res[DbVisit]]:
+def iter_all_visits(sources_subset: Iterable[str | int] = ()) -> Iterator[Res[DbVisit]]:
     cfg = config.get()
     output_dir = cfg.output_dir
     # not sure if belongs here??
@@ -74,7 +84,7 @@ def iter_all_visits(sources_subset: Iterable[Union[str, int]]=()) -> Iterator[Re
         logger.warning("unknown --sources: %s", ", ".join(repr(i) for i in sources_subset))
 
 
-def _do_index(dry: bool=False, sources_subset: Iterable[Union[str, int]]=(), overwrite_db: bool=False) -> Iterable[Exception]:
+def _do_index(*, dry: bool = False, sources_subset: Iterable[str | int] = (), overwrite_db: bool = False) -> Iterable[Exception]:
     # also keep & return errors for further display
     errors: list[Exception] = []
     def it() -> Iterable[Res[DbVisit]]:
@@ -98,9 +108,10 @@ def _do_index(dry: bool=False, sources_subset: Iterable[Union[str, int]]=(), ove
 
 def do_index(
     config_file: Path,
-    dry: bool=False,
-    sources_subset: Iterable[Union[str, int]]=(),
-    overwrite_db: bool=False,
+    *,
+    dry: bool = False,
+    sources_subset: Iterable[str | int] = (),
+    overwrite_db: bool = False,
 ) -> Sequence[Exception]:
     config.load_from(config_file) # meh.. should be cleaner
     try:
@@ -120,7 +131,8 @@ def demo_sources() -> dict[str, Callable[[], Extractor]]:
     def lazy(name: str) -> Callable[[], Extractor]:
         # helper to avoid failed imports etc, since people might be lacking necessary dependencies
         def inner() -> Extractor:
-            from . import sources
+            # TODO why this import??
+            from . import sources  # noqa: F401
             module = importlib.import_module(f'promnesia.sources.{name}')
             return getattr(module, 'index')
         return inner
@@ -145,7 +157,7 @@ def do_demo(
         config_file: Path | None,
         dry: bool=False,
         name: str='demo',
-        sources_subset: Iterable[Union[str, int]]=(),
+        sources_subset: Iterable[str | int]=(),
         overwrite_db: bool=False,
     ) -> None:
     with TemporaryDirectory() as tdir:
@@ -219,9 +231,10 @@ def _config_check(cfg: Path) -> Iterable[Exception]:
     logger.info('config: %s', cfg)
 
     def check(cmd: list[str | Path], **kwargs) -> Iterable[Exception]:
-        logger.debug(' '.join(map(str, cmd)))
-        res = run(cmd, **kwargs)
+        logger.debug(shlex.join(map(str, cmd)))
+        res = run(cmd, **kwargs)  # noqa: PLW1510
         if res.returncode > 0:
+            # TODO what's up with empty exception??
             yield Exception()
 
     logger.info('Checking syntax...')
@@ -239,7 +252,7 @@ def _config_check(cfg: Path) -> Iterable[Exception]:
     # todo not sure if should be more defensive than check_call here
     logger.info('Checking type safety...')
     try:
-        import mypy
+        import mypy  # noqa: F401
     except ImportError:
         logger.warning("mypy not found, can't use it to check config!")
     else:
@@ -291,7 +304,7 @@ def cli_doctor_server(args: argparse.Namespace) -> None:
     logger.info('You should see the database path and version above!')
 
 
-def _ordinal_or_name(s: str) -> Union[str, int]:
+def _ordinal_or_name(s: str) -> str | int:
     try:
         s = int(s)  # type: ignore
     except ValueError:
@@ -328,7 +341,7 @@ def main() -> None:
 
     F = lambda prog: argparse.ArgumentDefaultsHelpFormatter(prog, width=120)
     p = argparse.ArgumentParser(formatter_class=F)
-    subp = p.add_subparsers(dest='mode', )
+    subp = p.add_subparsers(dest='mode' )
     ep = subp.add_parser('index', help='Create/update the link database', formatter_class=F)
     add_index_args(ep, default_config_path())
     # TODO use some way to override or provide config only via cmdline?
@@ -348,7 +361,7 @@ def main() -> None:
     ap.add_argument('--no-serve', action='store_const', const=None, dest='port', help='Pass to only index without running server')
     ap.add_argument(
         '--as',
-        choices=list(sorted(demo_sources().keys())),
+        choices=sorted(demo_sources().keys()),
         default='guess',
         help='Promnesia source to index as (see https://github.com/karlicoss/promnesia/tree/master/src/promnesia/sources for the full list)',
     )
@@ -359,7 +372,7 @@ def main() -> None:
     install_server.setup_parser(isp)
 
     cp = subp.add_parser('config', help='Config management')
-    cp.set_defaults(func=lambda *args: cp.print_help())
+    cp.set_defaults(func=lambda *_args: cp.print_help())
     scp = cp.add_subparsers()
     ccp = scp.add_parser('check', help='Check config')
     ccp.set_defaults(func=config_check)
@@ -373,7 +386,7 @@ def main() -> None:
 
     dp = subp.add_parser('doctor', help='Troubleshooting assistant')
     dp.add_argument('--config', type=Path, default=default_config_path(), help='Config path')
-    dp.set_defaults(func=lambda *args: dp.print_help())
+    dp.set_defaults(func=lambda *_args: dp.print_help())
     sdp = dp.add_subparsers()
     sdp.add_parser('config'  , help='Check config'    ).set_defaults(func=config_check )
     sdp.add_parser('database', help='Inspect database').set_defaults(func=cli_doctor_db)

{promnesia-1.2.20240810 → promnesia-1.3.20241021}/src/promnesia/cannon.py

@@ -9,16 +9,17 @@ are same content, but you can't tell that by URL equality. Even canonical urls a
 
 Also some experiments to establish 'URL hierarchy'.
 """
-# TODO eh?? they fixed mobile.twitter.com?
+from __future__ import annotations
 
-from itertools import chain
 import re
 import typing
-from typing import Iterable, NamedTuple, Set, Optional, List, Sequence, Union, Tuple, Dict, Any, Collection
-
 import urllib.parse
-from urllib.parse import urlsplit, parse_qsl, urlunsplit, parse_qs, urlencode, SplitResult
+from collections.abc import Collection, Iterable, Sequence
 
+# TODO eh?? they fixed mobile.twitter.com?
+from itertools import chain
+from typing import Any, NamedTuple, Union
+from urllib.parse import SplitResult, parse_qsl, urlencode, urlsplit, urlunsplit
 
 # this has some benchmark, but quite a few librarires seem unmaintained, sadly
 # I guess i'll stick to default for now, until it's a critical bottleneck
@@ -108,11 +109,11 @@ default_qkeep = [
 
 # TODO perhaps, decide if fragment is meaningful (e.g. wiki) or random sequence of letters?
 class Spec(NamedTuple):
-    qkeep  : Optional[Union[Collection[str], bool]] = None
-    qremove: Optional[Set[str]] = None
+    qkeep  : Collection[str] | bool | None = None
+    qremove: set[str] | None = None
     fkeep  : bool = False
 
-    def keep_query(self, q: str) -> Optional[int]: # returns order
+    def keep_query(self, q: str) -> int | None: # returns order
         if self.qkeep is True:
             return 1
         qkeep = {
@@ -134,13 +135,13 @@ class Spec(NamedTuple):
         return None
 
     @classmethod
-    def make(cls, **kwargs) -> 'Spec':
+    def make(cls, **kwargs) -> Spec:
         return cls(**kwargs)
 
 S = Spec
 
 # TODO perhaps these can be machine learnt from large set of urls?
-specs: Dict[str, Spec] = {
+specs: dict[str, Spec] = {
     'youtube.com': S(
         # TODO search_query?
         qkeep=[ # note: experimental.. order matters here
@@ -178,7 +179,6 @@ specs: Dict[str, Spec] = {
 
             'source', 'tsid', 'refsrc', 'pnref', 'rc', '_rdr', 'src', 'hc_location', 'section', 'permPage', 'soft', 'pn_ref', 'action',
             'ti', 'aref', 'event_time_id', 'action_history', 'filter', 'ref_notif_type', 'has_source', 'source_newsfeed_story_type',
-            'ref_notif_type',
         },
     ),
     'physicstravelguide.com': S(fkeep=True), # TODO instead, pass fkeep marker object for shorter spec?
@@ -218,10 +218,10 @@ Spec2 = Any # TODO
 
 # TODO this should be a map
 Frag = Any
-Parts = Sequence[Tuple[str, str]]
+Parts = Sequence[tuple[str, str]]
 
 
-def _yc(domain: str, path: str, qq: Parts, frag: Frag) -> Tuple[Any, Any, Parts, Frag]:
+def _yc(domain: str, path: str, qq: Parts, frag: Frag) -> tuple[Any, Any, Parts, Frag]:
     if path[:5] == '/from':
         site = dict(qq).get('site')
         if site is not None:
@@ -232,7 +232,7 @@ def _yc(domain: str, path: str, qq: Parts, frag: Frag) -> Tuple[Any, Any, Parts,
     # TODO this should be in-place? for brevity?
     return (domain, path, qq, frag)
 
-def get_spec2(dom: str) -> Optional[Spec2]:
+def get_spec2(dom: str) -> Spec2 | None:
     return {
         'news.ycombinator.com': _yc,
     }.get(dom)
@@ -285,10 +285,10 @@ def transform_split(split: SplitResult):
     REST = r'(?P<rest>.*)'
 
     Left = Union[str, Sequence[str]]
-    Right = Tuple[str, str, str]
+    Right = tuple[str, str, str]
     # the idea is that we can unify certain URLs here and map them to the 'canonical' one
     # this is a dict only for grouping but should be a list really.. todo
-    rules: Dict[Left, Right] = {
+    rules: dict[Left, Right] = {
         # TODO m. handling might be quite common
         # f'm.youtube.com/{REST}': ('youtube.com', '{rest}'),
         (
@@ -322,9 +322,9 @@ def transform_split(split: SplitResult):
             continue
         gd = m.groupdict()
         if len(to) == 2:
-            to = to + ('', )
+            to = (*to, '')
 
-        (netloc, path, qq) = [t.format(**gd) for t in to]
+        (netloc, path, qq) = (t.format(**gd) for t in to)
         qparts.extend(parse_qsl(qq, keep_blank_values=True)) # TODO hacky..
         # TODO eh, qparts should really be a map or something...
         break
@@ -361,7 +361,7 @@ def myunsplit(domain: str, path: str, query: str, fragment: str) -> str:
 #     ]
 #     for re in regexes:
 
-def handle_archive_org(url: str) -> Optional[str]:
+def handle_archive_org(url: str) -> str | None:
     are = r'web.archive.org/web/(?P<timestamp>\d+)/(?P<rest>.*)'
     m = re.fullmatch(are, url)
     if m is None:
@@ -697,8 +697,8 @@ def groups(it, args): # pragma: no cover
     all_pats = get_patterns()
 
     from collections import Counter
-    c: typing.Counter[Optional[str]] = Counter()
-    unmatched: List[str] = []
+    c: typing.Counter[str | None] = Counter()
+    unmatched: list[str] = []
 
     def dump():
         print(c)
@@ -756,10 +756,10 @@ def groups(it, args): # pragma: no cover
 def display(it, args) -> None: # pragma: no cover
     # TODO better name?
     import difflib
-    # pylint: disable=import-error
-    from termcolor import colored as C # type: ignore
     from sys import stdout
 
+    from termcolor import colored as C  # type: ignore
+
     for line in it:
         line = line.strip()
         if args.human: