PFERD 3.6.0__tar.gz

pferd-3.6.0/LICENSE

@@ -0,0 +1,20 @@
+Copyright 2019-2024 Garmelon, I-Al-Istannen, danstooamerican, pavelzw,
+                    TheChristophe, Scriptim, thelukasprobst, Toorero,
+                    Mr-Pine, p-fruck
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

pferd-3.6.0/PFERD/__main__.py

@@ -0,0 +1,169 @@
+import argparse
+import asyncio
+import configparser
+import os
+import sys
+from pathlib import Path
+
+from .auth import AuthLoadError
+from .cli import PARSER, ParserLoadError, load_default_section
+from .config import Config, ConfigDumpError, ConfigLoadError, ConfigOptionError
+from .logging import log
+from .pferd import Pferd, PferdLoadError
+from .transformer import RuleParseError
+
+
+def load_config_parser(args: argparse.Namespace) -> configparser.ConfigParser:
+    log.explain_topic("Loading config")
+    parser = configparser.ConfigParser(interpolation=None)
+
+    if args.command is None:
+        log.explain("No CLI command specified, loading config from file")
+        Config.load_parser(parser, path=args.config)
+    else:
+        log.explain("CLI command specified, loading config from its arguments")
+        if args.command:
+            args.command(args, parser)
+
+    load_default_section(args, parser)
+
+    return parser
+
+
+def load_config(args: argparse.Namespace) -> Config:
+    try:
+        return Config(load_config_parser(args))
+    except ConfigLoadError as e:
+        log.error(str(e))
+        log.error_contd(e.reason)
+        sys.exit(1)
+    except ParserLoadError as e:
+        log.error(str(e))
+        sys.exit(1)
+
+
+def configure_logging_from_args(args: argparse.Namespace) -> None:
+    if args.explain is not None:
+        log.output_explain = args.explain
+    if args.status is not None:
+        log.output_status = args.status
+    if args.show_not_deleted is not None:
+        log.output_not_deleted = args.show_not_deleted
+    if args.report is not None:
+        log.output_report = args.report
+
+    # We want to prevent any unnecessary output if we're printing the config to
+    # stdout, otherwise it would not be a valid config file.
+    if args.dump_config_to == "-":
+        log.output_explain = False
+        log.output_status = False
+        log.output_report = False
+
+
+def configure_logging_from_config(args: argparse.Namespace, config: Config) -> None:
+    # In configure_logging_from_args(), all normal logging is already disabled
+    # whenever we dump the config. We don't want to override that decision with
+    # values from the config file.
+    if args.dump_config_to == "-":
+        return
+
+    try:
+        if args.explain is None:
+            log.output_explain = config.default_section.explain()
+        if args.status is None:
+            log.output_status = config.default_section.status()
+        if args.report is None:
+            log.output_report = config.default_section.report()
+        if args.show_not_deleted is None:
+            log.output_not_deleted = config.default_section.show_not_deleted()
+    except ConfigOptionError as e:
+        log.error(str(e))
+        sys.exit(1)
+
+
+def dump_config(args: argparse.Namespace, config: Config) -> None:
+    log.explain_topic("Dumping config")
+
+    if args.dump_config and args.dump_config_to is not None:
+        log.error("--dump-config and --dump-config-to can't be specified at the same time")
+        sys.exit(1)
+
+    try:
+        if args.dump_config:
+            config.dump()
+        elif args.dump_config_to == "-":
+            config.dump_to_stdout()
+        else:
+            config.dump(Path(args.dump_config_to))
+    except ConfigDumpError as e:
+        log.error(str(e))
+        log.error_contd(e.reason)
+        sys.exit(1)
+
+
+def main() -> None:
+    args = PARSER.parse_args()
+
+    # Configuring logging happens in two stages because CLI args have
+    # precedence over config file options and loading the config already
+    # produces some kinds of log messages (usually only explain()-s).
+    configure_logging_from_args(args)
+
+    config = load_config(args)
+
+    # Now, after loading the config file, we can apply its logging settings in
+    # all places that were not already covered by CLI args.
+    configure_logging_from_config(args, config)
+
+    if args.dump_config or args.dump_config_to is not None:
+        dump_config(args, config)
+        sys.exit()
+
+    try:
+        pferd = Pferd(config, args.crawler, args.skip)
+    except PferdLoadError as e:
+        log.unlock()
+        log.error(str(e))
+        sys.exit(1)
+
+    try:
+        if os.name == "nt":
+            # A "workaround" for the windows event loop somehow crashing after
+            # asyncio.run() completes. See:
+            # https://bugs.python.org/issue39232
+            # https://github.com/encode/httpx/issues/914#issuecomment-780023632
+            # TODO Fix this properly
+            loop = asyncio.get_event_loop()
+            loop.run_until_complete(pferd.run(args.debug_transforms))
+            loop.run_until_complete(asyncio.sleep(1))
+            loop.close()
+        else:
+            asyncio.run(pferd.run(args.debug_transforms))
+    except (ConfigOptionError, AuthLoadError) as e:
+        log.unlock()
+        log.error(str(e))
+        sys.exit(1)
+    except RuleParseError as e:
+        log.unlock()
+        e.pretty_print()
+        sys.exit(1)
+    except KeyboardInterrupt:
+        log.unlock()
+        log.explain_topic("Interrupted, exiting immediately")
+        log.explain("Open files and connections are left for the OS to clean up")
+        pferd.print_report()
+        # TODO Clean up tmp files
+        # And when those files *do* actually get cleaned up properly,
+        # reconsider if this should really exit with 1
+        sys.exit(1)
+    except Exception:
+        log.unlock()
+        log.unexpected_exception()
+        pferd.print_report()
+        sys.exit(1)
+    else:
+        pferd.print_report()
+
+
+if __name__ == "__main__":
+    main()

pferd-3.6.0/PFERD/auth/__init__.py

@@ -0,0 +1,29 @@
+from configparser import SectionProxy
+from typing import Callable, Dict
+
+from ..config import Config
+from .authenticator import Authenticator, AuthError, AuthLoadError, AuthSection  # noqa: F401
+from .credential_file import CredentialFileAuthenticator, CredentialFileAuthSection
+from .keyring import KeyringAuthenticator, KeyringAuthSection
+from .pass_ import PassAuthenticator, PassAuthSection
+from .simple import SimpleAuthenticator, SimpleAuthSection
+from .tfa import TfaAuthenticator
+
+AuthConstructor = Callable[[
+    str,                # Name (without the "auth:" prefix)
+    SectionProxy,       # Authenticator's section of global config
+    Config,             # Global config
+], Authenticator]
+
+AUTHENTICATORS: Dict[str, AuthConstructor] = {
+    "credential-file": lambda n, s, c:
+        CredentialFileAuthenticator(n, CredentialFileAuthSection(s), c),
+    "keyring": lambda n, s, c:
+        KeyringAuthenticator(n, KeyringAuthSection(s)),
+    "pass": lambda n, s, c:
+        PassAuthenticator(n, PassAuthSection(s)),
+    "simple": lambda n, s, c:
+        SimpleAuthenticator(n, SimpleAuthSection(s)),
+    "tfa": lambda n, s, c:
+        TfaAuthenticator(n),
+}

pferd-3.6.0/PFERD/auth/authenticator.py

@@ -0,0 +1,80 @@
+from abc import ABC, abstractmethod
+from typing import Tuple
+
+from ..config import Section
+
+
+class AuthLoadError(Exception):
+    pass
+
+
+class AuthError(Exception):
+    pass
+
+
+class AuthSection(Section):
+    def type(self) -> str:
+        value = self.s.get("type")
+        if value is None:
+            self.missing_value("type")
+        return value
+
+
+class Authenticator(ABC):
+    def __init__(self, name: str) -> None:
+        """
+        Initialize an authenticator from its name and its section in the config
+        file.
+
+        If you are writing your own constructor for your own authenticator,
+        make sure to call this constructor first (via super().__init__).
+
+        May throw an AuthLoadError.
+        """
+
+        self.name = name
+
+    @abstractmethod
+    async def credentials(self) -> Tuple[str, str]:
+        pass
+
+    async def username(self) -> str:
+        username, _ = await self.credentials()
+        return username
+
+    async def password(self) -> str:
+        _, password = await self.credentials()
+        return password
+
+    def invalidate_credentials(self) -> None:
+        """
+        Tell the authenticator that some or all of its credentials are invalid.
+
+        Authenticators should overwrite this function if they have a way to
+        deal with this issue that is likely to result in valid credentials
+        (e. g. prompting the user).
+        """
+
+        raise AuthError("Invalid credentials")
+
+    def invalidate_username(self) -> None:
+        """
+        Tell the authenticator that specifically its username is invalid.
+
+        Authenticators should overwrite this function if they have a way to
+        deal with this issue that is likely to result in valid credentials
+        (e. g. prompting the user).
+        """
+
+        raise AuthError("Invalid username")
+
+    def invalidate_password(self) -> None:
+        """
+        Tell the authenticator that specifically its password is invalid.
+
+        Authenticators should overwrite this function if they have a way to
+        deal with this issue that is likely to result in valid credentials
+        (e. g. prompting the user).
+        """
+
+        raise AuthError("Invalid password")

pferd-3.6.0/PFERD/auth/credential_file.py

@@ -0,0 +1,46 @@
+from pathlib import Path
+from typing import Tuple
+
+from ..config import Config
+from ..utils import fmt_real_path
+from .authenticator import Authenticator, AuthLoadError, AuthSection
+
+
+class CredentialFileAuthSection(AuthSection):
+    def path(self) -> Path:
+        value = self.s.get("path")
+        if value is None:
+            self.missing_value("path")
+        return Path(value)
+
+
+class CredentialFileAuthenticator(Authenticator):
+    def __init__(self, name: str, section: CredentialFileAuthSection, config: Config) -> None:
+        super().__init__(name)
+
+        path = config.default_section.working_dir() / section.path()
+        try:
+            with open(path, encoding="utf-8") as f:
+                lines = list(f)
+        except UnicodeDecodeError:
+            raise AuthLoadError(f"Credential file at {fmt_real_path(path)} is not encoded using UTF-8")
+        except OSError as e:
+            raise AuthLoadError(f"No credential file at {fmt_real_path(path)}") from e
+
+        if len(lines) != 2:
+            raise AuthLoadError("Credential file must be two lines long")
+        [uline, pline] = lines
+        uline = uline[:-1]  # Remove trailing newline
+        if pline.endswith("\n"):
+            pline = pline[:-1]
+
+        if not uline.startswith("username="):
+            raise AuthLoadError("First line must start with 'username='")
+        if not pline.startswith("password="):
+            raise AuthLoadError("Second line must start with 'password='")
+
+        self._username = uline[9:]
+        self._password = pline[9:]
+
+    async def credentials(self) -> Tuple[str, str]:
+        return self._username, self._password

pferd-3.6.0/PFERD/auth/keyring.py

@@ -0,0 +1,65 @@
+from typing import Optional, Tuple
+
+import keyring
+
+from ..logging import log
+from ..utils import agetpass, ainput
+from ..version import NAME
+from .authenticator import Authenticator, AuthError, AuthSection
+
+
+class KeyringAuthSection(AuthSection):
+    def username(self) -> Optional[str]:
+        return self.s.get("username")
+
+    def keyring_name(self) -> str:
+        return self.s.get("keyring_name", fallback=NAME)
+
+
+class KeyringAuthenticator(Authenticator):
+
+    def __init__(self, name: str, section: KeyringAuthSection) -> None:
+        super().__init__(name)
+
+        self._username = section.username()
+        self._password: Optional[str] = None
+        self._keyring_name = section.keyring_name()
+
+        self._password_invalidated = False
+        self._username_fixed = section.username() is not None
+
+    async def credentials(self) -> Tuple[str, str]:
+        # Request the username
+        if self._username is None:
+            async with log.exclusive_output():
+                self._username = await ainput("Username: ")
+
+        # First try looking it up in the keyring.
+        # Do not look it up if it was invalidated - we want to re-prompt in this case
+        if self._password is None and not self._password_invalidated:
+            self._password = keyring.get_password(self._keyring_name, self._username)
+
+        # If that fails it wasn't saved in the keyring - we need to
+        # read it from the user and store it
+        if self._password is None:
+            async with log.exclusive_output():
+                self._password = await agetpass("Password: ")
+                keyring.set_password(self._keyring_name, self._username, self._password)
+
+        self._password_invalidated = False
+        return self._username, self._password
+
+    def invalidate_credentials(self) -> None:
+        if not self._username_fixed:
+            self.invalidate_username()
+        self.invalidate_password()
+
+    def invalidate_username(self) -> None:
+        if self._username_fixed:
+            raise AuthError("Configured username is invalid")
+        else:
+            self._username = None
+
+    def invalidate_password(self) -> None:
+        self._password = None
+        self._password_invalidated = True

pferd-3.6.0/PFERD/auth/pass_.py

@@ -0,0 +1,98 @@
+import re
+import subprocess
+from typing import List, Tuple
+
+from ..logging import log
+from .authenticator import Authenticator, AuthError, AuthSection
+
+
+class PassAuthSection(AuthSection):
+    def passname(self) -> str:
+        if (value := self.s.get("passname")) is None:
+            self.missing_value("passname")
+        return value
+
+    def username_prefixes(self) -> List[str]:
+        value = self.s.get("username_prefixes", "login,username,user")
+        return [prefix.lower() for prefix in value.split(",")]
+
+    def password_prefixes(self) -> List[str]:
+        value = self.s.get("password_prefixes", "password,pass,secret")
+        return [prefix.lower() for prefix in value.split(",")]
+
+
+class PassAuthenticator(Authenticator):
+    PREFIXED_LINE_RE = r"([a-zA-Z]+):\s?(.*)"  # to be used with fullmatch
+
+    def __init__(self, name: str, section: PassAuthSection) -> None:
+        super().__init__(name)
+
+        self._passname = section.passname()
+        self._username_prefixes = section.username_prefixes()
+        self._password_prefixes = section.password_prefixes()
+
+    async def credentials(self) -> Tuple[str, str]:
+        log.explain_topic("Obtaining credentials from pass")
+
+        try:
+            log.explain(f"Calling 'pass show {self._passname}'")
+            result = subprocess.check_output(["pass", "show", self._passname], text=True)
+        except subprocess.CalledProcessError as e:
+            raise AuthError(f"Failed to get password info from {self._passname}: {e}")
+
+        prefixed = {}
+        unprefixed = []
+        for line in result.strip().splitlines():
+            if match := re.fullmatch(self.PREFIXED_LINE_RE, line):
+                prefix = match.group(1).lower()
+                value = match.group(2)
+                log.explain(f"Found prefixed line {line!r} with prefix {prefix!r}, value {value!r}")
+                if prefix in prefixed:
+                    raise AuthError(f"Prefix {prefix} specified multiple times")
+                prefixed[prefix] = value
+            else:
+                log.explain(f"Found unprefixed line {line!r}")
+                unprefixed.append(line)
+
+        username = None
+        for prefix in self._username_prefixes:
+            log.explain(f"Looking for username at prefix {prefix!r}")
+            if prefix in prefixed:
+                username = prefixed[prefix]
+                log.explain(f"Found username {username!r}")
+                break
+
+        password = None
+        for prefix in self._password_prefixes:
+            log.explain(f"Looking for password at prefix {prefix!r}")
+            if prefix in prefixed:
+                password = prefixed[prefix]
+                log.explain(f"Found password {password!r}")
+                break
+
+        if password is None and username is None:
+            log.explain("No username and password found so far")
+            log.explain("Using first unprefixed line as password")
+            log.explain("Using second unprefixed line as username")
+        elif password is None:
+            log.explain("No password found so far")
+            log.explain("Using first unprefixed line as password")
+        elif username is None:
+            log.explain("No username found so far")
+            log.explain("Using first unprefixed line as username")
+
+        if password is None:
+            if not unprefixed:
+                log.explain("Not enough unprefixed lines left")
+                raise AuthError("Password could not be determined")
+            password = unprefixed.pop(0)
+            log.explain(f"Found password {password!r}")
+
+        if username is None:
+            if not unprefixed:
+                log.explain("Not enough unprefixed lines left")
+                raise AuthError("Username could not be determined")
+            username = unprefixed.pop(0)
+            log.explain(f"Found username {username!r}")
+
+        return username, password

pferd-3.6.0/PFERD/auth/simple.py

@@ -0,0 +1,62 @@
+from typing import Optional, Tuple
+
+from ..logging import log
+from ..utils import agetpass, ainput
+from .authenticator import Authenticator, AuthError, AuthSection
+
+
+class SimpleAuthSection(AuthSection):
+    def username(self) -> Optional[str]:
+        return self.s.get("username")
+
+    def password(self) -> Optional[str]:
+        return self.s.get("password")
+
+
+class SimpleAuthenticator(Authenticator):
+    def __init__(self, name: str, section: SimpleAuthSection) -> None:
+        super().__init__(name)
+
+        self._username = section.username()
+        self._password = section.password()
+
+        self._username_fixed = self.username is not None
+        self._password_fixed = self.password is not None
+
+    async def credentials(self) -> Tuple[str, str]:
+        if self._username is not None and self._password is not None:
+            return self._username, self._password
+
+        async with log.exclusive_output():
+            if self._username is None:
+                self._username = await ainput("Username: ")
+            else:
+                print(f"Username: {self._username}")
+
+            if self._password is None:
+                self._password = await agetpass("Password: ")
+
+            # Intentionally returned inside the context manager so we know
+            # they're both not None
+            return self._username, self._password
+
+    def invalidate_credentials(self) -> None:
+        if self._username_fixed and self._password_fixed:
+            raise AuthError("Configured credentials are invalid")
+
+        if not self._username_fixed:
+            self._username = None
+        if not self._password_fixed:
+            self._password = None
+
+    def invalidate_username(self) -> None:
+        if self._username_fixed:
+            raise AuthError("Configured username is invalid")
+        else:
+            self._username = None
+
+    def invalidate_password(self) -> None:
+        if self._password_fixed:
+            raise AuthError("Configured password is invalid")
+        else:
+            self._password = None

pferd-3.6.0/PFERD/auth/tfa.py

@@ -0,0 +1,30 @@
+from typing import Tuple
+
+from ..logging import log
+from ..utils import ainput
+from .authenticator import Authenticator, AuthError
+
+
+class TfaAuthenticator(Authenticator):
+    def __init__(self, name: str) -> None:
+        super().__init__(name)
+
+    async def username(self) -> str:
+        raise AuthError("TFA authenticator does not support usernames")
+
+    async def password(self) -> str:
+        async with log.exclusive_output():
+            code = await ainput("TFA code: ")
+            return code
+
+    async def credentials(self) -> Tuple[str, str]:
+        raise AuthError("TFA authenticator does not support usernames")
+
+    def invalidate_username(self) -> None:
+        raise AuthError("TFA authenticator does not support usernames")
+
+    def invalidate_password(self) -> None:
+        pass
+
+    def invalidate_credentials(self) -> None:
+        pass

pferd-3.6.0/PFERD/cli/__init__.py

@@ -0,0 +1,14 @@
+# isort: skip_file
+
+# The order of imports matters because each command module registers itself
+# with the parser from ".parser" and the import order affects the order in
+# which they appear in the help. Because of this, isort is disabled for this
+# file. Also, since we're reexporting or just using the side effect of
+# importing itself, we get a few linting warnings, which we're disabling as
+# well.
+
+from . import command_local  # noqa: F401 imported but unused
+from . import command_ilias_web  # noqa: F401 imported but unused
+from . import command_kit_ilias_web  # noqa: F401 imported but unused
+from . import command_kit_ipd  # noqa: F401 imported but unused
+from .parser import PARSER, ParserLoadError, load_default_section  # noqa: F401 imported but unused

pferd-3.6.0/PFERD/cli/command_ilias_web.py

@@ -0,0 +1,56 @@
+import argparse
+import configparser
+
+from ..logging import log
+from .common_ilias_args import configure_common_group_args, load_common
+from .parser import CRAWLER_PARSER, SUBPARSERS, load_crawler
+
+COMMAND_NAME = "ilias-web"
+
+SUBPARSER = SUBPARSERS.add_parser(
+    COMMAND_NAME,
+    parents=[CRAWLER_PARSER],
+)
+
+GROUP = SUBPARSER.add_argument_group(
+    title=f"{COMMAND_NAME} crawler arguments",
+    description=f"arguments for the '{COMMAND_NAME}' crawler",
+)
+
+GROUP.add_argument(
+    "--base-url",
+    type=str,
+    metavar="BASE_URL",
+    help="The base url of the ilias instance"
+)
+
+GROUP.add_argument(
+    "--client-id",
+    type=str,
+    metavar="CLIENT_ID",
+    help="The client id of the ilias instance"
+)
+
+configure_common_group_args(GROUP)
+
+
+def load(
+        args: argparse.Namespace,
+        parser: configparser.ConfigParser,
+) -> None:
+    log.explain(f"Creating config for command '{COMMAND_NAME}'")
+
+    parser["crawl:ilias"] = {}
+    section = parser["crawl:ilias"]
+    load_crawler(args, section)
+
+    section["type"] = COMMAND_NAME
+    if args.ilias_url is not None:
+        section["base_url"] = args.ilias_url
+    if args.client_id is not None:
+        section["client_id"] = args.client_id
+
+    load_common(section, args, parser)
+
+
+SUBPARSER.set_defaults(command=load)