riko 0.68.1__py3-none-any.whl

riko/__init__.py

@@ -0,0 +1,321 @@
+# vim: sw=4:ts=4:expandtab
+"""
+riko
+~~~~
+Provides functions for analyzing and processing streams of structured data
+
+Examples:
+    basic usage::
+
+        >>> from riko.modules.itembuilder import pipe as itembuilder
+        >>> from riko.modules.strreplace import pipe as strreplace
+        >>> from riko.collections import SyncPipe
+        >>>
+        >>> ib_conf = {
+        ...     'attrs': [
+        ...         {'key': 'link', 'value': 'www.google.com'},
+        ...         {'key': 'title', 'value': 'google'},
+        ...         {'key': 'author', 'value': 'Tommy'}
+        ...      ]
+        ... }
+        >>>
+        >>> items = itembuilder(conf=ib_conf)
+        >>> next(items)
+        {'link': 'www.google.com', 'title': 'google', 'author': 'Tommy'}
+        >>> sr_conf = {
+        ...     'rule': [{'find': 'Tom', 'param': 'first', 'replace': 'Tim'}]
+        ... }
+        >>>
+        >>> items = itembuilder(conf=ib_conf)
+        >>> replaced = strreplace(next(items), conf=sr_conf, field='author')
+        >>> next(replaced)['strreplace']
+        'Timmy'
+
+"""
+
+from collections.abc import Callable, Generator, Iterable, Iterator, Mapping, Sequence
+from importlib.metadata import metadata, version
+from os import path as p
+from time import struct_time
+from typing import (
+    Any,
+    Literal,
+    TypeVar,
+    overload,
+)
+from typing import cast as cast_type
+
+from meza import compat
+from meza.fntools import Objectify as _Objectify
+
+from riko.types.general import (
+    BasicArg,
+    ComplexArg,
+    ComplexMapping,
+    IntermediateValue,
+    ObjconfParam,
+    ObjconfRule,
+    SyncItemFunc,
+)
+
+# https://github.com/astral-sh/uv/issues/7533#issuecomment-2472804995
+meta = metadata("riko")
+
+PACKAGE_INFO = {
+    "__version__": version("riko"),
+    "__title__": meta["Name"],
+    "__package_name__": meta["Name"],
+    "__description__": meta.get("Summary") or meta.get("Description", ""),
+    "__license__": meta.get("License-Expression") or meta.get("License", ""),
+    "__author__": meta.get("Author", ""),
+    "__email__": meta.get("Author-email", ""),
+}
+
+
+def __getattr__(name: str) -> str:
+    if name in PACKAGE_INFO:
+        return PACKAGE_INFO[name]
+    else:
+        msg = f"module {__name__} has no attribute {name}"
+        raise AttributeError(msg)
+
+
+__copyright__ = "Copyright 2015 Reuben Cummings"
+
+PARENT_DIR = p.abspath(p.dirname(__file__))
+ENCODING = "utf-8"
+
+
+def get_path(name: str):
+    if name.startswith(("http", "file:")):
+        url = name
+    else:
+        url = f"file://{p.join(PARENT_DIR, 'data', name)}"
+
+    return url
+
+
+def get_abspath(url: str, offline=False):
+    if url.startswith(("http", "file:///")):
+        pass
+    elif url.startswith("file://"):
+        parent = p.dirname(p.dirname(__file__))
+        rel_path = url[7:]
+        abspath = p.abspath(p.join(parent, rel_path))
+        url = f"file://{abspath}"
+    elif offline:
+        url = get_path(url)
+    else:
+        url = f"http://{url}" if url and "://" not in url else url
+
+    return compat.decode(url)
+
+
+def replacer(content: str, old: str, new="_") -> str:
+    if old:
+        replaced = content.replace(old, new)
+    elif content[0].isdecimal() or not content[0].isascii():
+        replaced = f"{new}{content}"
+    else:
+        replaced = content
+
+    return replaced
+
+
+class Context:
+    """
+    The context of a pipeline
+    verbose = debug printing during compilation and running
+    describe_input = return pipe input requirements
+    describe_dependencies = return a list of sub-pipelines used
+    test = takes input values from default (skips the console prompt)
+    inputs = a dictionary of values that overrides the defaults
+        e.g. {'name one': 'test value1'}
+    submodule = takes input values from inputs (or default)
+    """
+
+    def __init__(self, **kwargs):
+        self.verbose = kwargs.get("verbose", False)
+        self.test = kwargs.get("test", False)
+        self.describe_input = kwargs.get("describe_input", False)
+        self.describe_dependencies = kwargs.get("describe_dependencies", False)
+        self.inputs = kwargs.get("inputs", {})
+        self.submodule = kwargs.get("submodule", False)
+
+    def __repr__(self):
+        content = f"verbose={self.verbose}, test={self.test}, "
+        content += f"describe_input={self.describe_input}, "
+        content = f"describe_dependencies={self.describe_dependencies}, "
+        content = f"inputs={self.inputs}, submodule={self.submodule}"
+        return f"Context({content})"
+
+
+class Objectify(_Objectify):
+    """
+    Creates an object with dynamically set attributes. Useful
+    for accessing the kwargs of a function as attributes.
+    """
+
+    def __init__(self, data: ComplexMapping, *args, **kwargs):
+        """
+        Objectify constructor
+
+        Args:
+            data (dict): The attributes to set
+            defaults (dict): The default attributes
+
+        Examples:
+            >>> kw = Objectify({'KEY': 'foo'})
+            >>> kw.key
+            'foo'
+            >>> kw['key']
+            'foo'
+            >>> kw.get('key')
+            'foo'
+
+        """
+        _data = {k.lower(): v for k, v in data.items()}
+        super().__init__(_data, *args, **kwargs)
+
+    def __getattribute__(self, name: str) -> BasicArg:
+        return super().__getattribute__(name)
+
+    def __getitem__(self, name: str) -> BasicArg:
+        item = super().__getitem__(name)
+        return cast_type(BasicArg, item)
+
+    def __iter__(self) -> Iterator[str]:
+        return super().__iter__()
+
+    def iteritems(self) -> Iterator[tuple[str, BasicArg]]:
+        return super().iteritems()
+
+
+class Objconf(Objectify):
+    assign: str
+    attrs: Sequence[str]
+    base: str
+    col_names: Sequence[str]
+    combine: Literal["and", "or"]
+    count: str
+    currency: str
+    debug: bool
+    default: BasicArg
+    delay: int
+    delimiter: str
+    detag: bool
+    emit: bool
+    encoding: str
+    end: str
+    ext: str
+    format: str
+    group_key: str
+    html5: bool
+    join_key: str
+    length: int
+    limit: int
+    lower: bool
+    max_len: int
+    max_wait: int
+    multi: bool
+    name: str
+    other: str
+    op: str
+    other_join_key: str
+    param: ObjconfParam | Sequence[ObjconfParam]
+    parse_key: str
+    part: str
+    path: str | list[str]
+    permit: bool
+    precision: int
+    prompt: str
+    rule: ObjconfRule | Sequence[ObjconfRule]
+    skip_rows: int
+    sort: int
+    start: int
+    stop: str
+    strict: bool
+    stringify: bool
+    sum_key: str
+    times: int
+    token: str
+    token_key: str
+    type: str
+    unique_key: str
+    url: str
+    wait: int
+    xpath: str
+
+
+def objectify(
+    data: ComplexArg, func: SyncItemFunc | None = None, **defaults
+) -> ComplexArg:
+    if isinstance(data, Mapping):
+        objectified = Objectify(data, func=func, **defaults)
+    elif func:
+        if isinstance(data, (str, struct_time)):
+            objectified = func(data)
+        elif isinstance(data, Sequence):
+            objectified = [objectify(d, func) for d in data]
+        else:
+            objectified = func(data)
+    else:
+        objectified = data
+
+    return objectified
+
+
+# TODO: move back to meza
+K = TypeVar("K")
+T = TypeVar("T")
+
+
+@overload
+def listize(value: Mapping[K, T]) -> list[Mapping[K, T]]: ...
+@overload
+def listize(value: Callable[..., T]) -> list[Callable[..., T]]: ...
+@overload
+def listize(value: IntermediateValue) -> list[IntermediateValue]: ...
+@overload
+def listize(value: list[T]) -> list[T]: ...
+@overload
+def listize(value: Sequence[T]) -> Sequence[T]: ...
+@overload
+def listize(value: Generator[T, None, None]) -> Generator[T, None, None]: ...
+@overload
+def listize(value: Iterator[T]) -> Iterator[T]: ...
+@overload
+def listize(value: Iterable[T]) -> Iterable[T]: ...
+def listize(
+    value: Any,
+) -> list[Mapping | IntermediateValue | Callable] | Iterable | Sequence:
+    """
+    Create a listlike object from any value
+
+    Args:
+        value: The object to convert
+
+    Returns:
+        value as a listlike object (wrapped in a list or the value itself)
+
+    Examples:
+    >>> listize(x for x in range(3))  # doctest: +ELLIPSIS
+    <generator object <genexpr> at 0x...>
+    >>> listize([x for x in range(3)])
+    [0, 1, 2]
+    >>> listize(iter(x for x in range(3)))  # doctest: +ELLIPSIS
+    <generator object <genexpr> at 0x...>
+    >>> listize(range(3))
+    range(0, 3)
+
+    """
+    if not value:
+        result = []
+    elif isinstance(value, (Mapping, IntermediateValue, Callable)):
+        result = [value]
+    elif isinstance(value, (Iterable, Sequence)):
+        result = value
+    else:
+        result = [value]
+
+    return result

riko/autorss.py

@@ -0,0 +1,89 @@
+# vim: sw=4:ts=4:expandtab
+"""
+riko.autorss
+~~~~~~~~~~~~
+Provides functions for finding RSS feeds from a site's LINK tags
+"""
+
+from collections.abc import Generator, Iterator
+from io import StringIO, TextIOBase
+
+import pygogo as gogo
+from meza.compat import decode
+from twisted.internet.defer import Deferred
+
+from riko.bado import coroutine, microdom, return_value
+from riko.bado.io import async_url_open
+from riko.parsers import LinkParser
+from riko.utils import Fetch
+
+TIMEOUT = 10
+logger = gogo.Gogo(__name__, monolog=True).logger
+
+
+class RSSLinkParser(LinkParser):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, rss_only=True, **kwargs)
+
+
+def file2entries(
+    f: StringIO | Iterator[str] | TextIOBase, parser: RSSLinkParser
+) -> Iterator[dict]:
+    for line in f:
+        parser.feed(decode(line))
+
+        yield from parser.entry
+
+
+def doc2entries(document) -> Iterator[dict]:
+    for node in document.childNodes:
+        if hasattr(node, "attributes") and node.attributes:
+            entry = node.attributes
+            alternate = entry.get("rel") == "alternate"
+            rss = "rss" in entry.get("type", "")
+        else:
+            alternate = rss = None
+            entry = {}
+
+        if (alternate or rss) and "href" in entry:
+            entry["link"] = entry["href"]
+            entry["tag"] = node.nodeName
+            yield entry
+
+    for node in document.childNodes:
+        for entry in doc2entries(node):
+            yield entry
+
+
+@coroutine  # pyright: ignore[reportArgumentType]
+def async_get_rss(
+    url: str, **kwargs
+) -> Generator[Deferred[Iterator[dict]], Iterator[dict], None]:
+    try:
+        f = yield async_url_open(url, timeout=TIMEOUT)  # pyright: ignore[reportCallIssue]
+    except ValueError:
+        f = filter(None, url.splitlines())
+
+    document = microdom.parse(f, lenient=True)
+    return_value(doc2entries(document))
+
+
+def get_rss(
+    url: str, convert_charrefs=False, auto_sort=False, **kwargs
+) -> Iterator[dict]:
+    try:
+        parser = RSSLinkParser(convert_charrefs=convert_charrefs, **kwargs)
+    except TypeError:
+        parser = RSSLinkParser(**kwargs)
+
+    try:
+        f = Fetch(url, timeout=TIMEOUT)
+    except ValueError:
+        f = filter(None, url.splitlines())
+
+    entries = file2entries(f, parser)
+
+    if auto_sort:
+        entries = iter(sorted(entries, key=parser.keyfunc))
+
+    return entries

riko/bado/__init__.py

@@ -0,0 +1,36 @@
+# vim: sw=4:ts=4:expandtab
+"""
+riko.bado
+~~~~~~~~~
+Provides functions for creating asynchronous riko pipes
+
+Examples:
+    basic usage::
+
+        >>> from riko import get_path
+        >>> from riko.bado import react
+
+"""
+
+try:
+    from twisted.internet.task import react
+except ImportError:
+    react = lambda _, _reactor=None: None
+    inlineCallbacks = lambda _: lambda: None
+    returnValue = lambda _: lambda: None
+    backend = "empty"
+else:
+    from twisted.internet.defer import inlineCallbacks, returnValue
+
+    backend = "twisted"
+
+
+class Reactor:
+    fake = False
+
+
+reactor = Reactor()
+coroutine = inlineCallbacks
+return_value = returnValue
+_issync = backend == "empty"
+_isasync = not _issync

riko/bado/io.py

@@ -0,0 +1,150 @@
+# vim: sw=4:ts=4:expandtab
+"""
+riko.bado.io
+~~~~~~~~~~~~
+Provides functions for asynchronously reading files and urls
+
+Examples:
+    basic usage::
+
+        >>> from riko import get_path
+        >>> from riko.bado.io import async_url_open
+
+"""
+
+import builtins
+from collections.abc import Generator
+from io import StringIO
+from os import remove
+from tempfile import NamedTemporaryFile
+from typing import cast
+
+import pygogo as gogo
+
+from riko import ENCODING, get_abspath
+
+from . import coroutine, return_value
+
+try:
+    from twisted.internet.testing import AccumulatingProtocol
+except ImportError:
+    AccumulatingProtocol = callLater = StringTransport = FileSender = treq = object
+else:
+    import treq
+    from twisted.internet.defer import Deferred
+    from twisted.internet.reactor import callLater
+    from twisted.internet.testing import StringTransport
+    from twisted.protocols.basic import FileSender
+
+logger = gogo.Gogo(__name__, monolog=True).logger
+
+
+# http://stackoverflow.com/q/26314586/408556
+# http://stackoverflow.com/q/8157197/408556
+# http://stackoverflow.com/a/33708936/408556
+class FileReader(AccumulatingProtocol):
+    def __init__(self, filename, transform=None, delay=0, verbose=False):
+        self.f = builtins.open(filename, "rb")
+        self.transform = transform
+        self.delay = delay
+        self.producer = FileSender()
+        self.logger = gogo.Gogo(__name__, verbose=verbose).logger
+
+    def cleanup(self, *args):
+        self.f.close()
+        self.producer.stopProducing()
+
+    def resumeProducing(self):
+        chunk = self.file.read(self.CHUNK_SIZE) if self.file else ""
+
+        if not chunk:
+            self.file = None
+            self.consumer.unregisterProducer()
+
+            if self.deferred and self.delay:
+                callLater(self.delay, self.deferred.callback, self.lastSent)  # pyright: ignore[reportCallIssue]
+            elif self.deferred:
+                self.deferred.callback(self.lastSent)
+
+            self.deferred = None
+            return
+
+    def connectionLost(self, reason):
+        self.logger.debug(f"connectionLost: {reason}")
+        self.cleanup()
+
+    def connectionMade(self):
+        self.logger.debug(f"Connection made from {self.transport.getPeer()}")
+        args = (self.f, self.transport, self.transform)
+        self.d = self.closedDeferred = self.producer.beginFileTransfer(*args)
+
+        while not self.d.called:
+            self.producer.resumeProducing()
+
+        self.d.addErrback(self.logger.error)
+        self.d.addBoth(self.cleanup)
+
+
+@coroutine  # pyright: ignore[reportArgumentType]
+def async_read_file(
+    filename: str, transport, protocol=FileReader, encoding=ENCODING, **kwargs
+) -> Generator[Deferred[str], str, None]:
+    proto = protocol(filename.replace("file://", ""), **kwargs)
+    proto.makeConnection(transport)
+    yield proto.d
+    value: bytes = proto.transport.value()
+    return_value(value.decode(encoding))
+
+
+@coroutine  # pyright: ignore[reportArgumentType]
+def async_get_file(
+    filename: str, transport, protocol=FileReader, **kwargs
+) -> Generator[Deferred[StringIO], StringIO, None]:
+    proto = protocol(filename.replace("file://", ""), **kwargs)
+    proto.makeConnection(transport)
+    yield proto.d
+    proto.transport.io.seek(0)
+    f = cast(StringIO, proto.transport.io)
+    return_value(f)
+
+
+@coroutine  # pyright: ignore[reportArgumentType]
+def async_url_open(
+    url: str, timeout=0, encoding=ENCODING, **kwargs
+) -> Generator[Deferred[StringIO], StringIO, None]:
+    if url.startswith("http"):
+        page = NamedTemporaryFile(delete=False, mode="w")
+        new_url = page.name
+        response = yield treq.get(url)  # pyright: ignore[reportAttributeAccessIssue]
+        content = yield response.text()  # pyright: ignore[reportAttributeAccessIssue]
+        page.write(cast(str, content))
+        page.flush()
+        file_name = url.split("://")[1] if url.startswith("file") else None
+    else:
+        page, new_url, file_name = None, url, None
+
+    f = yield async_get_file(new_url, StringTransport(), **kwargs)  # pyright: ignore[reportCallIssue]
+
+    if not hasattr(f, "name") and file_name:
+        f.name = file_name
+
+    if page:
+        page.close()
+        remove(page.name)
+
+    return_value(f)
+
+
+@coroutine  # pyright: ignore[reportArgumentType]
+def async_url_read(
+    url: str, timeout=0, **kwargs
+) -> Generator[Deferred[str], str, None]:
+    url = get_abspath(url, offline=True)
+
+    if url.startswith("http"):
+        response = yield treq.get(url)  # pyright: ignore[reportAttributeAccessIssue]
+        content = yield response.text()  # pyright: ignore[reportAttributeAccessIssue]
+    else:
+        content = yield async_read_file(url, StringTransport(), **kwargs)  # pyright: ignore[reportReturnType, reportCallIssue]
+
+    return_value(content)

riko/bado/itertools.py

@@ -0,0 +1,102 @@
+# vim: sw=4:ts=4:expandtab
+"""
+riko.bado.itertools
+~~~~~~~~~~~~~~~~~~~
+Provides asynchronous ports of various builtin itertools functions
+
+Examples:
+    basic usage::
+
+        >>> from riko import get_path
+        >>> from riko.bado.itertools import coop_reduce
+
+"""
+
+import itertools as it
+from functools import partial
+
+from . import coroutine, reactor, return_value
+from .mock import FakeReactor
+
+try:
+    from twisted.internet.task import Cooperator
+except ImportError:
+    Cooperator = real_task = gatherResults = None
+else:
+    from twisted.internet import task as real_task
+    from twisted.internet.defer import gatherResults
+
+
+def get_task():
+    if reactor.fake:
+        task = Cooperator(
+            scheduler=partial(FakeReactor().callLater, FakeReactor._DELAY)
+        )
+    else:
+        task = real_task.Cooperator()
+
+    return task
+
+
+@coroutine
+def coop_reduce(func, iterable, initializer=None):
+    task = get_task()
+    iterable = iter(iterable)
+    x = initializer or next(iterable)
+    result = {}
+
+    def work(func, it, x):
+        for y in it:
+            result["value"] = x = func(x, y)
+            yield
+
+    _task = task.cooperate(work(func, iterable, x))
+    yield _task.whenDone()
+    return_value(result["value"])
+
+
+def async_reduce(async_func, iterable, initializer=None):
+    it = iter(iterable)
+    x = initializer or next(it)
+
+    @coroutine
+    def work(async_func, it, x):
+        for y in it:
+            x = yield async_func(x, y)
+
+        return_value(x)
+
+    return work(async_func, it, x)
+
+
+@coroutine
+def async_map(afunc, iterable, connections=0, **kwargs):
+    """
+    Parallel map for deferred callables using cooperative multitasking
+    http://stackoverflow.com/a/20376166/408556
+    """
+    if connections and not reactor.fake:
+        results = []
+        work = (afunc(x, **kwargs).addCallback(results.append) for x in iterable)
+        deferreds = [get_task().coiterate(work) for _ in range(connections)]
+        yield gatherResults(deferreds, consumeErrors=True)
+    else:
+        afunc = partial(afunc, **kwargs)
+        deferreds = map(afunc, iterable)
+        results = yield gatherResults(deferreds, consumeErrors=True)
+
+    return_value(results)
+
+
+def async_starmap(async_func, iterable):
+    """itertools.starmap for deferred callables"""
+    deferreds = it.starmap(async_func, iterable)
+    return gatherResults(deferreds, consumeErrors=True)
+
+
+def async_dispatch(split, *async_funcs, **kwargs):
+    return async_starmap(lambda item, f: f(item), zip(split, async_funcs, strict=False))
+
+
+def async_broadcast(item, *async_funcs, **kwargs):
+    return async_dispatch(it.repeat(item), *async_funcs, **kwargs)