danielutils 1.1.28__py3-none-any.whl

danielutils/functions/powerset.py

@@ -0,0 +1,31 @@
+import logging
+from typing import Iterable, Optional, Generator, Any
+import itertools
+from ..logging_.utils import get_logger
+
+logger = get_logger(__name__)
+
+
+def powerset(iterable: Iterable[Any], length: Optional[int] = None) -> Generator[tuple, None, None]:
+    """returns the powerset of specified length of an iterable
+    """
+    logger.debug("Generating powerset for iterable with length: %s", length)
+    if length is None:
+        if hasattr(iterable, "__len__"):
+            length = len(iterable)  # type:ignore
+            logger.debug("Auto-detected length: %s", length)
+        else:
+            logger.error("Cannot determine length of iterable")
+            raise ValueError(
+                "when using powerset must supply length explicitly or object should support len()")
+    
+    logger.debug("Generating combinations for lengths 0 to %s", length)
+    for i in range(length+1):
+        logger.debug("Generating combinations of length %s", i)
+        yield from itertools.combinations(iterable, i)
+    logger.debug("Powerset generation completed")
+
+
+__all__ = [
+    "powerset"
+]

danielutils/functions/subseteq.py

@@ -0,0 +1,25 @@
+import logging
+from ..logging_.utils import get_logger
+
+logger = get_logger(__name__)
+
+
+def subseteq(l1: list, l2: list) -> bool:
+    """return whether l1 is in list l2
+
+    Args:
+        l1 (list): first list
+        l2 (list): second list
+
+    Returns:
+        bool: boolean result
+    """
+    logger.debug("Checking if %s is subset of %s", l1, l2)
+    result = set(l1).issubset(set(l2))
+    logger.debug("Subset check result: %s", result)
+    return result
+
+
+__all__ = [
+    "subseteq"
+]

danielutils/functions/types_subseteq.py

@@ -0,0 +1,51 @@
+import logging
+from typing import Iterable, get_args, Union, Set as Set
+from ..reflection import get_python_version
+from ..logging_.utils import get_logger
+
+logger = get_logger(__name__)
+
+if get_python_version() >= (3, 9):
+    from builtins import set as Set
+
+
+def to_set(x: Union[type, Iterable[type]]) -> Set[int]:
+    """converts type/types to a set representing them
+    """
+    logger.debug("Converting type to set: %s", x)
+    res: Set[int] = set()
+    if hasattr(x, "__origin__") and x.__origin__ is Union:
+        logger.debug("Processing Union type")
+        for xi in get_args(x):
+            res.update(to_set(xi))
+    elif isinstance(x, Iterable):
+        logger.debug("Processing Iterable type")
+        for v in x:
+            res.update(to_set(v))
+        return res
+    else:
+        logger.debug("Adding single type ID: %s", id(x))
+        res.update(set([id(x)]))
+    logger.debug("Result set: %s", res)
+    return res
+
+
+def types_subseteq(a: Union[type, Iterable[type]], b: Union[type, Iterable[type]]) -> bool:
+    """checks if 'a' is contained in 'b' typing wise
+
+    Args:
+        a (type | Iterable[type])
+        b (type | Iterable[type])
+
+    Returns:
+        bool: result of containment
+    """
+    logger.debug("Checking type subset: %s ⊆ %s", a, b)
+    result = to_set(a).issubset(to_set(b))
+    logger.debug("Type subset result: %s", result)
+    return result
+
+
+__all__ = [
+    "types_subseteq"
+]

danielutils/generators/__init__.py

@@ -0,0 +1,3 @@
+from .join_generators import *
+from .generator_from_stream import *
+from .conditional_generator import *

danielutils/generators/conditional_generator.py

@@ -0,0 +1,59 @@
+import logging
+from typing import Generator, Callable, Any
+from ..logging_.utils import get_logger
+
+logger = get_logger(__name__)
+
+
+def generate_except(generator: Generator[Any, None, None],
+                    binary_consumer: Callable[[int, Any], bool]) -> Generator[Any, None, None]:
+    """will yield from generator except from when the predicate will return False
+
+    Args:
+        generator (Generator[Any, None, None]): generator
+        binary_consumer (Callable[[int, Any], bool]): predicate. (item_index, item)
+
+    Yields:
+        Generator[Any, None, None]: filtered generator
+    """
+    logger.info("Starting generate_except filtering")
+    items_processed = 0
+    items_yielded = 0
+    
+    for i, value in enumerate(generator):
+        items_processed += 1
+        if not binary_consumer(i, value):
+            items_yielded += 1
+            yield value
+    
+    logger.info("generate_except completed: processed %s items, yielded %s items", items_processed, items_yielded)
+
+
+def generate_when(generator: Generator[Any, None, None],
+                  binary_consumer: Callable[[int, Any], bool]) -> Generator[Any, None, None]:
+    """will yield from generator except from when the predicate will return True
+
+    Args:
+        generator (Generator[Any, None, None]): generator
+        binary_consumer (Callable[[int, Any], bool]): predicate. (item_index, item)
+
+    Yields:
+        Generator[Any, None, None]: filtered generator
+    """
+    logger.info("Starting generate_when filtering")
+    items_processed = 0
+    items_yielded = 0
+    
+    for i, value in enumerate(generator):
+        items_processed += 1
+        if binary_consumer(i, value):
+            items_yielded += 1
+            yield value
+    
+    logger.info("generate_when completed: processed %s items, yielded %s items", items_processed, items_yielded)
+
+
+__all__ = [
+    "generate_when",
+    "generate_except"
+]

danielutils/generators/generator_from_stream.py

@@ -0,0 +1,31 @@
+import logging
+from typing import IO, Generator, Any, Iterable, Union
+from ..decorators import validate
+from ..logging_.utils import get_logger
+
+logger = get_logger(__name__)
+
+
+@validate  # type:ignore
+def generator_from_stream(stream: Union[IO, Iterable[Any]]) -> Generator[Any, None, None]:
+    """will yield values from a given stream
+
+    Args:
+        stream (IO): the stream
+
+    Yields:
+        Generator[Any, None, None]: the resulting generator
+    """
+    logger.info("Starting generator_from_stream with stream type: %s", type(stream).__name__)
+    items_yielded = 0
+    
+    for v in stream:
+        items_yielded += 1
+        yield v
+    
+    logger.info("generator_from_stream completed, yielded %s items", items_yielded)
+
+
+__all__ = [
+    "generator_from_stream"
+]

danielutils/generators/join_generators.py

@@ -0,0 +1,109 @@
+import logging
+import time
+from typing import Generator, Any, Tuple as Tuple
+from threading import Semaphore  # , Condition
+from ..decorators import threadify
+from ..data_structures import AtomicQueue, Queue
+from ..better_builtins import AtomicCounter
+# from ..Print import aprint
+from ..reflection import get_python_version
+from ..logging_.utils import get_logger
+
+logger = get_logger(__name__)
+
+if get_python_version() >= (3, 9):
+    from builtins import tuple as Tuple  # type:ignore
+
+
+def join_generators_busy_waiting(*generators) -> Generator[Tuple[int, Any], None, None]:
+    """joins an arbitrary amount of generators to yield objects as soon someone yield an object
+
+    Yields:
+        Generator[tuple[int, Any], None, None]: resulting generator
+    """
+    logger.info("Starting join_generators_busy_waiting with %s generators", len(generators))
+    q: AtomicQueue[Tuple[int, Any]] = AtomicQueue()
+    threads_status = [False for _ in range(len(generators))]
+
+    @threadify  # type:ignore
+    def yield_from_one(thread_id: int, generator: Generator):
+        nonlocal threads_status
+        items_yielded = 0
+        for v in generator:
+            q.push((thread_id, v))
+            items_yielded += 1
+        logger.debug("Thread %s finished processing, yielded %s items", thread_id, items_yielded)
+        threads_status[thread_id] = True
+
+    for i, gen in enumerate(generators):
+        yield_from_one(i, gen)
+
+    # busy waiting
+    total_yielded = 0
+    while not all(threads_status):
+        while not q.is_empty():
+            item = q.pop()
+            total_yielded += 1
+            yield item
+    if not q.is_empty():
+        remaining_items = 0
+        for item in q:
+            remaining_items += 1
+            yield item
+    
+    logger.info("join_generators_busy_waiting completed, yielded %s items total", total_yielded)
+
+
+def join_generators(*generators) -> Generator[Tuple[int, Any], None, None]:
+    """will join generators to yield from all of them simultaneously 
+    without busy waiting, using semaphores and multithreading 
+
+    Yields:
+        Generator[Any, None, None]: one generator that combines all of the given ones
+    """
+    logger.info("Starting join_generators with %s generators", len(generators))
+    queue: Queue = Queue()
+    edit_queue_semaphore = Semaphore(1)
+    queue_status_semaphore = Semaphore(0)
+    finished_threads_counter = AtomicCounter()
+
+    @threadify  # type:ignore
+    def thread_entry_point(index: int, generator: Generator) -> None:
+        items_processed = 0
+        for value in generator:
+            with edit_queue_semaphore:
+                queue.push((index, value))
+            queue_status_semaphore.release()
+            items_processed += 1
+            time.sleep(0)
+        logger.debug("Thread %s finished processing, processed %s items", index, items_processed)
+        finished_threads_counter.increment()
+
+        if finished_threads_counter.get() == len(generators):
+            # re-release the lock once from the last thread because it
+            # gets stuck in the main loop after the generation has stopped
+            queue_status_semaphore.release()
+
+    for i, generator in enumerate(generators):
+        thread_entry_point(i, generator)
+
+    total_yielded = 0
+    while True:
+        queue_status_semaphore.acquire()
+        with edit_queue_semaphore:
+            while not queue.is_empty():
+                item = queue.pop()
+                total_yielded += 1
+                yield item
+        if finished_threads_counter.get() == len(generators):
+            with edit_queue_semaphore:
+                if queue.is_empty():
+                    break
+
+    logger.info("join_generators completed, yielded %s items total", total_yielded)
+
+
+__all__ = [
+    "join_generators_busy_waiting",
+    "join_generators"
+]

danielutils/internet.py

@@ -0,0 +1,92 @@
+import logging
+import urllib.request
+import urllib.parse
+from urllib.parse import urlparse
+import urllib
+from typing import Tuple as Tuple
+from .decorators import validate
+from .reflection import get_python_version
+from .logging_.utils import get_logger
+
+logger = get_logger(__name__)
+
+if get_python_version() >= (3, 9):
+    from builtins import tuple as Tuple  # type:ignore
+
+# def prettify_html(html: str) -> str:
+#     return html
+
+
+@validate  # type:ignore
+def get_html(url: str) -> str:
+    """returns the html for a given url
+
+    Args:
+        url (str): url
+
+    Returns:
+        str: the html as a string
+    """
+    logger.info("Fetching HTML from URL: %s", url)
+    user_agent = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7'
+    headers = {'User-Agent': user_agent, }
+    req = urllib.request.Request(url, headers=headers)
+    with urllib.request.urlopen(req) as f:
+        html = f.read().decode('UTF-8')
+    logger.info("Successfully fetched HTML, length: %d characters", len(html))
+    # return bs4(html, 'html.parser').prettify()
+    return html
+
+
+@validate  # type:ignore
+def get_url_details(url: str) -> "Tuple[str, str, str, str, str, str]":
+    """returns details about a url
+
+    Args:
+        url (str): url
+
+    Returns:
+        tuple[str, str, str, str, str]: scheme, netloc, path, params, query, fragment
+    """
+    scheme, netloc, path, params, query, fragment = urlparse(url)
+    return scheme, netloc, path, params, query, fragment
+
+
+@validate  # type:ignore
+def url_encode(s: str) -> str:
+    """encodes a string for url
+
+    Args:
+        s (str): string to encode
+
+    Returns:
+        str: encoded string
+    """
+    return urllib.parse.quote(s)
+
+
+@validate  # type:ignore
+def url_decode(s: str) -> str:
+    """decodes a url encoded string back to normal string
+
+    Args:
+        s (str): string to decode
+
+    Returns:
+        str: result string
+    """
+    return urllib.parse.unquote_plus(s)
+
+
+# def get_elements(html: str, tag: str) -> list[str]:
+#     return [str(v) for v in bs4(html, 'html.parser').find_all(tag)]
+
+
+__all__ = [
+    # "prettify_html",
+    "get_html",
+    "get_url_details",
+    "url_encode",
+    "url_decode",
+    # "get_elements"
+]