crosscompute-macros 0.1.0__tar.gz

crosscompute_macros-0.1.0/.gitignore

@@ -0,0 +1,20 @@
+# vim
+*.sw[op]
+*~
+# python
+*.egg*
+*.py[co]
+.cache
+.coverage
+.ipynb_checkpoints/
+.pytest_cache
+.tox
+__pycache__
+build/
+dist/
+# transient
+*.log
+*.tmp
+*_data
+.DS_Store
+.~lock.*

crosscompute_macros-0.1.0/PKG-INFO

@@ -0,0 +1,62 @@
+Metadata-Version: 2.4
+Name: crosscompute-macros
+Version: 0.1.0
+Summary: Reusable classes and functions needed by crosscompute packages
+Project-URL: Repository, https://github.com/crosscompute/crosscompute-macros
+Project-URL: Issues, https://github.com/crosscompute/crosscompute-macros/issues
+Author-email: Roy Hyunjin Han <rhh@crosscompute.com>
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Provides-Extra: all
+Requires-Dist: aiofiles; extra == 'all'
+Requires-Dist: aiofiles>=24.1.0; extra == 'all'
+Requires-Dist: aiohttp[speedups]>=3.12.12; extra == 'all'
+Requires-Dist: argon2-cffi; extra == 'all'
+Requires-Dist: jinja2; extra == 'all'
+Requires-Dist: markdown2; extra == 'all'
+Requires-Dist: msgpack; extra == 'all'
+Requires-Dist: ruamel-yaml; extra == 'all'
+Requires-Dist: ruamel-yaml-bytes; extra == 'all'
+Requires-Dist: shapely; extra == 'all'
+Requires-Dist: sqlalchemy; extra == 'all'
+Requires-Dist: starlette; extra == 'all'
+Provides-Extra: development
+Requires-Dist: pre-commit; extra == 'development'
+Provides-Extra: disk
+Requires-Dist: aiofiles>=24.1.0; extra == 'disk'
+Provides-Extra: iterable
+Provides-Extra: jinja
+Requires-Dist: jinja2; extra == 'jinja'
+Provides-Extra: log
+Provides-Extra: markdown
+Requires-Dist: markdown2; extra == 'markdown'
+Provides-Extra: msgpack
+Requires-Dist: msgpack; extra == 'msgpack'
+Provides-Extra: security
+Requires-Dist: argon2-cffi; extra == 'security'
+Provides-Extra: shapely
+Requires-Dist: aiofiles; extra == 'shapely'
+Requires-Dist: shapely; extra == 'shapely'
+Provides-Extra: sqlalchemy
+Requires-Dist: sqlalchemy; extra == 'sqlalchemy'
+Provides-Extra: starlette
+Requires-Dist: starlette; extra == 'starlette'
+Provides-Extra: test
+Requires-Dist: aiofiles>=24.1.0; extra == 'test'
+Requires-Dist: pytest-asyncio; extra == 'test'
+Requires-Dist: pytest-cov; extra == 'test'
+Requires-Dist: pytest-xdist; extra == 'test'
+Provides-Extra: web
+Requires-Dist: aiofiles; extra == 'web'
+Requires-Dist: aiohttp[speedups]>=3.12.12; extra == 'web'
+Provides-Extra: yaml
+Requires-Dist: ruamel-yaml; extra == 'yaml'
+Requires-Dist: ruamel-yaml-bytes; extra == 'yaml'
+Description-Content-Type: text/markdown
+
+# CrossCompute Macros
+
+Here are reusable functions and classes that are primarily used by the CrossCompute Software Development Kit and CrossCompute Platform.

crosscompute_macros-0.1.0/README.md

@@ -0,0 +1,3 @@
+# CrossCompute Macros
+
+Here are reusable functions and classes that are primarily used by the CrossCompute Software Development Kit and CrossCompute Platform.

crosscompute_macros-0.1.0/crosscompute_macros/abstract.py

@@ -0,0 +1,22 @@
+class Clay:
+
+    def __init__(self, instance=None, **kwargs):
+        d = self.__dict__
+        if instance:
+            d.update(instance.__dict__)
+        d.update(kwargs)
+
+
+class Mold:
+
+    def __init__(self, defaults=None):
+        self.defaults = defaults or {}
+
+    def set(self, k, d, f=None):
+        v = d.get(k)
+        if v is None:
+            x = self.defaults[k]
+            v = x() if callable(x) else x
+        if f:
+            v = f(v)
+        setattr(self, k, v)

crosscompute_macros-0.1.0/crosscompute_macros/datetime.py

@@ -0,0 +1,25 @@
+import calendar
+from datetime import datetime, UTC
+
+
+def get_longstamp(when=None):
+    return get_timestamp(when, template='%Y%m%d-%H%M%S-%f')
+
+
+def get_datestamp(when=None, template='%Y%m%d'):
+    return get_timestamp(when, template)
+
+
+def get_timestamp(when=None, template='%Y%m%d-%H%M', tz=None):
+    if when is None:
+        when = datetime.now(tz=tz)
+    return when.strftime(template)
+
+
+def get_utc_now():
+    return datetime.now(UTC)
+
+
+def get_day_count(year):
+    # https://stackoverflow.com/a/67395758/192092
+    return 365 + calendar.isleap(year)

crosscompute_macros-0.1.0/crosscompute_macros/disk.py

@@ -0,0 +1,202 @@
+import json
+from contextlib import suppress
+from logging import getLogger
+from os import pathconf
+from os.path import dirname, join, normpath
+
+import aiofiles
+import aiofiles.os
+
+from .error import DiskError, ParsingError
+from .iterable import LRUDict
+from .random import make_random_string
+
+
+class FileCache(LRUDict):
+
+    def __init__(self, *args, load, length: int, **kwargs):
+        super().__init__(*args, length=length, **kwargs)
+        self._load = load
+
+    async def set(self, path, d):
+        t = await get_modification_time(path)
+        value = t, d
+        super().__setitem__(str(path), value)
+
+    async def get(self, path):
+        path = str(path)
+        if path in self:
+            old_t, d = super().__getitem__(path)
+            new_t = await get_modification_time(path)
+            if old_t == new_t:
+                return d
+        x = await self._load(path)
+        await self.set(path, x)
+        return x
+
+
+async def make_random_folder(
+        base_folder, name_length=16, with_fixed_length=False,
+        length_increment=8, retry_count=3):
+    retry_index = 0
+    while True:
+        name = chop_name(make_random_string(name_length))
+        folder = join(base_folder, name)
+        try:
+            await make_folder(folder, with_existing=False)
+            break
+        except FileExistsError as e:
+            if retry_index < retry_count:
+                retry_index += 1
+                L.debug(f'folder {retry_index=}', path=base_folder)
+                continue
+            if with_fixed_length:
+                x = (
+                    'folder is nearing capacity and cannot support more '
+                    'random folders')
+                raise DiskError(x, path=base_folder) from e
+            name_length += length_increment
+            L.debug(f'folder {name_length=}', path=base_folder)
+    return folder
+
+
+async def make_folder(folder, with_existing=True):
+    await aiofiles.os.makedirs(folder, exist_ok=with_existing)
+    return folder
+
+
+async def copy_path(target_path, source_path):
+    byte_count = await get_byte_count(source_path)
+    await make_folder(target_path.parent)
+    async with aiofiles.open(
+        target_path, mode='wb',
+    ) as t, aiofiles.open(
+        source_path, mode='rb',
+    ) as s:
+        await aiofiles.os.sendfile(t.fileno(), s.fileno(), 0, byte_count)
+    return target_path
+
+
+async def get_byte_count(path):
+    s = await aiofiles.os.stat(path)
+    return s.st_size
+
+
+async def save_raw_text(path, text):
+    await make_folder(dirname(path))
+    async with aiofiles.open(path, mode='wt') as f:
+        await f.write(text)
+    return path
+
+
+async def load_raw_text(path):
+    try:
+        async with aiofiles.open(path, mode='rt') as f:
+            text = await f.read()
+    except OSError as e:
+        x = f'path is not accessible; {e}'
+        raise DiskError(x, path=path) from e
+    return text.rstrip()
+
+
+async def save_raw_json(path, dictionary):
+    await make_folder(dirname(path))
+    async with aiofiles.open(path, mode='wt') as f:
+        await f.write(json.dumps(dictionary))
+
+
+async def load_raw_json(path):
+    try:
+        async with aiofiles.open(path, mode='rt') as f:
+            dictionary = json.loads(await f.read())
+    except OSError as e:
+        x = f'path is not accessible; {e}'
+        raise DiskError(x, path=path) from e
+    except json.JSONDecodeError as e:
+        x = f'file is not valid json; {e}'
+        raise ParsingError(x, path=path) from e
+    return dictionary
+
+
+async def update_raw_json(path, dictionary):
+    if await is_existing_path(path):
+        async with aiofiles.open(path, mode='r+t') as f:
+            with suppress(json.JSONDecodeError):
+                dictionary = json.loads(await f.read()) | dictionary
+            await f.seek(0)
+            await f.write(json.dumps(dictionary))
+            await f.truncate()
+    else:
+        await save_raw_json(path, dictionary)
+    return dictionary
+
+
+async def make_soft_link(target_path, source_path):
+    await aiofiles.os.symlink(source_path, target_path)
+
+
+async def make_hard_link(target_path, source_path):
+    await aiofiles.os.link(source_path, target_path)
+
+
+async def get_real_path(path):
+    path = await get_absolute_path(path)
+    original_path = path
+    paths = [path]
+    while await is_link_path(path):
+        path = await get_absolute_path(join(
+            dirname(path), await aiofiles.os.readlink(path)))
+        if path in paths:
+            x = 'file is a circular symlink'
+            raise DiskError(x, path=original_path)
+        paths.append(path)
+    return path
+
+
+async def get_absolute_path(path):
+    return await aiofiles.os.path.abspath(path)
+
+
+async def is_path_in_folder(path, folder):
+    try:
+        path = await get_real_path(path)
+        folder = await get_real_path(folder)
+    except DiskError as e:
+        L.debug(e)
+        return False
+    return path.startswith(folder)
+
+
+def is_contained_path(path):
+    folder = '_'
+    return normpath(join(folder, path)).startswith(folder)
+
+
+def chop_name(name):
+    parts = []
+    name_length = len(name)
+    folder_count = name_length // MAXIMUM_FILE_NAME_LENGTH
+    for i in range(folder_count):
+        a = MAXIMUM_FILE_NAME_LENGTH * i
+        b = MAXIMUM_FILE_NAME_LENGTH * (i + 1)
+        parts.append(name[a:b])
+    parts.append(name[MAXIMUM_FILE_NAME_LENGTH * folder_count:])
+    return '/'.join(parts)
+
+
+def get_folder(path, relative_path):
+    return str(path).rsplit(relative_path)[0]
+
+
+get_modification_time = aiofiles.os.path.getmtime
+is_existing_path = aiofiles.os.path.exists
+is_file_path = aiofiles.os.path.isfile
+is_folder_path = aiofiles.os.path.isdir
+is_link_path = aiofiles.os.path.islink
+is_same_path = aiofiles.os.path.samefile
+list_paths = aiofiles.os.listdir
+remove_path = aiofiles.os.unlink
+
+
+L = getLogger(__name__)
+MAXIMUM_FILE_NAME_LENGTH = pathconf('/', 'PC_NAME_MAX')

crosscompute_macros-0.1.0/crosscompute_macros/error.py

@@ -0,0 +1,46 @@
+from .log import redact_path
+
+
+class MacroError(Exception):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args)
+        self.__dict__.update(kwargs)
+
+    def __str__(self):
+        texts = [super().__str__()]
+        if hasattr(self, 'path'):
+            texts.append(f'path="{redact_path(self.path)}"')
+        return '; '.join(texts)
+
+
+class DiskError(MacroError):
+    pass
+
+
+class PackageError(MacroError):
+    pass
+
+
+class SecurityError(MacroError):
+    pass
+
+
+class WebError(MacroError):
+    pass
+
+
+class WebConnectionError(WebError):
+    pass
+
+
+class WebRequestError(WebError):
+    pass
+
+
+class ParsingError(MacroError):
+    pass
+
+
+class FormattingError(MacroError):
+    pass

crosscompute_macros-0.1.0/crosscompute_macros/iterable.py

@@ -0,0 +1,81 @@
+import operator
+from collections import OrderedDict, defaultdict
+
+
+class LRUDict(OrderedDict):
+    # https://gist.github.com/davesteele/44793cd0348f59f8fadd49d7799bd306
+
+    def __init__(self, *args, length: int, **kwargs):
+        if length <= 0:
+            raise ValueError
+        self._length = length
+        super().__init__(*args, **kwargs)
+
+    def __setitem__(self, key, value):
+        super().__setitem__(key, value)
+        super().move_to_end(key)
+        while len(self) > self._length:
+            super().__delitem__(next(iter(self)))
+
+    def __getitem__(self, key):
+        value = super().__getitem__(key)
+        super().move_to_end(key)
+        return value
+
+
+class InfiniteDefaultDict(dict):
+
+    def __getitem__(self, key):
+        if key in self:
+            value = super().__getitem__(key)
+        else:
+            value = InfiniteDefaultDict()
+            super().__setitem__(key, value)
+        return value
+
+
+def apply_functions(value, function_names, function_by_name):
+    for function_name in function_names:
+        function_name = function_name.strip()
+        if not function_name:
+            continue
+        f = function_by_name[function_name]
+        value = f(value)
+    return value
+
+
+def get_unique_order(texts):
+    return list(dict.fromkeys([_.strip() for _ in texts]))
+
+
+def group_by_attribute(items, name):
+    d = defaultdict(list)
+    for item in items:
+        d[getattr(item, name)].append(item)
+    return dict(d)
+
+
+def find_item(
+        items, key, value, get_value=lambda item, key: getattr(item, key),
+        normalize=lambda _: _, compare=operator.eq):
+    normalized_value = normalize(value)
+
+    def is_match(item):
+        try:
+            v = get_value(item, key)
+        except KeyError:
+            is_match = False
+        else:
+            normalized_v = normalize(v)
+            is_match = compare(normalized_value, normalized_v)
+        return is_match
+
+    return next(filter(is_match, items))
+
+
+def drop_null_values(d):
+    keys = list(d.keys())
+    for k in keys:
+        if d[k] is None:
+            del d[k]
+    return d

crosscompute_macros-0.1.0/crosscompute_macros/jinja.py

@@ -0,0 +1,72 @@
+from os.path import dirname, getmtime, join, normpath, realpath
+from pathlib import PurePath
+from string import Template as StringTemplate
+
+from jinja2 import (
+    BaseLoader,
+    Environment,
+    Template as JinjaTemplate,
+    TemplateNotFound)
+
+
+class AssetStorage:
+
+    def __init__(self, folder):
+        self.folder = folder
+
+    def load_raw_text(self, file_name):
+        return (self.folder / file_name).read_text().strip()
+
+    def load_string_text(self, file_name):
+        return StringTemplate(self.load_raw_text(file_name))
+
+    def load_jinja_text(self, file_name):
+        return JinjaTemplate(self.load_raw_text(file_name), trim_blocks=True)
+
+
+class PathTemplateLoader(BaseLoader):
+
+    def __init__(self, encoding='utf-8'):
+        self.encoding = encoding
+
+    def get_source(self, environment, template):  # noqa: ARG002
+        'Support absolute template paths.'
+        try:
+            modification_time = getmtime(template)  # noqa: PTH204
+        except (OSError, TypeError) as e:
+            raise TemplateNotFound(template) from e
+
+        def is_latest():
+            try:
+                return modification_time == getmtime(template)  # noqa: PTH204
+            except OSError:
+                return False
+
+        with open(template, mode='rt', encoding=self.encoding) as f:  # noqa: PTH123
+            text = f.read()
+        return text, realpath(template), is_latest
+
+
+class RelativeTemplateEnvironment(Environment):
+
+    def join_path(self, template, parent):
+        'Support relative template paths via extends, import, include.'
+        template = get_asset_path(template)
+        return normpath(join(dirname(  # noqa: PTH118 PTH120
+            parent), template)) if template else template
+
+
+def get_asset_path(asset_uri):
+    asset_parts = asset_uri.split(':')
+    if len(asset_parts) > 1:
+        package_name, relative_path = asset_parts
+        package_name = package_name.strip()
+    else:
+        package_name, relative_path = '', asset_parts[0]
+    if package_name:
+        package = __import__(package_name)
+        package_folder = PurePath(package.__file__).parent
+        path = str(package_folder / relative_path)
+    else:
+        path = relative_path
+    return path

crosscompute_macros-0.1.0/crosscompute_macros/log.py

@@ -0,0 +1,44 @@
+import re
+from logging import (
+    basicConfig,
+    getLogger,
+    DEBUG,
+    INFO)
+from os.path import expanduser
+
+
+def configure_argument_parser_for_logging(argument_parser):
+    argument_parser.add_argument(
+        '--debug', dest='with_debug', action='store_true', default=False,
+        help='show debugging messages')
+
+
+def configure_logging_from(args, logging_level_by_package_name):
+    with_debug = args.with_debug
+    configure_logging(with_debug)
+    configure_logging_level_by_package_name(logging_level_by_package_name)
+
+
+def configure_logging(with_debug, timestamp='%Y%m%d-%H%M'):
+    if with_debug:
+        logging_level = DEBUG
+        logging_format = (
+            '%(asctime)s %(levelname)s %(name)s:%(lineno)s %(message)s')
+    else:
+        logging_level = INFO
+        logging_format = '%(asctime)s %(levelname)s %(message)s'
+    basicConfig(format=logging_format, datefmt=timestamp, level=logging_level)
+
+
+def configure_logging_level_by_package_name(logging_level_by_package_name):
+    for package_name, logging_level in logging_level_by_package_name.items():
+        getLogger(package_name).setLevel(logging_level)
+
+
+def redact_path(x):
+    return re.sub(r'^' + re.escape(expanduser('~')), '~', str(x))
+
+
+def make_count_text(message_text, item_count, singular_name, plural_name):
+    item_name = singular_name if item_count == 1 else plural_name
+    return f'{message_text} {item_count} {item_name}'

crosscompute_macros-0.1.0/crosscompute_macros/markdown.py

@@ -0,0 +1,33 @@
+import re
+
+from markdown2 import markdown
+
+
+OUTER_PARAGRAPH_PATTERN = re.compile(r'^<p>((?:(?!<p>).)*)</p>$', re.DOTALL)
+EXTRAS = [
+    'break-on-newline',
+    'code-friendly',
+    'cuddled-lists',
+    'fenced-code-blocks',
+    'footnotes',
+    'markdown-in-html',
+    'mermaid',
+    'strike',
+    'target-blank-links',
+    'tables',
+    'use-file-vars',
+    'task_list']
+
+
+def get_html_from_markdown(text, extras=EXTRAS):
+    return markdown(text, extras=extras).strip()
+
+
+def remove_outer_paragraph(html):
+    return OUTER_PARAGRAPH_PATTERN.sub(r'\g<1>', html)
+
+
+def remove_inner_paragraphs(html, a, b):
+    html = re.sub(r'<p>(\s*(?:[^<X]*X))'.replace('X', a), r'\g<1>', html)
+    html = re.sub(r'((?:X[^>]*)\s*)</p>'.replace('X', b), r'\g<1>', html)
+    return html

crosscompute_macros-0.1.0/crosscompute_macros/msgpack.py

@@ -0,0 +1,19 @@
+import msgpack
+
+from .sqlalchemy import EncryptedBinary
+
+
+class EncryptedMap(EncryptedBinary):
+    cache_ok = False
+
+    def process_bind_param(self, value, dialect):
+        if value is not None:
+            payload = msgpack.packb(value)
+            value = super().process_bind_param(payload, dialect)
+        return value
+
+    def process_result_value(self, value, dialect):
+        if value is not None:
+            payload = super().process_result_value(value, dialect)
+            value = msgpack.unpackb(payload)
+        return value

crosscompute_macros-0.1.0/crosscompute_macros/package.py

@@ -0,0 +1,40 @@
+from importlib import import_module
+from packaging.version import InvalidVersion, Version as PackageVersion
+
+from .error import PackageError
+
+
+class Version(PackageVersion):
+
+    def __init__(self, version_text):
+        try:
+            super().__init__(version_text)
+        except InvalidVersion as e:
+            x = f'version "{version_text}" is not valid'
+            raise PackageError(x) from e
+
+    def is_equivalent(self, version, depth=None):
+        if self.epoch != version.epoch:
+            return False
+        this_release = self.release
+        that_release = version.release
+        if depth:
+            this_release = this_release[:depth]
+            that_release = that_release[:depth]
+        return this_release == that_release
+
+
+def import_attribute(attribute_string):
+    module_string, attribute_name = attribute_string.rsplit('.', maxsplit=1)
+    return getattr(import_module(module_string), attribute_name)
+
+
+def is_newer_version(new_version_text, old_version_text):
+    return Version(new_version_text) > Version(old_version_text)
+
+
+def is_equivalent_version(
+        new_version_text, old_version_text, version_depth=None):
+    new_version = Version(new_version_text)
+    old_version = Version(old_version_text)
+    return new_version.is_equivalent(old_version, depth=version_depth)

crosscompute_macros-0.1.0/crosscompute_macros/process.py

@@ -0,0 +1,21 @@
+import asyncio
+import subprocess
+
+
+async def run_process(args, cwd=None, env=None, text=None, check=False):
+    p = await asyncio.create_subprocess_exec(
+        *args,
+        cwd=cwd,
+        env=env,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE)
+    return_code = await p.wait()
+    if check and return_code:
+        stdout = await p.stdout.read()
+        stderr = await p.stderr.read()
+        raise subprocess.CalledProcessError(
+            returncode=return_code,
+            cmd=args,
+            output=stdout.decode() if text else stdout,
+            stderr=stderr.decode() if text else stderr)
+    return p

crosscompute_macros-0.1.0/crosscompute_macros/pydantic.py

@@ -0,0 +1,2 @@
+def get_schema_map(schema):
+    return schema.model_dump(mode='json', exclude_defaults=True)

crosscompute_macros-0.1.0/crosscompute_macros/random.py

@@ -0,0 +1,6 @@
+import secrets
+from string import ascii_letters, digits
+
+
+def make_random_string(length, alphabet=ascii_letters + digits):
+    return ''.join(secrets.choice(alphabet) for _ in range(length))

crosscompute_macros-0.1.0/crosscompute_macros/security.py

@@ -0,0 +1,19 @@
+from argon2 import PasswordHasher
+from argon2.exceptions import InvalidHash, VerifyMismatchError
+
+from .error import SecurityError
+
+
+def verify_text(encoded_hash, candidate_text):
+    try:
+        password_hasher.verify(encoded_hash, candidate_text)
+    except VerifyMismatchError as e:
+        x = 'text does not match hash'
+        raise SecurityError(x) from e
+    except InvalidHash as e:
+        x = 'hash is not valid'
+        raise SecurityError(x) from e
+
+
+password_hasher = PasswordHasher()
+hash_text = password_hasher.hash

crosscompute_macros-0.1.0/crosscompute_macros/shapely.py

@@ -0,0 +1,51 @@
+import random
+
+from shapely.geometry import GeometryCollection, LineString, Point, shape
+from shapely.ops import split, unary_union
+
+from .disk import load_raw_json
+
+
+async def load_geometry_collection(geojson_path):
+    d = await load_raw_json(geojson_path)
+    return unary_union([shape(_) for _ in d['features']])
+
+
+def make_random_points(source_geometry, target_count):
+    points = []
+    minimum_x, minimum_y, maximum_x, maximum_y = source_geometry.bounds
+    while len(points) < target_count:
+        # Generate random points inside bounds
+        random_points = [Point(
+            random.uniform(minimum_x, maximum_x),  # noqa: S311
+            random.uniform(minimum_y, maximum_y),  # noqa: S311
+        ) for _ in range(target_count)]
+        # Retain points inside region
+        collection = unary_union(random_points + points)
+        intersection = collection.intersection(source_geometry)
+        if intersection.geom_type == 'Point':
+            points = [intersection]
+        else:
+            points = list(intersection.geoms)
+    # Trim if there are too many
+    return points[:target_count]
+
+
+def slice_geometry(source_geometry, target_x_count=1, target_y_count=1):
+    minimum_x, minimum_y, maximum_x, maximum_y = source_geometry.bounds
+    dx = (maximum_x - minimum_x) / target_x_count
+    dy = (maximum_y - minimum_y) / target_y_count
+    x_lines = [LineString([
+        (minimum_x + i * dx, minimum_y),
+        (minimum_x + i * dx, maximum_y),
+    ]) for i in range(1, target_x_count)]
+    y_lines = [LineString([
+        (minimum_x, minimum_y + i * dy),
+        (maximum_x, minimum_y + i * dy),
+    ]) for i in range(1, target_y_count)]
+    geometry_collection = source_geometry
+    for line in x_lines + y_lines:
+        geometry_collection = split(geometry_collection, line)
+    if geometry_collection.geom_type != 'GeometryCollection':
+        geometry_collection = GeometryCollection(geometry_collection)
+    return geometry_collection

crosscompute_macros-0.1.0/crosscompute_macros/sqlalchemy.py

@@ -0,0 +1,107 @@
+from datetime import UTC
+from pathlib import Path
+
+from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
+from sqlalchemy.ext.mutable import Mutable
+from sqlalchemy.types import DateTime, LargeBinary, String, TypeDecorator
+
+from .security import hash_text
+
+
+class MutableMap(Mutable, dict):
+
+    @classmethod
+    def coerce(cls, key, value):
+        if not isinstance(value, MutableMap):
+            if isinstance(value, dict):
+                return MutableMap(value)
+            return Mutable.coerce(key, value)
+        return value
+
+    def __setitem__(self, key, value):
+        dict.__setitem__(self, key, value)
+        self.changed()
+
+    def __delitem__(self, key):
+        dict.__delitem__(self, key)
+        self.changed()
+
+
+class EncryptedBinary(TypeDecorator):
+    impl = LargeBinary
+    cache_ok = False
+    context = None
+
+    def process_bind_param(self, value, dialect):
+        if value is not None:
+            value = self.context.encrypt(value)
+        return value
+
+    def process_result_value(self, value, dialect):
+        if value is not None:
+            value = self.context.decrypt(value)
+        return value
+
+
+class EncryptedString(TypeDecorator):
+    impl = String
+    cache_ok = False
+    encoding = 'utf-8'
+    context = None
+
+    def process_bind_param(self, value, dialect):
+        if value is not None:
+            encoded_value = bytes(value, encoding=self.encoding)
+            value = self.context.encrypt(encoded_value)
+        return value
+
+    def process_result_value(self, value, dialect):
+        if value is not None:
+            encoded_value = self.context.decrypt(value)
+            value = bytes.decode(encoded_value, encoding=self.encoding)
+        return value
+
+
+class HashedString(TypeDecorator):
+    impl = String
+    cache_ok = False
+
+    def process_bind_param(self, value, dialect):
+        if value is not None:
+            value = hash_text(value)
+        return value
+
+
+class UTCDateTime(TypeDecorator):
+    impl = DateTime
+    cache_ok = True
+
+    def process_bind_param(self, value, dialect):
+        if value is not None:
+            value = value.astimezone(UTC).replace(tzinfo=None)
+        return value
+
+    def process_result_value(self, value, dialect):
+        if value is not None:
+            value = value.replace(tzinfo=UTC)
+        return value
+
+
+def get_database_engine(database_uri):
+    if database_uri.startswith('sqlite') and not database_uri.endswith('://'):
+        database_path = Path(database_uri.split(':///', maxsplit=1)[1])
+        database_folder = database_path.parent
+        database_folder.mkdir(parents=True, exist_ok=True)
+    return create_async_engine(database_uri)
+
+
+def define_get_database_session(database_engine):
+    return async_sessionmaker(
+        database_engine,
+        autoflush=False,
+        expire_on_commit=False)
+
+
+async def make_tables(database_engine, database_metadata):
+    async with database_engine.begin() as database_connection:
+        await database_connection.run_sync(database_metadata.create_all)

crosscompute_macros-0.1.0/crosscompute_macros/starlette.py

@@ -0,0 +1,46 @@
+import asyncio
+import json
+from logging import getLogger
+
+from starlette.templating import Jinja2Templates
+
+
+class TemplateResponseFactory(Jinja2Templates):
+
+    def __init__(self, environment, context_processors=None):
+        'Assume nothing about the template environment.'
+        self.env = environment
+        self.context_processors = context_processors or []
+
+
+async def yield_map_while_connected(websocket, timeout_in_seconds=1):
+    async for x in yield_packet_while_connected(websocket, timeout_in_seconds):
+        if x and 'text' in x:
+            try:
+                x = json.loads(x['text'])
+            except json.JSONDecodeError:
+                x = {}
+            else:
+                if not isinstance(x, dict):
+                    x = {}
+        else:
+            x = {}
+        yield x
+
+
+async def yield_packet_while_connected(websocket, timeout_in_seconds=1):
+    while True:
+        try:
+            packet = await asyncio.wait_for(
+                websocket.receive(), timeout=timeout_in_seconds)
+        except TimeoutError:
+            yield
+        except RuntimeError:
+            break
+        else:
+            if packet['type'] == 'websocket.disconnect':
+                break
+            yield packet
+
+
+L = getLogger(__name__)

crosscompute_macros-0.1.0/crosscompute_macros/text.py

@@ -0,0 +1,64 @@
+import re
+
+
+def format_name(text):
+    return normalize_key(
+        text,
+        word_separator=' ',
+        separate_camel_case=True,
+        separate_letter_digit=True,
+        process_text=title_conservatively)
+
+
+def format_slug(text):
+    return normalize_key(
+        text,
+        word_separator='-')
+
+
+def normalize_key(
+        key,
+        word_separator=' ',
+        separate_camel_case=False,
+        separate_letter_digit=False,
+        process_text=str.lower):
+    '''Normalize key using a variation of the method described in http://stackoverflow.com/a/1176023/192092.
+
+    ONETwo   one two
+    OneTwo   one two
+    one-two  one two
+    one_two  one two
+    one2     one 2
+    1two     1 two
+    '''
+    if separate_camel_case:
+        key = UPPER_LOWER_PATTERN.sub(r'\1 \2', key)
+        key = LOWER_UPPER_PATTERN.sub(r'\1 \2', key)
+    if separate_letter_digit:
+        key = LETTER_DIGIT_PATTERN.sub(r'\1 \2', key)
+        key = DIGIT_LETTER_PATTERN.sub(r'\1 \2', key)
+    word_separators = [r'\W_']
+    if word_separator not in word_separators:
+        word_separators.append(word_separator)
+    word_separator_expression = '[' + ''.join(word_separators) + ']'
+    word_separator_pattern = re.compile(word_separator_expression)
+    key = word_separator_pattern.sub(' ', key)
+    key = compact_whitespace(key)
+    key = key.replace(' ', word_separator)
+    return process_text(key)
+
+
+def compact_whitespace(string):
+    return WHITESPACE_PATTERN.sub(' ', string).strip()
+
+
+def title_conservatively(text):
+    'Title camelCase as CamelCase.'
+    return ' '.join(_[0].upper() + _[1:] for _ in text.split(' '))
+
+
+UPPER_LOWER_PATTERN = re.compile(r'(.)([A-Z][a-z]+)')
+LOWER_UPPER_PATTERN = re.compile(r'([a-z0-9])([A-Z])')
+LETTER_DIGIT_PATTERN = re.compile(r'([A-Za-z])([0-9])')
+DIGIT_LETTER_PATTERN = re.compile(r'([0-9])([A-Za-z])')
+WHITESPACE_PATTERN = re.compile(r'\s+', re.MULTILINE)

crosscompute_macros-0.1.0/crosscompute_macros/web.py

@@ -0,0 +1,152 @@
+import socket
+from contextlib import suppress
+from functools import partial
+from os.path import dirname
+from random import randint
+
+import aiofiles
+from aiohttp import request
+from aiohttp.client_exceptions import ClientError
+
+from .disk import (
+    make_folder)
+from .error import (
+    DiskError,
+    WebConnectionError,
+    WebRequestError)
+from .iterable import (
+    drop_null_values)
+
+
+async def upload(
+        target_uri, source_path, client_session=None, chunk_size=1024 * 1024,
+        method='PUT', headers=None, params=None):
+    fetch = _get_fetch(client_session, method)
+    if headers:
+        drop_null_values(headers)
+    try:
+        async with aiofiles.open(source_path, mode='rb') as f:
+            async def yield_chunk(chunk_size):
+                while (chunk := await f.read(chunk_size)):
+                    yield chunk
+            async with fetch(
+                url=target_uri,
+                data=yield_chunk(chunk_size),
+                headers=headers,
+                params=params,
+            ) as response:
+                response_status = response.status
+                response_text = await response.text()
+                if response_status != 200:
+                    raise WebRequestError(
+                        response_text, uri=target_uri, code=response_status)
+    except OSError as e:
+        x = 'path is not accessible'
+        raise DiskError(x, path=source_path) from e
+    except ClientError as e:
+        raise WebConnectionError(e, uri=target_uri) from e
+    return response_text
+
+
+async def download(
+        target_path, source_uri, client_session=None, chunk_size=1024 * 1024,
+        method='GET', headers=None, params=None):
+    fetch = _get_fetch(client_session, method)
+    if headers:
+        drop_null_values(headers)
+    try:
+        async with fetch(
+            url=source_uri, headers=headers, params=params,
+        ) as response:
+            response_status = response.status
+            if response_status != 200:
+                raise WebRequestError(
+                    await response.text(), uri=source_uri,
+                    code=response_status)
+            await make_folder(dirname(target_path))
+            async with aiofiles.open(target_path, mode='wb') as f:
+                async for chunk in response.content.iter_chunked(chunk_size):
+                    await f.write(chunk)
+    except ClientError as e:
+        raise WebConnectionError(e, uri=source_uri) from e
+
+
+async def make_error(Error, message_text, response=None, error=None):
+    error_texts = [message_text]
+    kwargs = {}
+    if response:
+        response_text = (await response.text()).strip()
+        if response_text:
+            error_texts.append(response_text)
+        kwargs['uri'] = response.url
+        kwargs['code'] = response.status
+    elif error:
+        error_text = str(error).strip()
+        if error_text:
+            error_texts.append(error_text)
+        kwargs['uri'] = error.uri
+        if hasattr(error, 'code'):
+            kwargs['code'] = error.code
+    return Error('; '.join(error_texts), **kwargs)
+
+
+def escape_quotes_html(x):
+    with suppress(AttributeError):
+        x = x.replace('"', '"').replace("'", ''')
+    return x
+
+
+def escape_quotes_js(x):
+    with suppress(AttributeError):
+        x = x.replace('"', '\\"').replace("'", "\\'")
+    return x
+
+
+def find_open_port(
+        default_port=None,
+        minimum_port=1024,
+        maximum_port=65535):
+
+    def get_new_port():
+        return randint(minimum_port, maximum_port)  # noqa: S311
+
+    port = default_port or get_new_port()
+    port_count = maximum_port - minimum_port + 1
+    closed_ports = set()
+    while True:
+        if not is_port_in_use(port):
+            break
+        closed_ports.add(port)
+        if len(closed_ports) == port_count:
+            x = (
+                'could not find an open port in '
+                f'[{minimum_port}, {maximum_port}]')
+            raise OSError(x)
+        port = get_new_port()
+    return port
+
+
+def is_port_in_use(port):
+    # https://stackoverflow.com/a/52872579
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        is_in_use = s.connect_ex(('127.0.0.1', int(port))) == 0
+    return is_in_use
+
+
+def replace_localhost(netloc):
+    domain_name = netloc.split(':')[0]
+    if domain_name == 'localhost':
+        netloc = netloc.replace('localhost', '127.0.0.1')
+    return netloc
+
+
+def _get_fetch(client_session, method_name):
+    method_name = method_name.lower()
+    if client_session:
+        fetch = getattr(client_session, method_name)
+    else:
+        fetch = partial(request, method=method_name)
+    return fetch
+
+
+# ruff: noqa: PLR2004

crosscompute_macros-0.1.0/crosscompute_macros/yaml.py

@@ -0,0 +1,59 @@
+import aiofiles
+from ruamel.yaml import YAML
+from ruamel.yaml.error import YAMLError
+
+from .error import DiskError, FormattingError, ParsingError
+
+
+async def load_raw_yaml(path, with_comments=False):
+    yaml = YAML(typ='rt' if with_comments else 'safe')
+    try:
+        async with aiofiles.open(path, mode='rt') as f:
+            dictionary = yaml.load(await f.read())
+    except OSError as e:
+        x = f'path is not accessible; {e}'
+        raise DiskError(x, path=path) from e
+    except YAMLError as e:
+        x = f'file is not yaml; {e}'
+        raise ParsingError(x, path=path) from e
+    return dictionary or {}
+
+
+async def save_raw_yaml(path, x, with_comments=False):
+    yaml = YAML(typ=['rt' if with_comments else 'safe', 'bytes'])
+    try:
+        async with aiofiles.open(path, mode='wb') as f:
+            await f.write(yaml.dump_to_bytes(x))
+    except OSError as e:
+        x = f'path is not accessible; {e}'
+        raise DiskError(x, path=path) from e
+    except YAMLError as e:
+        x = f'value cannot be yaml; {e}'
+        raise FormattingError(x, path=path) from e
+
+
+def sync_load_raw_yaml(path, with_comments=False):
+    yaml = YAML(typ='rt' if with_comments else 'safe')
+    try:
+        with path.open(mode='rt') as f:
+            dictionary = yaml.load(f.read())
+    except OSError as e:
+        x = f'path is not accessible; {e}'
+        raise DiskError(x, path=path) from e
+    except YAMLError as e:
+        x = f'file is not valid yaml; {e}'
+        raise ParsingError(x, path=path) from e
+    return dictionary or {}
+
+
+def sync_save_raw_yaml(path, x, with_comments=False):
+    yaml = YAML(typ=['rt' if with_comments else 'safe', 'bytes'])
+    try:
+        with path.open(mode='wb') as f:
+            f.write(yaml.dump_to_bytes(x))
+    except OSError as e:
+        x = f'path is not accessible; {e}'
+        raise DiskError(x, path=path) from e
+    except YAMLError as e:
+        x = f'value cannot be yaml; {e}'
+        raise FormattingError(x, path=path) from e

crosscompute_macros-0.1.0/pyproject.toml

@@ -0,0 +1,52 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "crosscompute-macros"
+version = "0.1.0"
+dependencies = []
+requires-python = ">= 3.10"
+authors = [
+    {name = "Roy Hyunjin Han", email = "rhh@crosscompute.com"},
+]
+maintainers = []
+description = "Reusable classes and functions needed by crosscompute packages"
+readme = "README.md"
+keywords = []
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+]
+
+[project.optional-dependencies]
+disk = ["aiofiles >= 24.1.0"]
+iterable = []
+jinja = ["jinja2"]
+log = []
+markdown = ["markdown2"]
+msgpack = ["msgpack"]
+security = ["argon2-cffi"]
+shapely = ["aiofiles", "shapely"]
+sqlalchemy = ["sqlalchemy"]
+starlette = ["starlette"]
+web = ["aiofiles", "aiohttp[speedups] >= 3.12.12"]
+yaml = ["ruamel.yaml", "ruamel.yaml.bytes"]
+development = ["pre-commit"]
+test = ["crosscompute_macros[disk]", "pytest-asyncio", "pytest-cov", "pytest-xdist"]
+all = ["crosscompute_macros[disk,iterable,jinja,log,markdown,msgpack,security,shapely,sqlalchemy,starlette,web,yaml]"]
+
+[project.urls]
+# Homepage = ""
+# Documentation = ""
+Repository = "https://github.com/crosscompute/crosscompute-macros"
+Issues = "https://github.com/crosscompute/crosscompute-macros/issues"
+# Changelog = ""
+
+[tool.hatch.build]
+packages = ["sources/crosscompute_macros"]
+
+[tool.pytest.ini_options]
+asyncio_default_fixture_loop_scope = "session"