dsgrid-toolkit 0.3.3__cp313-cp313-win_amd64.whl

dsgrid/utils/files.py

@@ -0,0 +1,179 @@
+"""File utility functions"""
+
+import hashlib
+import logging
+import os
+import json
+import shutil
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Any
+
+import json5
+
+
+logger = logging.getLogger(__name__)
+
+
+def compute_file_hash(filename) -> str:
+    """Compute a hash of the contents of a file.
+
+    Parameters
+    ----------
+    filename : str
+
+    Returns
+    -------
+    str
+        hash in the form of a hex number converted to a string
+
+    """
+    return compute_hash(Path(filename).read_bytes())
+
+
+def compute_hash(text: bytes) -> str:
+    hash_obj = hashlib.sha256()
+    hash_obj.update(text)
+    return hash_obj.hexdigest()
+
+
+def delete_if_exists(path: Path | str) -> None:
+    """Delete a file or directory if it exists."""
+    path = Path(path) if isinstance(path, str) else path
+    if path.exists():
+        if path.is_dir():
+            shutil.rmtree(path)
+        else:
+            path.unlink()
+
+
+def dump_data(data, filename, **kwargs) -> None:
+    """Dump data to the filename.
+    Supports JSON, TOML, or custom via kwargs.
+
+    Parameters
+    ----------
+    data : dict
+        data to dump
+    filename : str
+        file to create or overwrite
+
+    """
+    mod = _get_module_from_extension(filename, **kwargs)
+    with open(filename, "w") as f_out:
+        mod.dump(data, f_out, **kwargs)
+
+    logger.debug("Dumped data to %s", filename)
+
+
+def load_data(filename, **kwargs) -> dict[str, Any]:
+    """Load data from the file.
+    Supports JSON, JSON5, or custom via kwargs.
+
+    Parameters
+    ----------
+    filename : str
+
+    Returns
+    -------
+    dict
+
+    """
+    mod = _get_module_from_extension(filename, **kwargs)
+    with open(filename) as f_in:
+        try:
+            data = mod.load(f_in)
+        except Exception:
+            logger.exception("Failed to load data from %s", filename)
+            raise
+
+    logger.debug("Loaded data from %s", filename)
+    return data
+
+
+def dump_json_file(data, filename, indent=None) -> None:
+    """Dump data to the JSON or JSON5 filename."""
+    dump_data(data, filename, indent=indent)
+
+
+def load_json_file(filename: Path | str) -> dict[str, Any]:
+    """Load data from the JSON or JSON5 file."""
+    return load_data(filename)
+
+
+def dump_line_delimited_json(data, filename, mode="w"):
+    """Dump a list of objects to the file as line-delimited JSON.
+
+    Parameters
+    ----------
+    data : list
+    filename : str
+    mode : str
+        Mode to use for opening the file, defaults to "w"
+
+    """
+    with open(filename, mode, encoding="utf-8-sig") as f_out:
+        for obj in data:
+            f_out.write(json.dumps(obj))
+            f_out.write("\n")
+
+    logger.debug("Dumped data to %s", filename)
+
+
+def load_line_delimited_json(filename):
+    """Load data from the file that is stored as line-delimited JSON.
+
+    Parameters
+    ----------
+    filename : str
+
+    Returns
+    -------
+    dict
+
+    """
+    objects = []
+    with open(filename, encoding="utf-8-sig") as f_in:
+        for i, line in enumerate(f_in):
+            text = line.strip()
+            if not text:
+                continue
+            try:
+                objects.append(json.loads(text))
+            except Exception:
+                logger.exception("Failed to decode line number %s in %s", i, filename)
+                raise
+
+    logger.debug("Loaded data from %s", filename)
+    return objects
+
+
+@contextmanager
+def in_other_dir(path: Path):
+    """Change to another directory while user code runs.
+
+    Parameters
+    ----------
+    path : Path
+    """
+    orig = os.getcwd()
+    os.chdir(path)
+    try:
+        yield
+    finally:
+        os.chdir(orig)
+
+
+def _get_module_from_extension(filename, **kwargs):
+    ext = os.path.splitext(filename)[1].lower()
+    if ext == ".json":
+        mod = json
+    elif ext == ".json5":
+        mod = json5
+    elif "mod" in kwargs:
+        mod = kwargs["mod"]
+    else:
+        msg = f"Unsupported extension {filename}"
+        raise NotImplementedError(msg)
+
+    return mod

dsgrid/utils/filters.py

@@ -0,0 +1,125 @@
+import numpy as np
+from dsgrid.exceptions import DSGInvalidParameter
+
+ACCEPTED_OPS = ["==", "!=", "contains", "not contains"]
+
+
+def transform_and_validate_filters(filter_expressions):
+    """
+    Validate whether an operation exists, return tranformed/formatted filters
+
+    Parameters
+    ------------
+    filter_expressions : set(str)
+       - each expression takes the format 'field operation value'
+       - 'field' and 'value' are case-insenstive and can accept spaces since expression is split by 'operation'
+
+    Returns
+    --------
+    transformed_filters : list
+        list of validated and transformed filtering fields
+    """
+
+    transformed_filters = []
+    for expr_str in filter_expressions:
+        check_ops = []
+        op = None
+        for opp in ACCEPTED_OPS:
+            check_ops.append(opp in expr_str)
+            if opp in expr_str:
+                op = opp
+
+        if np.sum(check_ops) < 1:
+            msg = f"invalid operation detected, valid ops: {ACCEPTED_OPS}"
+            raise DSGInvalidParameter(msg)
+        elif np.sum(check_ops) > 2:
+            msg = f"too many operations detected, choose one of the valid ops: {ACCEPTED_OPS}"
+            raise DSGInvalidParameter(msg)
+
+        fields = [x.strip() for x in expr_str.split(op) if x != ""]
+
+        if len(fields) < 2:
+            msg = (
+                f"filter expression: '{expr_str}' contains too few arguments, must be in the format 'field operation value' "
+                "(ex: 'Submitter == username')"
+            )
+            raise DSGInvalidParameter(msg)
+        elif len(fields) > 2:
+            msg = (
+                f"filter expression: '{expr_str}' contains too many arguments, must be in the format 'field operation value', "
+                "(ex: 'Submitter == username')"
+            )
+            raise DSGInvalidParameter(msg)
+
+        field = fields[0]
+        value = fields[1]
+        transformed_filters.append(
+            [field.lower(), op, value.lower()]
+        )  # to accept case-insensitive comparison
+
+    return transformed_filters
+
+
+def matches_filters(row, field_to_index, transformed_filters):
+    """
+    Validate field name in transformed filter_expressions, return TRUE for rows matching all filters
+
+    Parameters
+    ------------
+    row : str
+        row in `list` registry table (manager.show())
+    field_to_index : dict
+        key = column names, val = column index, in registry table (or manager.show())
+    transformed_filters : list
+        transformed/formatted fields for filtering rows
+
+    Returns
+    --------
+    bool
+        return TRUE for rows matching all filters
+    """
+
+    field_to_index_lower = dict(
+        (k.lower(), v) for k, v in field_to_index.items()
+    )  # to accept case-insensitive comparison
+
+    for tfilter in transformed_filters:
+        [field, op, value] = tfilter
+        if field not in field_to_index_lower:
+            msg = f"field='{field}' is not a valid column name, valid fields: {list(field_to_index.keys())}"
+            raise DSGInvalidParameter(msg)
+        obj_val = row[field_to_index_lower[field]].lower()  # to accept case-insensitive comparison
+        if not matches_filter(val=obj_val, op=op, required_value=value):
+            return False
+    return True
+
+
+def matches_filter(val, op, required_value):
+    """
+    check if table content matches filtered value
+
+    Parameters
+    ------------
+    val : str
+        value from registry table to be compared
+    op : str
+        filtering operation to be performed
+    required_value : str
+        value to match from filter_expression
+
+    Returns
+    --------
+    bool
+        return TRUE if vtable content matches filtered value, FALSE otherwise
+    """
+
+    if op == "==":
+        return val == required_value
+    elif op == "!=":
+        return val != required_value
+    elif op == "contains":
+        return required_value in val
+    elif op == "not contains":
+        return required_value not in val
+    else:
+        assert False, op

dsgrid/utils/id_remappings.py

@@ -0,0 +1,100 @@
+"""Contains utility functions to map to/from dimension/mapping names and IDs."""
+
+from dsgrid.registry.dimension_registry_manager import DimensionRegistryManager
+from dsgrid.utils.files import dump_data, load_data
+
+
+def map_dimension_names_to_ids(dimension_mgr: DimensionRegistryManager):
+    mapping = {}
+    for dim in dimension_mgr.db.dimensions:
+        if dim["name"] in mapping:
+            assert mapping[dim["name"]] == dim["dimension_id"], dim
+        mapping[dim["name"]] = dim["dimension_id"]
+    return mapping
+
+
+def map_dimension_ids_to_names(dimension_mgr):
+    mapping = {}
+    for dim in dimension_mgr.db.dimensions:
+        assert dim["dimension_id"] not in mapping, dim
+        mapping[dim["dimension_id"]] = dim["name"]
+    return mapping
+
+
+def map_dimension_mapping_names_to_ids(dimension_mapping_mgr, dim_id_to_name):
+    mapping = {}
+    for dmap in dimension_mapping_mgr.db.dimension_mappings:
+        key = (
+            dim_id_to_name[dmap["from_dimension"]["dimension_id"]],
+            dim_id_to_name[dmap["to_dimension"]["dimension_id"]],
+        )
+        if key in mapping:
+            assert mapping[key] == dmap["mapping_id"], dmap
+        mapping[key] = dmap["mapping_id"]
+    return mapping
+
+
+def replace_dimension_names_with_current_ids(filename, mappings):
+    data = load_data(filename)
+    assert isinstance(data, dict)
+
+    def perform_replacements(mappings, dimensions):
+        changed = False
+        for ref in dimensions:
+            if "name" in ref:
+                ref["dimension_id"] = mappings[ref.pop("name")]
+                changed = True
+        return changed
+
+    changed = False
+    if "dimension_references" in data:
+        # This is True for a dataset config file.
+        if perform_replacements(mappings, data["dimension_references"]):
+            changed = True
+
+    if "dimensions" in data and "base_dimension_references" in data["dimensions"]:
+        # This is True for a project config file.
+        if perform_replacements(mappings, data["dimensions"]["base_dimension_references"]):
+            changed = True
+        if perform_replacements(mappings, data["dimensions"]["supplemental_dimension_references"]):
+            changed = True
+
+    if "mappings" in data:
+        # This is True for a dimension mappings file.
+        for mapping in data["mappings"]:
+            if perform_replacements(
+                mappings, [mapping["from_dimension"], mapping["to_dimension"]]
+            ):
+                changed = True
+
+    if changed:
+        dump_data(data, filename, indent=2)
+
+
+def replace_dimension_mapping_names_with_current_ids(filename, mappings):
+    data = load_data(filename)
+    assert isinstance(data, dict)
+
+    def perform_replacements(mappings, references):
+        changed = False
+        for ref in references:
+            if "mapping_names" in ref:
+                item = ref.pop("mapping_names")
+                ref["mapping_id"] = mappings[(item["from"], item["to"])]
+                changed = True
+        return changed
+
+    changed = False
+    if "dimension_mappings" in data:
+        # This is True for a project config file.
+        refs = data["dimension_mappings"]["base_to_supplemental_references"]
+        if perform_replacements(mappings, refs):
+            changed = True
+
+    if "references" in data:
+        # This is True for a dataset-to-project dimension mapping reference file.
+        if perform_replacements(mappings, data["references"]):
+            changed = True
+
+    if changed:
+        dump_data(data, filename, indent=2)

dsgrid/utils/py_expression_eval/LICENSE

@@ -0,0 +1,19 @@
+The MIT License (MIT)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dsgrid/utils/py_expression_eval/README.md

@@ -0,0 +1,8 @@
+We cloned this code from https://github.com/axiacore/py-expression-eval and kept the LICENSE.
+
+We needed a modification that would interpret `|` as a union for Python sets and SQL
+queries. This modification almost certainly does not belong in the main repository, and so
+we are storing it here.
+
+Note to future dsgrid developers: Keep this in sync with updates from the main repository.
+The code here is based on commit 02dc9de711da50735890bef38a938ce6db9737b9.