fable-client 0.4.1__py3-none-any.whl

fable_client/__init__.py

@@ -0,0 +1,5 @@
+__all__ = ["FableClient", "FableError", "estimate", "types"]
+
+from ._client import FableClient, FableError
+from . import _estimate as estimate
+from . import types

fable_client/_cli.py

@@ -0,0 +1,439 @@
+import contextlib
+import csv
+import itertools
+import json
+from pathlib import Path
+from typing import Any, TypeVar
+
+import click
+import httpx
+from fable_model import (
+    BitVectorEntity,
+    BaseMatchRequest,
+    MatchMethod,
+    BaseTransformRequest,
+    AttributeValueEntity,
+    BaseMaskRequest,
+    TransformConfig,
+    EmptyValueHandling,
+    GlobalTransformerConfig,
+    NormalizationTransformer,
+    WeightedAttributeConfig,
+)
+from pydantic import BaseModel
+
+from ._client import FableClient
+from ._estimate import compute_attribute_stats
+from ._model import FakerGeneratorConfig, FakerGeneratorSpec
+
+
+def create_client(ctx: click.Context) -> FableClient:
+    return FableClient(client=httpx.Client(base_url=ctx.obj["BASE_URL"], timeout=int(ctx.obj["TIMEOUT_SECS"])))
+
+
+def read_bit_vector_entity_file(reader: csv.DictReader, id_column: str, value_column: str):
+    """
+    Read a CSV file containing bit vector entities.
+
+    Args:
+        reader: CSV dict reader instance
+        id_column: name of ID column
+        value_column: name of value column
+
+    Returns:
+        list of bit vector entities
+    """
+    return [BitVectorEntity(id=row[id_column], value=row[value_column]) for row in reader]
+
+
+def read_attribute_value_entity_file(reader: csv.DictReader, id_column: str):
+    field_names: list[str] = list(reader.fieldnames)
+
+    if id_column not in field_names:
+        click.echo(f"Column {id_column} not found in CSV file", err=True)
+        raise click.exceptions.Exit(1)
+
+    def _row_to_entity(row: dict[str, Any]):
+        return AttributeValueEntity(
+            id=str(row[id_column]),
+            attributes={
+                attribute_name: str(attribute_value)
+                for attribute_name, attribute_value in row.items()
+                if attribute_name != id_column
+            },
+        )
+
+    entities = list(_row_to_entity(row) for row in reader)
+
+    return field_names, entities
+
+
+_M = TypeVar("_M", bound=BaseModel)
+
+
+def parse_json_file_into(ctx: click.Context, path: str | Path, model: type[_M]) -> _M:
+    with open(path, mode="r", encoding=ctx.obj["ENCODING"]) as f:
+        return model(**json.load(f))
+
+
+@contextlib.contextmanager
+def read_csv_file(ctx: click.Context, path: str | Path, mode: str = "r"):
+    with open(path, mode=mode, encoding=ctx.obj["ENCODING"], newline="") as f:
+        yield csv.DictReader(f, delimiter=ctx.obj["DELIMITER"])
+
+
+@contextlib.contextmanager
+def write_csv_file(
+    ctx: click.Context, path: str | Path, fieldnames: list[str], mode: str = "w", write_header: bool = True
+):
+    with open(path, mode=mode, encoding=ctx.obj["ENCODING"], newline="") as f:
+        writer = csv.DictWriter(f, delimiter=ctx.obj["DELIMITER"], fieldnames=fieldnames)
+
+        if write_header:
+            writer.writeheader()
+
+        yield writer
+
+
+@click.group()
+@click.pass_context
+@click.option("--base-url", default="http://localhost:8080", help="base URL to HTTP-based PPRL service")
+@click.option(
+    "-b", "--batch-size", type=click.IntRange(min=1), default=1_000, help="amount of bit vectors to match at a time"
+)
+@click.option("--timeout-secs", type=click.IntRange(min=1), default=30, help="seconds until a request times out")
+@click.option("--delimiter", type=str, default=",", help="column delimiter for CSV files")
+@click.option("--encoding", type=str, default="utf-8", help="character encoding for files")
+def app(ctx: click.Context, base_url: str, batch_size: int, timeout_secs: int, delimiter: str, encoding: str):
+    ctx.ensure_object(dict)
+    ctx.obj["BASE_URL"] = base_url
+    ctx.obj["BATCH_SIZE"] = batch_size
+    ctx.obj["TIMEOUT_SECS"] = timeout_secs
+    ctx.obj["DELIMITER"] = delimiter
+    ctx.obj["ENCODING"] = encoding
+
+
+@app.command()
+@click.pass_context
+@click.argument("base_match_request_file_path", type=click.Path(exists=True, path_type=Path))
+@click.argument("vector_file_path", type=click.Path(exists=True, path_type=Path, dir_okay=False), nargs=-1)
+@click.argument("output_file_path", type=click.Path(path_type=Path, dir_okay=False))
+@click.option("--id-column", type=str, default="id", help="column name in input CSV file containing vector ID")
+@click.option("--value-column", type=str, default="value", help="column name in input CSV file containing vector value")
+def match(
+    ctx: click.Context,
+    base_match_request_file_path: Path,
+    vector_file_path: tuple[Path, ...],
+    output_file_path: Path,
+    id_column: str,
+    value_column: str,
+):
+    """
+    Match bit vectors from CSV files against each other.
+
+    BASE_MATCH_REQUEST_FILE_PATH is the path to a JSON file containing the base match request.
+    VECTOR_FILE_PATH is the path to a CSV file containing bit vectors.
+    At least two files must be specified.
+    OUTPUT_FILE_PATH is the path of the CSV file where the matches should be written to.
+    """
+    if len(vector_file_path) < 2:
+        click.echo("Must specify at least two CSV files containing vectors", err=True)
+        ctx.exit(1)
+
+    client = create_client(ctx)
+    base_match_request = parse_json_file_into(ctx, base_match_request_file_path, BaseMatchRequest)
+
+    batch_size = int(ctx.obj["BATCH_SIZE"])
+    file_count = len(vector_file_path)
+    vectors_per_file: list[list[BitVectorEntity]] = []
+
+    for p in vector_file_path:
+        with read_csv_file(ctx, p, mode="r") as reader:
+            vectors_per_file.append(read_bit_vector_entity_file(reader, id_column, value_column))
+
+    # check that all files have the same amount of entries
+    do_pairwise_matching = base_match_request.config.method == MatchMethod.pairwise
+
+    if do_pairwise_matching:
+        vector_lens = set(len(v) for v in vectors_per_file)
+
+        if len(vector_lens) != 1:
+            click.echo(
+                "All bit vector files must have the same amount of vectors for pairwise matching, got"
+                f"{', '.join([str(len(v) for v in vectors_per_file)])}"
+            )
+            ctx.exit(1)
+
+    with write_csv_file(
+        ctx,
+        output_file_path,
+        ["domain_id", "domain_file", "range_id", "range_file", "similarity"],
+        mode="w",
+        write_header=True,
+    ) as writer:
+        for domain_idx in range(0, file_count - 1):
+            for range_idx in range(domain_idx + 1, file_count):
+                # get domain and range vectors
+                domain_vectors, range_vectors = vectors_per_file[domain_idx], vectors_per_file[range_idx]
+                # these are tracked for user feedback
+                domain_file_path, range_file_path = vector_file_path[domain_idx], vector_file_path[range_idx]
+
+                # construct the starting indices for batch-wise processing
+                domain_start_idx = list(range(0, len(domain_vectors), batch_size))
+                range_start_idx = list(range(0, len(range_vectors), batch_size))
+
+                # when doing pairwise matching, matching will be performed row-wise
+                if do_pairwise_matching:
+                    idx_pairs = zip(domain_start_idx, range_start_idx)
+                # otherwise, cross-wise matching is performed
+                else:
+                    idx_pairs = itertools.product(domain_start_idx, range_start_idx)
+
+                with click.progressbar(
+                    idx_pairs, label=f"Matching bit vectors from {domain_file_path.name} and {range_file_path.name}"
+                ) as pbar:
+                    # iterate over pairs of starting indices for domain and range
+                    for idx_tpl in pbar:
+                        domain_idx, range_idx = idx_tpl[0], idx_tpl[1]
+
+                        # retrieve batch of vectors
+                        domain_vector_batch = domain_vectors[domain_idx : domain_idx + batch_size]
+                        range_vector_batch = range_vectors[range_idx : range_idx + batch_size]
+
+                        # and perform matching
+                        r = client.match(
+                            base_match_request.with_vectors(
+                                domain_lst=domain_vector_batch, range_lst=range_vector_batch
+                            )
+                        )
+
+                        writer.writerows(
+                            [
+                                {
+                                    "domain_id": m.domain.id,
+                                    "domain_file": domain_file_path.name,
+                                    "range_id": m.range.id,
+                                    "range_file": range_file_path.name,
+                                    "similarity": m.similarity,
+                                }
+                                for m in r.matches
+                            ]
+                        )
+
+
+@app.command()
+@click.pass_context
+@click.argument("base_transform_request_file_path", type=click.Path(exists=True, path_type=Path))
+@click.argument("entity_file_path", type=click.Path(exists=True, path_type=Path))
+@click.argument("output_file_path", type=click.Path(path_type=Path, dir_okay=False))
+@click.option("--entity-id-column", type=str, default="id", help="column name in entity CSV file containing ID")
+def transform(
+    ctx: click.Context,
+    base_transform_request_file_path: Path,
+    entity_file_path: Path,
+    output_file_path: Path,
+    entity_id_column: str,
+):
+    """
+    Perform pre-processing on a CSV file with entities.
+
+    BASE_TRANSFORM_REQUEST_FILE_PATH is the path to a JSON file containing the base transform request.
+    ENTITY_FILE_PATH is the path to the CSV file containing entities.
+    OUTPUT_FILE_PATH is the path of the CSV file where the pre-processed entities should be written to.
+    """
+    client = create_client(ctx)
+    base_transform_request = parse_json_file_into(ctx, base_transform_request_file_path, BaseTransformRequest)
+
+    # read entities
+    with read_csv_file(ctx, entity_file_path, mode="r") as reader:
+        field_names, entities = read_attribute_value_entity_file(reader, entity_id_column)
+
+    # create list of indices for batching
+    batch_size = int(ctx.obj["BATCH_SIZE"])
+    idx = list(range(0, len(entities), batch_size))
+
+    with (
+        write_csv_file(ctx, output_file_path, field_names, mode="w", write_header=True) as writer,
+        click.progressbar(idx, label="Transforming entities") as pbar,
+    ):
+        for i in pbar:
+            # create batch
+            entity_batch = entities[i : i + batch_size]
+            r = client.transform(base_transform_request.with_entities(entity_batch))
+
+            # write results
+            writer.writerows([{entity_id_column: entity.id, **entity.attributes} for entity in r.entities])
+
+
+@app.command()
+@click.pass_context
+@click.argument("base_mask_request_file_path", type=click.Path(exists=True, path_type=Path))
+@click.argument("entity_file_path", type=click.Path(exists=True, path_type=Path))
+@click.argument("output_file_path", type=click.Path(dir_okay=False, file_okay=True, path_type=Path))
+@click.option("--entity-id-column", type=str, default="id", help="column name in entity CSV file containing ID")
+@click.option(
+    "--entity-value-column", type=str, default="value", help="column name in output CSV file containing vector value"
+)
+def mask(
+    ctx: click.Context,
+    base_mask_request_file_path: Path,
+    entity_file_path: Path,
+    output_file_path: Path,
+    entity_id_column: str,
+    entity_value_column: str,
+):
+    """
+    Mask a CSV file with entities.
+
+    BASE_MASK_REQUEST_FILE_PATH is the path to a JSON file containing the base mask request.
+    ENTITY_FILE_PATH is the path to the CSV file containing entities.
+    OUTPUT_FILE_PATH is the path of the CSV file where the masked entities should be written to.
+    """
+    client = create_client(ctx)
+    base_mask_request = parse_json_file_into(ctx, base_mask_request_file_path, BaseMaskRequest)
+
+    with read_csv_file(ctx, entity_file_path, mode="r") as reader:
+        _, entities = read_attribute_value_entity_file(reader, entity_id_column)
+
+    # create list of indices for batching
+    batch_size = int(ctx.obj["BATCH_SIZE"])
+    idx = list(range(0, len(entities), batch_size))
+
+    with (
+        write_csv_file(
+            ctx, output_file_path, [entity_id_column, entity_value_column], mode="w", write_header=True
+        ) as writer,
+        click.progressbar(idx, label="Masking entities") as pbar,
+    ):
+        for i in pbar:
+            # create batch
+            entity_batch = entities[i : i + batch_size]
+            r = client.mask(base_mask_request.with_entities(entity_batch))
+
+            # write results
+            writer.writerows(
+                [{entity_id_column: entity.id, entity_value_column: entity.value} for entity in r.entities]
+            )
+
+
+@app.group()
+def estimate():
+    """Estimate attribute weights based on randomly generated data."""
+    pass
+
+
+def common_estimate_options(fn):
+    fn = click.option(
+        "--base-transform-request-file-path",
+        type=click.Path(exists=True, path_type=Path),
+        help="path to file containing attribute-level and global transformer definitions",
+    )(fn)
+    fn = click.option(
+        "-q",
+        "--token-size",
+        type=click.IntRange(min=2),
+        default=2,
+        help="size of tokens to split each attribute value into",
+    )(fn)
+    fn = click.option(
+        "-p", "--padding", type=str, default="_", help="padding to use when splitting attribute values into tokens"
+    )(fn)
+
+    return fn
+
+
+@estimate.command()
+@click.pass_context
+@click.argument("GENERATOR_CONFIG_FILE_PATH", type=click.Path(exists=True, path_type=Path))
+@click.argument("ATTRIBUTE_CONFIG_OUTPUT_FILE_PATH", type=click.Path(path_type=Path))
+@common_estimate_options
+def faker(
+    ctx: click.Context,
+    generator_config_file_path: Path,
+    attribute_config_output_file_path: Path,
+    base_transform_request_file_path: Path | None,
+    token_size: int,
+    padding: str,
+):
+    """
+    Estimate attribute weights based on data generated by Faker.
+
+    GENERATOR_CONFIG_FILE_PATH is the file which defines the Faker providers to use.
+    ATTRIBUTE_CONFIG_OUTPUT_FILE_PATH is the path to the file where the attribute weights will be written to.
+    """
+
+    try:
+        from faker import Faker
+    except ImportError:
+        click.echo("Faker not found, install it with `pip install fable_client[faker]`", err=True)
+        raise click.exceptions.Exit(1)
+
+    # set up vars
+    client = create_client(ctx)
+    faker_generator_config = parse_json_file_into(ctx, generator_config_file_path, FakerGeneratorConfig)
+    batch_size = int(ctx.obj["BATCH_SIZE"])
+
+    # load base transform request, if specified
+    if base_transform_request_file_path is None:
+        base_transform_request = BaseTransformRequest(
+            config=TransformConfig(empty_value=EmptyValueHandling.skip),
+            global_transformers=GlobalTransformerConfig(before=[NormalizationTransformer()]),
+        )
+    else:
+        base_transform_request = parse_json_file_into(ctx, base_transform_request_file_path, BaseTransformRequest)
+
+    # create faker instance
+    fake = Faker(faker_generator_config.locale)
+    fake.seed_instance(faker_generator_config.seed)
+
+    # creates a callable function from a generator specification
+    def _create_faker_generator(generator: FakerGeneratorSpec):
+        generator_fn = getattr(fake, generator.function_name, None)
+
+        # check that the function actually exists
+        if not callable(generator_fn):
+            click.echo(f"Invalid faker function: {generator.function_name}", err=True)
+            raise click.exceptions.Exit(1)
+
+        # wrapper function with no args that calls the generator
+        def _generate():
+            return str(generator_fn(**generator.args))
+
+        return _generate
+
+    # attribute name -> generator fn
+    attribute_name_to_generator_fn = {
+        generator.attribute_name: _create_faker_generator(generator) for generator in faker_generator_config.generators
+    }
+
+    entities = [
+        # loop by amount of entities to generate
+        AttributeValueEntity(
+            # loop by attribute name and generator function pairs
+            id=str(i),
+            attributes={
+                attribute_name: generator_fn()
+                for attribute_name, generator_fn in attribute_name_to_generator_fn.items()
+            },
+        )
+        for i in range(faker_generator_config.count)
+    ]
+
+    # compute stats for each attribute
+    attribute_name_to_stats = compute_attribute_stats(
+        client, entities, base_transform_request, token_size, padding, batch_size
+    )
+
+    # create list of attribute configs
+    attribute_configs = [
+        WeightedAttributeConfig(
+            attribute_name=attribute_name,
+            weight=attribute_stats["ngram_entropy"],
+            average_token_count=attribute_stats["average_tokens"],
+        )
+        for attribute_name, attribute_stats in attribute_name_to_stats.items()
+    ]
+
+    # export them
+    with open(attribute_config_output_file_path, mode="w", encoding="utf-8") as f:
+        json.dump([cfg.model_dump(mode="json", exclude_none=True) for cfg in attribute_configs], f, indent=2)

fable_client/_client.py

@@ -0,0 +1,91 @@
+from json import JSONDecodeError
+from typing import TypeVar
+
+import httpx
+from fable_model import (
+    VectorMatchRequest,
+    VectorMatchResponse,
+    EntityTransformRequest,
+    EntityTransformResponse,
+    EntityMaskRequest,
+    EntityMaskResponse,
+)
+from pydantic import BaseModel, ValidationError
+
+_MI = TypeVar("_MI", bound=BaseModel)
+_MO = TypeVar("_MO", bound=BaseModel)
+
+
+class GenericErrorResponse(BaseModel):
+    detail: str
+
+
+class ValidationErrorDetail(BaseModel):
+    loc: list[str]
+    msg: str
+    type: str
+
+
+class ValidationErrorResponse(BaseModel):
+    detail: list[ValidationErrorDetail]
+
+
+class FableError(httpx.HTTPError):
+    def __init__(
+        self, message: str, request: httpx.Request, error: GenericErrorResponse | ValidationErrorResponse = None
+    ):
+        super().__init__(message)
+        self._request = request
+        self.error_response = error
+
+        self.error_type = "unknown"
+
+        if isinstance(error, GenericErrorResponse):
+            self.error_type = "default"
+
+        if isinstance(error, ValidationErrorResponse):
+            self.error_type = "validation"
+
+
+def new_error_from_response(r: httpx.Response):
+    error_response = None
+    error_message = f"received status code {r.status_code}"
+
+    # validation error (422 by default with FastAPI)
+    if r.status_code == httpx.codes.UNPROCESSABLE_ENTITY.value:
+        try:
+            error_response = ValidationErrorResponse(**r.json())
+            error_message += ": invalid request"
+        except (ValidationError, JSONDecodeError):
+            pass
+    else:
+        try:
+            error_response = GenericErrorResponse(**r.json())
+            error_message += f": {error_response.detail}"
+        except (ValidationError, JSONDecodeError):
+            pass
+
+    return FableError(error_message, r.request, error_response)
+
+
+class FableClient(object):
+    def __init__(self, client: httpx.Client = None, base_url: str = None):
+        self._client = client or httpx.Client(base_url=base_url)
+
+    def _request(self, path: str, model_in: _MI, model_out: type[_MO]) -> _MO:
+        r = self._client.post(path, json=model_in.model_dump(mode="json"))
+
+        # we generally expect a 200 here
+        if r.status_code != httpx.codes.OK.value:
+            raise new_error_from_response(r)
+
+        return model_out(**r.json())
+
+    def match(self, request: VectorMatchRequest):
+        return self._request("match/", request, VectorMatchResponse)
+
+    def transform(self, request: EntityTransformRequest):
+        return self._request("transform/", request, EntityTransformResponse)
+
+    def mask(self, request: EntityMaskRequest):
+        return self._request("mask/", request, EntityMaskResponse)

fable_client/_estimate.py

@@ -0,0 +1,98 @@
+__all__ = [
+    "split_into_wordlist",
+    "tokenize_wordlist",
+    "compute_average_tokens_for_token_list",
+    "count_tokens_in_token_list",
+    "compute_ngram_entropy",
+    "compute_attribute_stats",
+    "AttributeStats",
+]
+
+import math
+from collections import defaultdict, Counter
+from typing import TypedDict
+
+import fable_core
+from fable_model import AttributeValueEntity, BaseTransformRequest
+
+from ._client import FableClient
+
+
+class AttributeStats(TypedDict):
+    average_tokens: float
+    ngram_entropy: float
+
+
+def split_into_wordlist(entities: list[AttributeValueEntity]) -> dict[str, list[str]]:
+    """Split a list of entities into a dictionary of attribute names to values."""
+    attr_name_to_wordlist: dict[str, list[str]] = defaultdict(list)
+
+    for entity in entities:
+        for attr_name, attr_value in entity.attributes.items():
+            attr_name_to_wordlist[attr_name].append(attr_value)
+
+    return attr_name_to_wordlist
+
+
+def tokenize_wordlist(wordlist: list[str], token_size=2, padding="_") -> list[set[str]]:
+    return [fable_core.common.tokenize(word, q=token_size, padding=padding) for word in wordlist]
+
+
+def compute_average_tokens_for_token_list(token_list: list[set[str]]) -> float:
+    total_token_count = sum(len(tokens) for tokens in token_list)
+
+    if total_token_count == 0:
+        return 0
+
+    return total_token_count / len(token_list)
+
+
+def count_tokens_in_token_list(token_list: list[set[str]]) -> dict[str, int]:
+    token_counter: dict[str, int] = Counter()
+
+    for word_tokens in token_list:
+        for token in word_tokens:
+            token_counter[token] += 1
+
+    return token_counter
+
+
+def compute_ngram_entropy(token_counts: dict[str, int]) -> float:
+    total_ngram_count = sum(c for c in token_counts.values())
+    entropy = 0
+
+    for count in token_counts.values():
+        p = count / total_ngram_count
+        entropy += p * math.log2(p)
+
+    return -entropy
+
+
+def compute_attribute_stats(
+    client: FableClient,
+    entities: list[AttributeValueEntity],
+    base_transform_request: BaseTransformRequest,
+    token_size: int = 2,
+    padding: str = "_",
+    batch_size: int = 100,
+):
+    processed_entities: list[AttributeValueEntity] = []
+
+    for i in range(0, len(entities), batch_size):
+        req = base_transform_request.with_entities(entities[i : i + batch_size])
+        res = client.transform(req)
+        processed_entities.extend(res.entities)
+
+    attribute_name_to_wordlist = split_into_wordlist(processed_entities)
+
+    def _compute_stats_for_wordlist(wordlist: list[str]) -> AttributeStats:
+        token_list = tokenize_wordlist(wordlist, token_size=token_size, padding=padding)
+        average_tokens = compute_average_tokens_for_token_list(token_list)
+        token_counts = count_tokens_in_token_list(token_list)
+        ngram_entropy = compute_ngram_entropy(token_counts)
+
+        return {"average_tokens": average_tokens, "ngram_entropy": ngram_entropy}
+
+    return {
+        attr_name: _compute_stats_for_wordlist(wordlist) for attr_name, wordlist in attribute_name_to_wordlist.items()
+    }

fable_client/_model.py

@@ -0,0 +1,20 @@
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class FakerGeneratorSpec(BaseModel):
+    function_name: str
+    attribute_name: str
+    args: dict[str, Any] = Field(default_factory=dict)
+
+
+def _default_faker_locale():
+    return ["en_US"]
+
+
+class FakerGeneratorConfig(BaseModel):
+    seed: int
+    count: int = Field(ge=0)
+    locale: list[str] = Field(default_factory=_default_faker_locale)
+    generators: list[FakerGeneratorSpec]

fable_client/main.py

@@ -0,0 +1,9 @@
+from ._cli import app
+
+
+def run_cli():
+    app(max_content_width=120)
+
+
+if __name__ == "__main__":
+    run_cli()

fable_client/types.py

@@ -0,0 +1,4 @@
+__all__ = ["AttributeStats", "FakerGeneratorConfig", "FakerGeneratorSpec"]
+
+from ._estimate import AttributeStats
+from ._model import FakerGeneratorConfig, FakerGeneratorSpec

fable_client-0.4.1.dist-info/METADATA

@@ -0,0 +1,489 @@
+Metadata-Version: 2.4
+Name: fable-client
+Version: 0.4.1
+Summary: HTTP-based client for interacting with the FABLE service for privacy-preserving record linkage with Bloom filters.
+License-Expression: MIT
+License-File: LICENSE
+Keywords: record linkage,privacy,bloom filter,bitarray,cryptography,service,client,cli
+Author: Maximilian Jugl
+Requires-Python: >=3.10,<4
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: End Users/Desktop
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Security :: Cryptography
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Provides-Extra: faker
+Requires-Dist: click (>=8.0.0,<9.0.0)
+Requires-Dist: fable-core (>=0.1.5,<0.2.0)
+Requires-Dist: fable-model (>=0.1.7,<0.2.0)
+Requires-Dist: faker (>=26.0.0) ; extra == "faker"
+Requires-Dist: httpx (>=0.28.0,<0.29.0)
+Project-URL: Repository, https://github.com/ul-mds/fable-client
+Description-Content-Type: text/markdown
+
+[![PyPI](https://img.shields.io/pypi/v/fable-client?cacheSeconds=0)](https://pypi.org/project/fable-client/)
+[![Python Versions](https://img.shields.io/pypi/pyversions/fable-client?cacheSeconds=0)](https://pypi.org/project/fable-client/)
+![Code Coverage](https://img.shields.io/badge/Coverage-92%25-green.svg)
+[![License](https://img.shields.io/pypi/l/fable-client?cacheSeconds=0)](https://pypi.org/project/fable-client/)
+
+# FABLE Client
+
+This package contains a HTTP-based client for working with the server provided by
+the [PPRL service](https://github.com/ul-mds/fable-pprl-service) which is part of the FABLE
+(**F**ederated **A**nonymized **B**loom filter **L**inkage **E**ngine) ecosystem.
+It also contains a command-line application which uses the library to process CSV files.
+
+## Installation
+
+```bash
+pip install fable-client
+```
+
+Weight estimation requires additional packages which are not shipped by default.
+To add them, install this package using the following command.
+
+```bash
+pip install fable-client[faker]
+```
+
+## Library methods
+
+The library exposes functions for entity pre-processing, masking and bit vector matching.
+They follow the data model that is also used by the FABLE PPRL service, which is exposed through
+the [FABLE model package](https://github.com/ul-mds/fable-model).
+
+### Entity transformation
+
+```python
+import fable_client
+from fable_model import (
+    EntityTransformRequest,
+    TransformConfig,
+    EmptyValueHandling,
+    AttributeValueEntity,
+    GlobalTransformerConfig,
+    NormalizationTransformer,
+)
+
+client = fable_client.FableClient(base_url="http://localhost:8080")
+
+response = client.transform(
+    EntityTransformRequest(
+        config=TransformConfig(empty_value=EmptyValueHandling.error),
+        entities=[AttributeValueEntity(id="001", attributes={"first_name": "Müller", "last_name": "Ludenscheidt"})],
+        global_transformers=GlobalTransformerConfig(before=[NormalizationTransformer()]),
+    )
+)
+
+print(response.entities)
+# => [AttributeValueEntity(id='001', attributes={'first_name': 'muller', 'last_name': 'ludenscheidt'})]
+```
+
+### Entity masking
+
+```python
+import fable_client
+from fable_model import (
+    EntityMaskRequest,
+    MaskConfig,
+    HashConfig,
+    HashFunction,
+    HashAlgorithm,
+    RandomHash,
+    CLKFilter,
+    AttributeValueEntity,
+)
+
+client = fable_client.FableClient(base_url="http://localhost:8080")
+
+response = client.mask(
+    EntityMaskRequest(
+        config=MaskConfig(
+            token_size=2,
+            hash=HashConfig(
+                function=HashFunction(algorithms=[HashAlgorithm.sha1], key="s3cr3t_k3y"), strategy=RandomHash()
+            ),
+            filter=CLKFilter(hash_values=5, filter_size=256),
+        ),
+        entities=[AttributeValueEntity(id="001", attributes={"first_name": "muller", "last_name": "ludenscheidt"})],
+    )
+)
+
+print(response.entities)
+# => [BitVectorEntity(id='001', value='SKkgqBHBCJJCANICEKSpWMAUBYCQEMLuZgEQGBKRC8A=')]
+```
+
+### Bit vector matching
+
+```python
+import fable_client
+from fable_model import VectorMatchRequest, MatchConfig, SimilarityMeasure, BitVectorEntity
+
+client = fable_client.FableClient(base_url="http://localhost:8080")
+
+response = client.match(
+    VectorMatchRequest(
+        config=MatchConfig(measure=SimilarityMeasure.jaccard, threshold=0.8),
+        domain=[BitVectorEntity(id="001", value="SKkgqBHBCJJCANICEKSpWMAUBYCQEMLuZgEQGBKRC8A=")],
+        range=[
+            BitVectorEntity(id="100", value="UKkgqBHBDJJCANICELSpWMAUBYCMEMLrZgEQGBKRC7A="),
+            BitVectorEntity(id="101", value="H5DN45iUeEjrjbHZrzHb3AyQk9O4IgxcpENKKzEKRLE="),
+        ],
+    )
+)
+
+print(response.matches)
+# => [Match(domain=BitVectorEntity(id='001', value='SKkgqBHBCJJCANICEKSpWMAUBYCQEMLuZgEQGBKRC8A='), range=BitVectorEntity(id='100', value='UKkgqBHBDJJCANICELSpWMAUBYCMEMLrZgEQGBKRC7A='), similarity=0.8536585365853658)]
+```
+
+### Attribute weight estimation
+
+```python
+import fable_client
+from fable_model import (
+    AttributeValueEntity,
+    BaseTransformRequest,
+    TransformConfig,
+    EmptyValueHandling,
+    GlobalTransformerConfig,
+    NormalizationTransformer,
+)
+
+client = fable_client.FableClient(base_url="http://localhost:8080")
+
+stats = fable_client.estimate.compute_attribute_stats(
+    client,
+    [
+        AttributeValueEntity(id="001", attributes={"given_name": "Max", "last_name": "Mustermann", "gender": "m"}),
+        AttributeValueEntity(id="002", attributes={"given_name": "Maria", "last_name": "Musterfrau", "gender": "f"}),
+    ],
+    BaseTransformRequest(
+        config=TransformConfig(empty_value=EmptyValueHandling.skip),
+        global_transformers=GlobalTransformerConfig(before=[NormalizationTransformer()]),
+    ),
+)
+
+print(stats)
+# => {'given_name': {'average_tokens': 5.0, 'ngram_entropy': 2.9219280948873623}, 'last_name': {'average_tokens': 11.0, 'ngram_entropy': 3.913977073182751}, 'gender': {'average_tokens': 2.0, 'ngram_entropy': 2.0}}
+```
+
+## Command line interface
+
+The `fable` command exposes all the library's functions and adapts them to work with CSV files.
+Running `fable --help` provides an overview of the command options.
+
+```
+$ fable --help
+Usage: fable [OPTIONS] COMMAND [ARGS]...
+
+  HTTP client for performing PPRL based on Bloom filters.
+
+Options:
+  --base-url TEXT                 base URL to HTTP-based PPRL service
+  -b, --batch-size INTEGER RANGE  amount of bit vectors to match at a time  [x>=1]
+  --timeout-secs INTEGER RANGE    seconds until a request times out  [x>=1]
+  --delimiter TEXT                column delimiter for CSV files
+  --encoding TEXT                 character encoding for files
+  --help                          Show this message and exit.
+
+Commands:
+  estimate   Estimate attribute weights based on randomly generated data.
+  mask       Mask a CSV file with entities.
+  match      Match bit vectors from CSV files against each other.
+  transform  Perform pre-processing on a CSV file with entities
+```
+
+The `fable` command works on two basic types of CSV files that follow a simple structure.
+Entity files are CSV files that contain a column with a unique identifier and arbitrary additional columns which
+contain values for certain attributes that identify an entity.
+Each row is representative of a single entity.
+
+```csv
+id,first_name,last_name,date_of_birth,gender
+001,Natalie,Sampson,1956-12-16,female
+002,Eric,Lynch,1910-01-11,female
+003,Pam,Vaughn,1983-10-05,male
+004,David,Jackson,2006-01-27,male
+005,Rachel,Dyer,1904-02-02,female
+```
+
+Bit vector files contain an ID column and a value column which contains a representative bit vector.
+These bit vectors are generally generated by masking a record from an entity file.
+
+```csv
+id,value
+001,0Dr8t+kE5ltI+xdM85fwx0QLrTIgvFN35/0YvODNdOE0AaUHPphikXYy4LlArE4UqfjPs+wKtT233R7lBzSp5mwkCjTzA1tl0N7s+sFeKyIrOiGk0gNIYvA=
+002,QMEIkE9TN1Quv0K0QAIk1RZD3qF7nQh0IyOYqVDf8IQkyaLGcFjiLHsEgBpU8CRSCuATbWpjEwGi3dilizySQy4miGiJolilYmwKysjseq+IFsAU3T1IRjA=
+003,BqFoNZhrAVBq9SV1wBK0dUZLHDM9hCBoO4XdKCzvasSUELQeAB8+DV5tAhDl5KCSJfDCB6JG4WSoCFbozXqBYSUMqEQJE0JwhpRK6oLOcRRoGwGESDBMZwA=
+004,8C9KItMTwtz4oXQvo8G0t1bTnwspnghmJwyqqcL2RIHASb4XJHAqybMCXQBm5mq6h/kdxGbblxBjhy79jRUcI60haqZhNsst0n7OUAxM/UoZVumIilRIbCA=
+005,CFk4I0sKwnRoiTEOQASy1QZfHCGB1GBgYQDcZwDDtIkGGLOmLRhrQyOSlQDUDoYTbvaBRVqbkRnqmYQbDTEGlG+2y60FMmBEKtxsr0I4I00oMpuoXAsDWmA=
+```
+
+Pre-processing is done with the `fable transform` command.
+It requires a base transform request file, an entity file and an output file to write the pre-processed entities to.
+Attribute and global transformer configurations can be provided, but at least one must be specified.
+
+In this example, a global normalization transformer which is executed before all other attribute-specific transformers
+is defined.
+Date time reformatting is applied to the "date of birth" column in the input file.
+
+_request.json_
+
+```json
+{
+  "config": {
+    "empty_value": "skip"
+  },
+  "attribute_transformers": [
+    {
+      "attribute_name": "date_of_birth",
+      "transformers": [
+        {
+          "name": "date_time",
+          "input_format": "%Y-%m-%d",
+          "output_format": "%Y%m%d"
+        }
+      ]
+    }
+  ],
+  "global_transformers": {
+    "before": [
+      {
+        "name": "normalization"
+      }
+    ]
+  }
+}
+```
+
+```
+$ fable transform ./request.json ./input.csv ./output.csv  
+Transforming entities  [####################################]  100%
+```
+
+_output.csv_
+
+```csv
+id,first_name,last_name,date_of_birth,gender
+001,natalie,sampson,19561216,female
+002,eric,lynch,19100111,female
+003,pam,vaughn,19831005,male
+004,david,jackson,20060127,male
+005,rachel,dyer,19040202,female
+```
+
+Masking is done with `fable mask` and its subcommands.
+It requires a base mask request file, an entity file and an output file to write the masked entities to.
+
+_request.json_
+
+```json
+{
+  "config": {
+    "token_size": 2,
+    "hash": {
+      "function": {
+        "algorithms": ["sha256"],
+        "key": "s3cr3t_k3y",
+        "strategy": {
+          "name": "random_hash"
+        }
+      }
+    },
+    "prepend_attribute_name": true,
+    "filter": {
+      "type": "clk",
+      "filter_size": 512,
+      "hash_values": 5,
+      "padding": "_",
+      "hardeners": [
+        {
+          "name": "permute",
+          "seed": 727
+        },
+        {
+          "name": "rehash",
+          "window_size": 16,
+          "window_step": 8,
+          "samples": 2
+        }
+      ]
+    }
+  }
+}
+```
+
+_input.csv_
+
+```csv
+id,first_name,last_name,date_of_birth,gender
+001,natalie,sampson,19561216,female
+002,eric,lynch,19100111,female
+003,pam,vaughn,19831005,male
+004,david,jackson,20060127,male
+005,rachel,dyer,19040202,female
+```
+
+```
+$ fable mask ./request.json ./input.csv ./output.csv
+Masking entities  [####################################]  100%
+```
+
+_output.csv_
+
+```csv
+id,value
+001,wAWgITvQ1/VACpRYC2EKrfCkWziyEhmyKwi5sMsFrAQVoIBygTQScPRoIIAto0AwS0ihlcAIFAcQRwccY5IOmQ==
+002,cFCwQIABQ+TgSSdlGM/z54BEUgmYhA1GKtCxQAKAXFIWiPAFIQYaFArgM61pUAAeATwBlBEOEw4Oowe0rbcMGw==
+003,IgK16AAISCRoCuVAb1UBZYBBhGgxSEkKeMkTUCKAx4IAsNGJBS4ShgBAGIapBIQWJLiBFEEKAIWAGYS8ZZGMKw==
+004,ZlBkyoYIEWmeaxbPDNng5JjHACkCAJwjlBCJQBJ4ZBSyOAukACUahOAFQ20oNwTQEDRA005+VUUfsUQcKCGNxg==
+005,cUekQFQkI7TpTcRwmcNDoodRRBshlSEiAUjBQiMlxBLTmODMJICmDmxgUqYKonQEMFD58QsogRQFIgYUwJDOHA==
+```
+
+Matching is done with the `fable match` command.
+It allows the matching of multiple bit vector input files at once.
+If more than two files are provided, the command will pick out pairs of files and matches their contents against one
+another.
+
+In this example, the bit vectors of two files are matched against each other.
+The Jaccard index is used as a similarity measure and a match threshold of 70% is applied.
+
+_request.json_
+
+```json
+{
+  "config": {
+    "measure": "jaccard",
+    "threshold": 0.7
+  }
+}
+```
+
+_domain.csv_
+
+```csv
+id,value
+001,wAWgITvQ1/VACpRYC2EKrfCkWziyEhmyKwi5sMsFrAQVoIBygTQScPRoIIAto0AwS0ihlcAIFAcQRwccY5IOmQ==
+002,cFCwQIABQ+TgSSdlGM/z54BEUgmYhA1GKtCxQAKAXFIWiPAFIQYaFArgM61pUAAeATwBlBEOEw4Oowe0rbcMGw==
+003,IgK16AAISCRoCuVAb1UBZYBBhGgxSEkKeMkTUCKAx4IAsNGJBS4ShgBAGIapBIQWJLiBFEEKAIWAGYS8ZZGMKw==
+004,ZlBkyoYIEWmeaxbPDNng5JjHACkCAJwjlBCJQBJ4ZBSyOAukACUahOAFQ20oNwTQEDRA005+VUUfsUQcKCGNxg==
+005,cUekQFQkI7TpTcRwmcNDoodRRBshlSEiAUjBQiMlxBLTmODMJICmDmxgUqYKonQEMFD58QsogRQFIgYUwJDOHA==
+```
+
+_range.csv_
+
+```csv
+id,value
+101,kUSyxIgtIDSAB7ZYDkFQRZpFoMkCjCCCbDTWAUJTRAAEBpspBX4PNUZKi1AIVCABAjg6EAoKuwVleeUYgRBYoQ==
+102,IAA0YE4MGexIiYdEjwNzoOKmIA4CEHEiKQASYFPhxQTQlPAAgYW3AWBYmQJ8YMoaAj0ZkoOrFyUmFo52TDcIKw==
+103,BFAwREkkQbTdzddgDHFWgMRJMyxAMW+jq2ASICMBtIEr+YDCBRUgxEDIsQpciO4mAK3h2cIbXFQCMlaVpJPZIQ==
+104,wBWgITvQ2/VACpRYC2EKrfCkWxiyEhmyKwi5sMsFrBQVoIBygTQScPRoIIAto0AwS0ihldAIFAcQRwccY5IOmQ==
+105,QCCwIKQAED5AjaZYmodDcZAEBKkIxgAiDfEUoDKEdgEAEJAMAwcfQEbQkaQ4ANAABqiUscAKPQZEMJxRhTGIGQ==
+```
+
+```
+$ fable match request.json domain.csv range.csv output.csv
+Matching bit vectors from domain.csv and range.csv  [####################################]  100%
+```
+
+_output.csv_
+
+```csv
+domain_id,domain_file,range_id,range_file,similarity
+001,domain.csv,104,range.csv,0.9690721649484536
+```
+
+Weight estimation is done with the `fable estimate` command.
+It generates random data based off of user specification and computes estimates for attribute weights.
+Data can be generated using [Faker](https://faker.readthedocs.io/).
+
+*faker.json*
+
+```json
+{
+  "seed": 727,
+  "count": 5000,
+  "locale": ["de_DE"],
+  "generators": [
+    {"function_name": "first_name_nonbinary", "attribute_name": "given_name"},
+    {"function_name": "last_name", "attribute_name": "last_name"},
+    {"function_name": "random_element", "attribute_name": "gender", "args": {"elements": ["m", "f"]}},
+    {"function_name": "street_name", "attribute_name": "street_name"},
+    {"function_name": "city", "attribute_name": "municipality"},
+    {"function_name": "postcode", "attribute_name": "postcode"}
+  ]
+}
+```
+
+```
+$ fable estimate faker faker.json faker-output.json
+```
+
+*faker-output.json*
+
+```json
+[
+  {
+    "attribute_name": "given_name",
+    "weight": 7.657958943890718,
+    "average_token_count": 7.5686
+  },
+  {
+    "attribute_name": "last_name",
+    "weight": 7.444573503220938,
+    "average_token_count": 7.5204
+  },
+  {
+    "attribute_name": "gender",
+    "weight": 1.9999971146079947,
+    "average_token_count": 2.0
+  },
+  {
+    "attribute_name": "street_name",
+    "weight": 7.605565770282046,
+    "average_token_count": 16.2188
+  },
+  {
+    "attribute_name": "municipality",
+    "weight": 7.659422921807241,
+    "average_token_count": 9.952
+  },
+  {
+    "attribute_name": "postcode",
+    "weight": 6.7812429085107,
+    "average_token_count": 5.9464
+  }
+]
+```
+
+## Configuring pytest
+
+In order to run integration tests, the FABLE PPRL service is needed.
+The first option is to spin up the service independently and direct pytest to it.
+Alternatively, pytest can start a Docker test container for the duration of the test run.
+The following table shows all available configuration options.
+These variables can be defined in `.env` or `.env.test`.
+
+| **Environment variable**          | **Description**                                                             | **Default** |
+|-----------------------------------|-----------------------------------------------------------------------------|-------------|
+| PYTEST_PPRL_BASE_URL<sup>1)</sup> | Base URL for the FABLE PPRL service                                         |             |
+| PYTEST_PPRL_SERVICE_VERSION       | Tag of the FABLE PPRL service image that will run inside the test container | latest      |
+| PYTEST_PRRL_SERVICE_PORT          | Port that will be exposed by the test container                             | 8080        |
+
+<sup>1)</sup> If defined, pytest will not spin up a test container.
+
+## License
+
+MIT.
+

fable_client-0.4.1.dist-info/RECORD

@@ -0,0 +1,12 @@
+fable_client/__init__.py,sha256=Y3swMGPFlHZLsTJyKiBDsRRigHmwildneq-imdBBAf8,163
+fable_client/_cli.py,sha256=_WXKcIhkw0EtR7ma7cX24O0-jCTTjJ134OHRr1mViLM,17095
+fable_client/_client.py,sha256=HfsYRAq8ZBiELu8euJPG-x_xfiEZp1-zscWRmCm1FTc,2711
+fable_client/_estimate.py,sha256=GrMIOxzm9lxsILhhqW78zEOXITr6FQbMtYL0IhvkdqM,3085
+fable_client/_model.py,sha256=SiuTYVBbryJvVI1ophhF4YifgkuAN_EwhwRd6B4LPEU,449
+fable_client/main.py,sha256=TcxUSffggdRNGZhlt_iSvQ7hQzJ3WVfZuuQ4fHDXmGE,113
+fable_client/types.py,sha256=eiYF458YKlLU_QUocVP7t_bmSkXG6TYDnYVH0vk1Peo,175
+fable_client-0.4.1.dist-info/METADATA,sha256=4_9Vgxye_tZOZyWbkZ5_RCrv19xOvIHU8gRybaxAA5c,16211
+fable_client-0.4.1.dist-info/WHEEL,sha256=EGEvSphFYqXKs23-kQBeyNoJP1nrT8ZJKQoi5p5DYL8,88
+fable_client-0.4.1.dist-info/entry_points.txt,sha256=7W7ZrPoF3jcq4IEAcIzs1IdMKP0NcJOHje7yO2Zlxao,51
+fable_client-0.4.1.dist-info/licenses/LICENSE,sha256=Q-Ktj_VJi4SeAGoslkUojX0DD9cZFxaw36u4FR5f61o,1117
+fable_client-0.4.1.dist-info/RECORD,,

fable_client-0.4.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.4.0
+Root-Is-Purelib: true
+Tag: py3-none-any

fable_client-0.4.1.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+fable=fable_client.main:run_cli
+

fable_client-0.4.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 University Medical Center Leipzig, Dept. Medical Data Science
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.