gemba 0.1.0__py3-none-any.whl

gemba/prompt.py

@@ -0,0 +1,139 @@
+import re
+from termcolor import colored
+
+
+def parse_and_check_numerical_answer(answer, min=None, max=None):
+    attempt = parse_numerical_answer(answer, min, max)
+    if attempt is not None:
+        if attempt < min or attempt > max:
+            return None
+        return attempt
+
+    return None
+
+
+def parse_numerical_answer(answer, min=None, max=None):
+    # get all numbers in a string
+    numbers = re.findall(r'\d+', answer)
+    if len(numbers) == 1:
+        return int(numbers[0])
+
+    # check if the answer is in form ['100'] and extract the number
+    r1 = re.match(r"^\[['\"][0-9]*['\"]\]$", answer)
+    if r1 is not None:
+        return int(answer[2:-2])
+
+    if max is not None:
+        # check if the answer is in a form of 0/100
+        r2 = re.match(rf"^[0-9]*/{max}$", answer)
+        if r2 is not None:
+            return int(answer.split("/")[0])
+
+    return None
+
+
+def validate_number(x, min=0, max=100):
+    attempt = parse_and_check_numerical_answer(x, min, max)
+    if attempt is not None:
+        return attempt
+    return None
+
+
+def parse_classes(answer, classes):
+    final_class = None
+    for i in range(len(classes)):
+        if classes[i].lower() in answer.lower():
+            if final_class is None:
+                final_class = i
+            else:
+                print(colored(f"Two classes found in answer {answer}", "red"))
+                return None
+
+    return final_class
+
+
+def validate_stars(x):
+    x = x.lower()
+    # try to find all possible answers as sometimes it seems to be explaining itself
+    possible_answers = set()
+
+    # check if string x contains * characters
+    if "*" in x:
+        possible_answers.add(x.count("*"))
+    if "★" in x:
+        possible_answers.add(x.count("★"))
+
+    x = f" {x} ".replace("\n", " ")
+    # possible answers: "five stars", "5 stars", "five", "five starts: perfect translation", ...
+    if " one " in x or "1 star" in x:
+        possible_answers.add(1)
+    if " two " in x or "2 star" in x:
+        possible_answers.add(2)
+    if " three " in x or "3 star" in x:
+        possible_answers.add(3)
+    if " four " in x or "4 star" in x:
+        possible_answers.add(4)
+    if " five " in x or "5 star" in x:
+        possible_answers.add(5)
+
+    numerical = parse_numerical_answer(x)
+    if numerical is not None:
+        possible_answers.add(numerical)
+
+    if len(possible_answers) == 1:
+        answer = possible_answers.pop()
+        if 1 <= answer <= 5:
+            return answer
+    return None
+
+
+language_codes = {
+    "en": "English",
+    "de": "German",
+    "zh": "Chinese",
+    "ru": "Russian",
+}
+
+prompts = {
+    "GEMBA-DA": {
+        "prompt": 'Score the following translation from {source_lang} to {target_lang} on a continuous scale from 0 to 100, where a score of zero means "no meaning preserved" and score of one hundred means "perfect meaning and grammar".\n\n{source_lang} source: "{source_seg}"\n{target_lang} translation: "{target_seg}"\nScore: ',
+        "validate_answer": lambda x: validate_number(x),
+        "use_ref": False},
+
+    "GEMBA-DA_ref": {
+        "prompt": 'Score the following translation from {source_lang} to {target_lang} with respect to human reference on a continuous scale 0 to 100 where score of zero means "no meaning preserved" and score of one hundred means "perfect meaning and grammar".\n\n{source_lang} source: "{source_seg}"\n{target_lang} human reference: {reference_seg}\n{target_lang} machine translation: "{target_seg}"\nScore: ',
+        "validate_answer": lambda x: validate_number(x),
+        "use_ref": True},
+
+    "GEMBA-SQM": {
+        "prompt": 'Score the following translation from {source_lang} to {target_lang} on a continuous scale from 0 to 100 that starts on "No meaning preserved", goes through "Some meaning preserved", then "Most meaning preserved and few grammar mistakes", up to "Perfect meaning and grammar".\n\n{source_lang} source: "{source_seg}"\n{target_lang} translation: "{target_seg}"\nScore (0-100): ',
+        "validate_answer": lambda x: validate_number(x),
+        "use_ref": False},
+
+    "GEMBA-SQM_ref": {
+        "prompt": 'Score the following machine translation from {source_lang} to {target_lang} with respect to the human reference on a continuous scale from 0 to 100 that starts with "No meaning preserved", goes through "Some meaning preserved", then "Most meaning preserved and few grammar mistakes", up to "Perfect meaning and grammar".\n\n{source_lang} source: "{source_seg}"\n{target_lang} human reference: "{reference_seg}"\n{target_lang} machine translation: "{target_seg}"\nScore (0-100): ',
+        "validate_answer": lambda x: validate_number(x),
+        "use_ref": True},
+
+    "GEMBA-stars": {
+        "prompt": 'Score the following translation from {source_lang} to {target_lang} with one to five stars. Where one star means "Nonsense/No meaning preserved", two stars mean "Some meaning preserved, but not understandable", three stars mean "Some meaning preserved and understandable", four stars mean "Most meaning preserved with possibly few grammar mistakes", and five stars mean "Perfect meaning and grammar".\n\n{source_lang} source: "{source_seg}"\n{target_lang} translation: "{target_seg}"\nStars: ',
+        "validate_answer": lambda x: validate_stars(x),
+        "use_ref": False},
+
+    "GEMBA-stars_ref": {
+        "prompt": 'Score the following translation from {source_lang} to {target_lang} with respect to the human reference with one to five stars. Where one star means "Nonsense/No meaning preserved", two stars mean "Some meaning preserved, but not understandable", three stars mean "Some meaning preserved and understandable", four stars mean "Most meaning preserved with possibly few grammar mistakes", and five stars mean "Perfect meaning and grammar".\n\n{source_lang} source: "{source_seg}"\n{target_lang} human reference: "{reference_seg}"\n{target_lang} translation: "{target_seg}"\nStars: ',
+        "validate_answer": lambda x: validate_stars(x),
+        "use_ref": True},
+
+    "GEMBA-classes": {
+        "prompt": 'Classify the quality of machine translation from {source_lang} to {target_lang} into one of following classes: "No meaning preserved", "Some meaning preserved, but not understandable", "Some meaning preserved and understandable", "Most meaning preserved, minor issues", "Perfect translation".\n\n{source_lang} source: "{source_seg}"\n{target_lang} machine translation: "{target_seg}"\nClass: ',
+        "use_ref": False,
+        "validate_answer": lambda x, classes=["No meaning preserved", "Some meaning preserved, but not understandable", "Some meaning preserved and understandable", "Most meaning preserved, minor issues", "Perfect translation"]: parse_classes(x, classes),
+        "max_tokens": 100},
+
+    "GEMBA-classes_ref": {
+        "prompt": 'Classify the quality of machine translation from {source_lang} to {target_lang} with respect to the human reference into one of following classes: "No meaning preserved", "Some meaning preserved, but not understandable", "Some meaning preserved and understandable", "Most meaning preserved, minor issues", "Perfect translation".\n\n{source_lang} source: "{source_seg}"\n{target_lang} human reference: "{reference_seg}"\n{target_lang} machine translation: "{target_seg}"\nClass: ',
+        "use_ref": True,
+        "validate_answer": lambda x, classes=["No meaning preserved", "Some meaning preserved, but not understandable", "Some meaning preserved and understandable", "Most meaning preserved, minor issues", "Perfect translation"]: parse_classes(x, classes),
+        "max_tokens": 100},
+}

gemba/scores.py

@@ -0,0 +1,103 @@
+from pathlib import Path
+import os
+import pandas as pd
+
+
+class Scores:
+    def __init__(self, name, testset, refname, output_path=None):
+        self.name = name
+        self.testset = testset
+        self.refname = refname
+        if output_path is None:
+            output_path = testset.basepath
+
+        self.output_path = output_path
+
+        self.seg_scores = None
+        self.metadata = None
+        self.prefix = None
+        self.load()
+
+    def load(self):
+        output_folder = f"{self.testset.basepath}/{self.testset.dataset}/metric-scores/{self.testset.lp}"
+        Path(output_folder).mkdir(parents=True, exist_ok=True)
+
+        if self.refname is not None:
+            self.prefix = f"{output_folder}/{self.name}-{self.refname}"
+        else:
+            self.prefix = f"{output_folder}/{self.name}-src"
+
+        seg_scores_file = self.get_seg_path()
+        if os.path.isfile(f"{seg_scores_file}"):
+            self.seg_scores = pd.read_csv(seg_scores_file, sep="\t", names=["system", "score"], index_col=False)
+        else:
+            self.seg_scores = pd.DataFrame(columns=["system", "score"])
+
+        if os.path.isfile(f"{self.get_meta_path()}"):
+            self.metadata = pd.read_csv(self.get_meta_path(), sep="\t", names=["system", "temperature"], index_col=False)
+        else:
+            self.metadata = pd.DataFrame(columns=["system", "temperature"])
+
+        # generate placeholders for scores
+        segment_count = len(self.testset.sources)
+        for system in self.testset.systems.keys():
+            if system not in self.seg_scores["system"].values:
+                # generate placeholders for scores
+                placeholder = pd.DataFrame([{"system": system, 'score': 'None'}] * segment_count)
+                self.seg_scores = pd.concat([self.seg_scores, placeholder], ignore_index=True)
+
+            if system not in self.metadata["system"].values:
+                placeholder = pd.DataFrame([{"system": system, 'temperature': 'None'}] * segment_count)
+                self.metadata = pd.concat([self.metadata, placeholder], ignore_index=True)
+
+            # check that all systems are present and have correct number of scores
+            assert len(self.seg_scores[self.seg_scores.system == system]) == segment_count
+            assert len(self.metadata[self.metadata.system == system]) == segment_count
+
+    def get_seg_path(self):
+        return f"{self.prefix}.seg.score"
+
+    def get_sys_path(self):
+        return f"{self.prefix}.sys.score"
+
+    def get_domain_path(self):
+        return f"{self.prefix}.domain.score"
+
+    def get_meta_path(self):
+        return f"{self.prefix}.seg.meta"
+
+    def _remap_index(self, system, hypothesis_index):
+        # the order of systems may be different
+        # get id of the first hypothesis of the system
+        index = (self.seg_scores["system"] == system).argmax()
+        h = hypothesis_index % len(self.testset.sources)
+        return index + h
+
+    def get_score(self, system, hypothesis_index):
+        index = self._remap_index(system, hypothesis_index)
+        return self.seg_scores.iloc[index]['score']
+
+    def assign_score(self, system, hypothesis_index, answer, temperature=None):
+        index = self._remap_index(system, hypothesis_index)
+        self.seg_scores.iloc[index]['score'] = answer
+        self.metadata.iloc[index]['temperature'] = temperature
+
+    def save(self):
+        # segment level scores
+        self.seg_scores.to_csv(self.get_seg_path(), sep="\t", index=False, header=False, na_rep="None")
+
+        # system scores
+        self.seg_scores.score = self.seg_scores.score.replace("None", None).astype(float)
+        sys_scores_df = self.seg_scores.groupby(['system'], as_index=False, dropna=True).mean()
+        sys_scores_df.to_csv(self.get_sys_path(), sep="\t", index=False, header=False, na_rep="None")
+
+        # domain scores
+        df = self.seg_scores.copy()
+        documents = self.testset.documents
+        total_systems = len(self.testset.systems)
+        df["domains"] = pd.DataFrame([x.split("\t")[0] for x in documents] * total_systems)[0]
+        df = df.groupby(["domains", 'system'], as_index=False, dropna=True).mean()
+        df.to_csv(self.get_domain_path(), sep="\t", index=False, header=False, na_rep="None")
+
+        # metadata
+        self.metadata.to_csv(self.get_meta_path(), sep="\t", index=False, header=False, na_rep="None")

gemba/testset.py

@@ -0,0 +1,58 @@
+import glob
+import os
+
+
+class Testset:
+    def __init__(self, basepath, dataset, lp):
+        self.basepath = basepath
+        self.dataset = dataset
+        self.lp = lp
+
+        self.sources = []
+        self.references = {}
+        self.systems = {}
+        self.documents = []
+        self.main_ref = None
+
+        self.load()
+
+    def load(self):
+        dataset = f"{self.basepath}/{self.dataset}"
+
+        self.sources = self.load_segment_files(f"{dataset}/sources/{self.lp}.txt")
+
+        # list all files in references folder
+        refs = glob.glob(f"{dataset}/references/{self.lp}.*.txt")
+        for reffile in refs:
+            refname = reffile.split('.')[-2]
+            if self.main_ref is None:
+                self.main_ref = refname
+            self.references[refname] = self.load_segment_files(reffile)
+
+        systems = f"{dataset}/system-outputs/{self.lp}"
+        # keep systems in order
+        all_systems = sorted(os.listdir(systems))
+        for system in all_systems:
+            systemname = system.replace(".txt", "")
+            self.systems[systemname] = self.load_segment_files(f"{systems}/{system}")
+
+        self.documents = self.load_segment_files(f"{dataset}/documents/{self.lp}.docs")
+
+    def iterate_over_all(self, reference=None):
+        for system in self.systems.keys():
+            if reference is None:
+                for src, hyp in zip(self.sources, self.systems[system]):
+                    yield src, hyp, None, system
+            else:
+                for src, hyp, ref in zip(self.sources, self.systems[system], self.references[reference]):
+                    yield src, hyp, ref, system
+
+    def load_segment_files(self, path):
+        segments = []
+        with open(path, "r") as fh:
+            for line in fh:
+                segments.append(line.rstrip())
+        return segments
+
+    def segments_count(self):
+        return len(self.sources)*len(self.systems)

gemba/utils.py

@@ -0,0 +1,78 @@
+import ipdb
+import pandas as pd
+import diskcache as dc
+from gemba.gpt_api import GptApi
+from gemba.gemba_mqm_utils import TEMPLATE_GEMBA_MQM, apply_template, parse_mqm_answer
+from gemba.gemba_esa import TEMPLATE_GEMBA_ESA_ERROR_SPANS, TEMPLATE_GEMBA_ESA_RANKING
+from gemba.prompt import prompts, validate_number
+
+
+def get_gemba_scores(source, hypothesis, source_lang, target_lang, method="GEMBA-MQM_norm", model="gpt-4o", cache_dir=".cache"):
+    """Get GEMBA scores for machine translation evaluation.
+    
+    This function evaluates machine translation quality using various GEMBA methods by leveraging
+    large language models to analyze source and translated text pairs.
+
+    Args:
+        source (list): List of source language segments to be evaluated
+        hypothesis (list): List of target language translations to be evaluated
+        source_lang (str): Source language code (e.g. 'en' for English)
+        target_lang (str): Target language code (e.g. 'de' for German) 
+        method (str): Evaluation method to use. One of:
+            - "GEMBA-MQM": MQM-style error annotation and scoring
+            - "GEMBA-MQM_norm": MQM-style error annotation and scoring with normalization
+            - "GEMBA-DA": Direct assessment scoring
+            - "GEMBA-DA_ref": Direct assessment with reference
+            - "GEMBA-SQM": Scalar quality metrics
+            - "GEMBA-SQM_ref": Scalar quality metrics with reference
+            - "GEMBA-stars": Star rating evaluation
+            - "GEMBA-stars_ref": Star rating with reference
+            - "GEMBA-classes": Classification-based evaluation
+            - "GEMBA-classes_ref": Classification with reference
+            - "GEMBA-ESA": Error span annotation and ranking
+        model (str): Name of the LLM model to use for evaluation
+        cache_dir (str): Directory to store the cache in
+
+    Returns:
+        list: List of scores/evaluations for each source-hypothesis pair. The format depends on the method:
+            - For MQM: Negative scores where higher is better (max 0, min -25)
+            - For MQM_norm: Normalized scores to 0-100 range
+            - For DA/SQM: Numeric scores
+            - For stars: 1-5 star ratings
+            - For classes: Classification labels
+            - For ESA: Numeric rankings based on error spans
+        list: List of error classes for each source-hypothesis pair. Only returned for MQM methods.
+
+    The function uses disk caching to store results and avoid redundant API calls. Cache is stored
+    in a '{cache_dir}/{model}_{method}' directory.
+    """
+    
+    df = pd.DataFrame({'source_seg': source, 'target_seg': hypothesis})
+    df['source_lang'] = source_lang
+    df['target_lang'] = target_lang
+
+    cache = dc.Cache(f'{cache_dir}/{model}_{method}', expire=None, size_limit=int(10e10), cull_limit=0, eviction_policy='none')
+    gptapi = GptApi()
+
+    if method in ["GEMBA-MQM", "GEMBA-MQM_norm"]:
+        df["prompt"] = df.apply(lambda x: apply_template(TEMPLATE_GEMBA_MQM, x), axis=1)
+        parse_answer = lambda x: parse_mqm_answer(x, list_mqm_errors=True, full_desc=True, normalize=method == "GEMBA-MQM_norm")
+        answers = gptapi.bulk_request(df, model, parse_answer, cache=cache, max_tokens=500)
+    elif method in ["GEMBA-DA", "GEMBA-DA_ref", "GEMBA-SQM", "GEMBA-SQM_ref", "GEMBA-stars", "GEMBA-stars_ref", "GEMBA-classes", "GEMBA-classes_ref"]:
+        df["prompt"] = df.apply(lambda x: apply_template(prompts[method]['prompt'], x), axis=1)
+        parse_answer = prompts[method]["validate_answer"]
+        answers = gptapi.bulk_request(df, model, parse_answer, cache=cache, max_tokens=500)
+    elif method == "GEMBA-ESA":
+        df["prompt"] = df.apply(lambda x: apply_template(TEMPLATE_GEMBA_ESA_ERROR_SPANS, x), axis=1)
+        parse_answer = lambda x: x
+        error_spans = gptapi.bulk_request(df, model, parse_answer, cache=cache)
+        df['error_spans'] = pd.DataFrame(error_spans)['answer']
+
+        df["prompt"] = df.apply(lambda x: apply_template(TEMPLATE_GEMBA_ESA_RANKING, x), axis=1)
+        parse_answer = validate_number
+        answers = gptapi.bulk_request(df, model, parse_answer, cache=cache)
+    else:
+        raise Exception(f"Method {method} not supported.")
+    
+    df = pd.DataFrame(answers)
+    return df['answer'].tolist(), df['errors'].tolist()

gemba-0.1.0.dist-info/METADATA

@@ -0,0 +1,136 @@
+Metadata-Version: 2.3
+Name: gemba
+Version: 0.1.0
+Summary: GEMBA — GPT Estimation Metric Based Assessment
+Project-URL: Homepage, https://github.com/joelniklaus/gemba
+Author-email: Joel Niklaus <joel@niklaus.ai>
+License: MIT
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.8
+Requires-Dist: absl-py
+Requires-Dist: diskcache
+Requires-Dist: ipdb
+Requires-Dist: openai>=1.0.0
+Requires-Dist: pandas
+Requires-Dist: pexpect
+Requires-Dist: scipy
+Requires-Dist: termcolor
+Description-Content-Type: text/markdown
+
+# GEMBA-MQM and GEMBA-DA
+
+## Setup
+
+Install required packages with python >= 3.8 (tested with 3.12.7)
+
+```
+pip install -r requirements.txt
+```
+
+Set up secrets either for Azure API or OpenAI API: 
+
+```
+export OPENAI_AZURE_ENDPOINT=
+export OPENAI_AZURE_KEY=
+```
+
+or
+
+```
+export OPENAI_API_KEY=
+```
+
+## Scoring with GEMBA
+
+Install the gemba package with `pip install gemba` and use the following code:
+
+```python
+from gemba import get_gemba_scores
+
+source = ["Hello, how are you?", "I am fine, thank you.", "I am not fine, thank you."]
+hypothesis = ["Hallo, wie geht es dir?", "Ich bin gut, danke.", "Ich bin Adolf, wer bist du?"]
+source_lang = "en"
+target_lang = "de"
+
+answers, errors = get_gemba_scores(source, hypothesis, source_lang, target_lang, method="GEMBA-MQM_norm", model="gpt-4o")
+
+for answer, error in zip(answers, errors):
+    print(answer, error)
+
+```
+
+Alternatively, you can run the main file on two text files. It assumes two files with the same number of lines. It prints the score for each line pair:
+
+```bash
+python main.py --source=source.txt --hypothesis=hypothesis.txt --source_lang=English --target_lang=Czech --method="GEMBA-MQM" --model="gpt-4"
+```
+
+The main recommended methods: `GEMBA-MQM` and `GEMBA-DA` with the model `gpt-4`.
+
+## Collecting and evaluating experiments for GEMBA-DA
+
+Get mt-metric-eval and download resources:
+
+```
+git clone https://github.com/google-research/mt-metrics-eval.git
+cd mt-metrics-eval
+pip install .
+alias mtme='python3 -m mt_metrics_eval.mtme'
+mtme --download
+cd ..
+mv ~/.mt-metrics-eval/mt-metrics-eval-v2 mt-metrics-eval-v2
+```
+
+Collect data and run the scorer
+
+```
+python gemba_da.py 
+
+export PYTHONPATH=mt-metrics-eval:$PYTHONPATH
+python evaluate.py
+```
+
+## License
+GEMBA code and data are released under the [CC BY-SA 4.0 license](https://github.com/MicrosoftTranslator/GEMBA/blob/main/LICENSE.md).
+
+## Paper
+You can read more about GEMBA-DA [in our arXiv paper](https://arxiv.org/pdf/2302.14520.pdf) 
+or GEMBA-MQM [in our arXiv paper](https://arxiv.org/pdf/2310.13988.pdf).
+
+## How to Cite
+
+
+### GEMBA-MQM 
+
+    @inproceedings{kocmi-federmann-2023-gemba-mqm,
+        title = {GEMBA-MQM: Detecting Translation Quality Error Spans with GPT-4},
+        author = {Kocmi, Tom  and Federmann, Christian},
+        booktitle = "Proceedings of the Eighth Conference on Machine Translation",
+        month = dec,
+        year = "2023",
+        address = "Singapore",
+        publisher = "Association for Computational Linguistics",
+    }
+
+### GEMBA-DA
+
+    @inproceedings{kocmi-federmann-2023-large,
+        title = "Large Language Models Are State-of-the-Art Evaluators of Translation Quality",
+        author = "Kocmi, Tom and Federmann, Christian",
+        booktitle = "Proceedings of the 24th Annual Conference of the European Association for Machine Translation",
+        month = jun,
+        year = "2023",
+        address = "Tampere, Finland",
+        publisher = "European Association for Machine Translation",
+        url = "https://aclanthology.org/2023.eamt-1.19",
+        pages = "193--203",
+    }
+
+
+
+
+
+
+

gemba-0.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+gemba/__init__.py,sha256=0ZuEumkUMWPI5wQMY7OxLolELI9GYYlup-iJw8SwBgc,67
+gemba/gemba_da.py,sha256=YCOKKP7kZBL9e1d44Zr7aTa23BqLFvh4KDOfbNSMgOU,2360
+gemba/gemba_esa.py,sha256=nBCeFjrS24wXLOcAXHRSmZFYJSkUzRS4hfp2LEqYwp8,4461
+gemba/gemba_mqm_utils.py,sha256=qiIdJv7IDx0eeqpsTCHMoUeo8EUOhG6k-YfrzkRfxyw,9612
+gemba/gpt_api.py,sha256=Igp8uQn6chKL1QWFMqKP2VR9Fbzxm8Xk83ELxk5NfM8,6671
+gemba/mtme_tools.py,sha256=xpLxCzfnLHFIxsq_LOi1Lpb-gkyFGYqFXiq9y6O315Q,4667
+gemba/prompt.py,sha256=AuPBhO2OBL3EB5I37p-GX10sx29gRw35xFAnB3bqtII,7578
+gemba/scores.py,sha256=FmmBJ-ds-abExphcVUw9qaPMnKttPWobuXNwZKLAtEs,4388
+gemba/testset.py,sha256=tDvi6xQIBXrODg02WWINrYg9jNQqruCmhBrxe9AaK48,1926
+gemba/utils.py,sha256=Re5uW5dcFj3ITWIGpxjXdAKNDKQ7i4H-Tr_s74SQgmk,4311
+gemba-0.1.0.dist-info/METADATA,sha256=9_jYmIPKmAz5cmPn-fTUB7a5xHLbYrlTXpdzhEYaSSw,3692
+gemba-0.1.0.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
+gemba-0.1.0.dist-info/licenses/LICENSE.md,sha256=XkNv-P-7d9hgciDpvOIMiRXYYAEP7rbB6-9ahWiOmzk,20137
+gemba-0.1.0.dist-info/RECORD,,

gemba-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.26.3
+Root-Is-Purelib: true
+Tag: py3-none-any