matchescu-reference-extraction 0.1.0__tar.gz

matchescu_reference_extraction-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 matchescu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

matchescu_reference_extraction-0.1.0/PKG-INFO

@@ -0,0 +1,56 @@
+Metadata-Version: 2.1
+Name: matchescu-reference-extraction
+Version: 0.1.0
+Summary: Extract references from a multitude of data sources
+License: MIT
+Author: Andrei Olar
+Author-email: andrei.olar@samlex.ro
+Requires-Python: >=3.12,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: matchescu-base (>=0.10.0,<0.11.0)
+Requires-Dist: polars (>=1.25.2,<2.0.0)
+Description-Content-Type: text/markdown
+
+from matchescu.data_sources import CsvDataSource
+
+# matchescu-reference-extraction
+
+This package implements an entity reference extraction subsystem for entity
+resolution.
+The main concepts that are relevant here are:
+
+* a generic attribute-based data record implementation (can access data by `str`
+or `int` key),
+* various `data_sources` which support reading records from different data
+stores, and
+* generic `extraction_engines` that convert data records to entity references.
+
+# Development
+
+Run the following commands to ensure you have a proper environment.
+
+```shell
+$ pyenv install 3.12
+$ poetry install
+$ poetry run pytest
+```
+
+When you contribute code, open a new `feature/*` or `hotfix/*` branch.
+
+# Usage
+
+```python
+from matchescu.data_sources import CsvDataSource
+from matchescu.extraction import Traits
+
+traits = list(
+    Traits().int(["id"])
+    .string(["name", "description", "manufacturer"])
+    .currency(["price"])
+)
+csv = CsvDataSource("./path/to/csv/file", list(traits))
+
+```

matchescu_reference_extraction-0.1.0/README.md

@@ -0,0 +1,40 @@
+from matchescu.data_sources import CsvDataSource
+
+# matchescu-reference-extraction
+
+This package implements an entity reference extraction subsystem for entity
+resolution.
+The main concepts that are relevant here are:
+
+* a generic attribute-based data record implementation (can access data by `str`
+or `int` key),
+* various `data_sources` which support reading records from different data
+stores, and
+* generic `extraction_engines` that convert data records to entity references.
+
+# Development
+
+Run the following commands to ensure you have a proper environment.
+
+```shell
+$ pyenv install 3.12
+$ poetry install
+$ poetry run pytest
+```
+
+When you contribute code, open a new `feature/*` or `hotfix/*` branch.
+
+# Usage
+
+```python
+from matchescu.data_sources import CsvDataSource
+from matchescu.extraction import Traits
+
+traits = list(
+    Traits().int(["id"])
+    .string(["name", "description", "manufacturer"])
+    .currency(["price"])
+)
+csv = CsvDataSource("./path/to/csv/file", list(traits))
+
+```

matchescu_reference_extraction-0.1.0/pyproject.toml

@@ -0,0 +1,23 @@
+[tool.poetry]
+name = "matchescu-reference-extraction"
+version = "0.1.0"
+description = "Extract references from a multitude of data sources"
+authors = ["Andrei Olar <andrei.olar@samlex.ro>"]
+license = "MIT"
+readme = "README.md"
+packages = [{include="matchescu", from="src"}]
+
+[tool.poetry.dependencies]
+python = "^3.12"
+matchescu-base = "^0.10.0"
+polars = "^1.25.2"
+
+[tool.poetry.group.dev.dependencies]
+black = "^25.1.0"
+pytest = "^8.3.5"
+mypy = "^1.15.0"
+ruff = "^0.11.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

matchescu_reference_extraction-0.1.0/src/matchescu/data_sources/__init__.py

@@ -0,0 +1,5 @@
+from matchescu.data_sources._record import Record
+from matchescu.data_sources._csv import CsvDataSource
+
+
+__all__ = ["Record", "CsvDataSource"]

matchescu_reference_extraction-0.1.0/src/matchescu/data_sources/_csv.py

@@ -0,0 +1,39 @@
+from collections.abc import Sequence, Iterator
+from os import PathLike
+from pathlib import Path
+
+import polars as pl
+
+from matchescu.typing import Trait
+
+from matchescu.data_sources._record import Record
+
+
+class CsvDataSource:
+    def __init__(
+        self,
+        file_path: str | PathLike,
+        traits: Sequence[Trait],
+        has_header: bool = True,
+    ):
+        file_path = Path(file_path)
+        self.name = file_path.name.replace(file_path.suffix, "")
+        self.traits = traits
+
+        self.__file_path = file_path
+        self.__df: pl.DataFrame | None = None
+        self.__header = has_header
+
+    def read(self) -> "CsvDataSource":
+        self.__df = pl.read_csv(
+            self.__file_path, ignore_errors=True, has_header=self.__header
+        )
+        return self
+
+    def __iter__(self) -> Iterator[Record]:
+        if self.__df is None:
+            return iter([])
+        return iter(map(Record, self.__df.iter_rows(named=True)))
+
+    def __len__(self) -> int:
+        return self.__df.shape[0] if self.__df is not None else 0

matchescu_reference_extraction-0.1.0/src/matchescu/data_sources/_list.py

@@ -0,0 +1,30 @@
+from typing import Iterable, Iterator
+
+from matchescu.typing import Trait
+from matchescu.data_sources._record import Record
+
+
+class ListDataSource:
+    def __init__(self, name: str, traits: Iterable[Trait]):
+        self.name = name
+        self.traits = traits
+        self._items: list[Record] = []
+
+    def append(self, item: Record | Iterable[Record]) -> "ListDataSource":
+        if isinstance(item, Iterable):
+            item = Record.merge(item)
+        self._items.append(item)
+        return self
+
+    def extend(
+        self, items: Iterable[Record] | Iterable[Iterable[Record]]
+    ) -> "ListDataSource":
+        for item in items:
+            self.append(item)
+        return self
+
+    def __iter__(self) -> Iterator[Record]:
+        return iter(self._items)
+
+    def __len__(self) -> int:
+        return len(self._items)

matchescu_reference_extraction-0.1.0/src/matchescu/data_sources/_record.py

@@ -0,0 +1,57 @@
+from collections.abc import Iterable
+from typing import Any, Iterator
+
+
+class Record:
+    def __init__(self, value: Iterable) -> None:
+        if isinstance(value, Record):
+            self.__values = value.__values
+            self.__attr_names = value.__attr_names
+        else:
+            tuples = list(self.__init_data(value))
+            self.__values = tuple(x[1] for x in tuples)
+            self.__attr_names = {x[0]: i for i, x in enumerate(tuples)}
+
+    @staticmethod
+    def __get_attr_key(key: str | int) -> str:
+        return f"column_{key}" if isinstance(key, int) else key
+
+    @staticmethod
+    def merge(records: Iterable["Record"]) -> "Record":
+        merge_record = {}
+        for record in records:
+            merge_record.update(
+                {k: record.__values[i] for k, i in record.__attr_names.items()}
+            )
+        return Record(merge_record)
+
+    def __init_data(self, value: Iterable) -> Iterable[tuple]:
+        if isinstance(value, dict):
+            return ((self.__get_attr_key(k), v) for k, v in value.items())
+        elif isinstance(value, (tuple, list, set)):
+            return ((self.__get_attr_key(i), v) for i, v in enumerate(value, start=1))
+        raise ValueError(
+            f"can't initialize data record from '{type(value).__name__}' values"
+        )
+
+    def __getitem__(self, key: str | int) -> Any:
+        if isinstance(key, str):
+            return (
+                self.__values[self.__attr_names[key]]
+                if key in self.__attr_names
+                else None
+            )
+        elif isinstance(key, int):
+            return self.__values[key]
+        raise ValueError(f"can't get data record using '{type(key).__name__}' keys")
+
+    def __getattr__(self, key: str) -> Any:
+        if key not in self.__attr_names:
+            raise AttributeError(f"Record attribute '{key}' not found")
+        return self[key]
+
+    def __len__(self) -> int:
+        return len(self.__values)
+
+    def __iter__(self) -> Iterator[Any]:
+        return iter(self.__values)

matchescu_reference_extraction-0.1.0/src/matchescu/extraction/__init__.py

@@ -0,0 +1,10 @@
+from matchescu.extraction._traits import Traits
+from matchescu.extraction._basic import EntityReferenceExtractionFromRecords
+from matchescu.extraction._csv import CsvEntityReferenceExtraction
+
+
+__all__ = [
+    "CsvEntityReferenceExtraction",
+    "EntityReferenceExtractionFromRecords",
+    "Traits",
+]

matchescu_reference_extraction-0.1.0/src/matchescu/extraction/_basic.py

@@ -0,0 +1,9 @@
+from typing import Iterable
+from matchescu.data import EntityReferenceExtraction
+
+from matchescu.data_sources._record import Record
+
+
+class EntityReferenceExtractionFromRecords(EntityReferenceExtraction[Record]):
+    def _merge_records(self, records: Iterable[Record]) -> Record:
+        return Record.merge(records)

matchescu_reference_extraction-0.1.0/src/matchescu/extraction/_csv.py

@@ -0,0 +1,20 @@
+from typing import Generator
+from matchescu.typing import RecordAdapter, DataSource
+
+from matchescu.data_sources import CsvDataSource
+from matchescu.extraction._basic import EntityReferenceExtractionFromRecords
+
+
+class CsvEntityReferenceExtraction(EntityReferenceExtractionFromRecords):
+    def __init__(
+        self,
+        ds: CsvDataSource,
+        record_adapter: RecordAdapter,
+    ):
+        super().__init__(
+            ds, record_adapter, CsvEntityReferenceExtraction._sample_csv_records
+        )
+
+    @staticmethod
+    def _sample_csv_records(ds: DataSource) -> Generator[list, None, None]:
+        yield from map(lambda x: [x], ds)

matchescu_reference_extraction-0.1.0/src/matchescu/extraction/_traits.py

@@ -0,0 +1,62 @@
+import locale
+
+from typing import Any, Callable, Iterator, Union
+
+from matchescu.typing import Trait
+from matchescu.data_sources import Record
+
+
+def _process_string(value: Any) -> str:
+    return str(value)
+
+
+def _process_int(value: Any) -> int:
+    return int(value)
+
+
+def _process_float(value: Any) -> float:
+    return float(value)
+
+
+def _process_currency(value: Any) -> float | None:
+    if value is None:
+        return None
+    str_val = str(value).replace(",", "").strip()
+    return locale.atof(str_val.lstrip("$"))
+
+
+class RecordExtractionTrait:
+    def __init__(self, mapping: Callable[[Any], Any], keys: list[int | str]) -> None:
+        self.__mapping = mapping
+        self.__keys = keys
+
+    def __call__(self, input_records: list[Record]) -> Record:
+        if len(input_records) == 0:
+            return Record([])
+        return Record(
+            {key: self.__mapping(input_records[0][key]) for key in self.__keys}
+        )
+
+
+class Traits:
+    def __init__(self):
+        self._traits = []
+
+    def string(self, keys: list[Union[int, str]]) -> "Traits":
+        self._traits.append(RecordExtractionTrait(_process_string, keys))
+        return self
+
+    def int(self, keys: list[Union[int, str]]) -> "Traits":
+        self._traits.append(RecordExtractionTrait(_process_int, keys))
+        return self
+
+    def float(self, keys: list[Union[int, str]]) -> "Traits":
+        self._traits.append(RecordExtractionTrait(_process_float, keys))
+        return self
+
+    def currency(self, keys: list[Union[int, str]]) -> "Traits":
+        self._traits.append(RecordExtractionTrait(_process_currency, keys))
+        return self
+
+    def __iter__(self) -> Iterator[Trait]:
+        return iter(self._traits)