pyprocessors-iptc_mapper 0.5.7__py3-none-any.whl

pyprocessors_iptc_mapper/__init__.py

@@ -0,0 +1,2 @@
+"""Sherpa IPTC category mapper"""
+__version__ = "0.5.7"

pyprocessors_iptc_mapper/iptc_mapper.py

@@ -0,0 +1,85 @@
+from collections import namedtuple
+from functools import lru_cache
+from itertools import groupby
+from pathlib import Path
+from typing import Type, cast, List, Dict
+
+import pandas as pd
+from log_with_context import add_logging_context, Logger
+from pydantic import BaseModel, Field
+from pymultirole_plugins.v1.processor import ProcessorParameters, ProcessorBase
+from pymultirole_plugins.v1.schema import Document, Category
+
+logger = Logger("pymultirole")
+
+
+class IPTCMapperParameters(ProcessorParameters):
+    label2iptc_mapping: Dict[str, str] = Field(None, description="Label to iptc mediatopic mapping",
+                                               extra="key:label")
+
+
+def is_iptc_category(cat: Category):
+    ok = "0123456789_"
+    return all(c in ok for c in cat.labelName)
+
+
+class IPTCMapper(ProcessorBase):
+    """Create categories from annotations"""
+
+    def process(
+            self, documents: List[Document], parameters: ProcessorParameters
+    ) -> List[Document]:  # noqa: C901
+        params: IPTCMapperParameters = cast(IPTCMapperParameters, parameters)
+        topics = get_mediatopics()
+        mapping = {k: topics.get(v, None) for k, v in params.label2iptc_mapping.items()} if (params.label2iptc_mapping and len(
+            params.label2iptc_mapping) > 0) else {}
+        for document in documents:
+            with add_logging_context(docid=document.identifier):
+                iptc_categories = {}
+                mapped_categories = {}
+                sorted_categories = sorted(document.categories or [], key=lambda c: c.labelName, reverse=True)
+                for is_iptc, group in groupby(sorted_categories, is_iptc_category):
+                    if is_iptc:
+                        iptc_categories.update({c.labelName: c for c in group})
+                    else:
+                        for c in group:
+                            if c.labelName in mapping:
+                                mediatopic = mapping[c.labelName]
+                                clabel = f"{mediatopic.path.replace('_', '/')} ({mediatopic.label})"
+                                mapped_categories[mediatopic.path] = Category(labelName=mediatopic.path, label=clabel, score=c.score)
+                categories = list(iptc_categories.values())
+                for k, mapped_category in mapped_categories.items():
+                    if k not in iptc_categories:
+                        categories.append(mapped_category)
+                document.categories = categories
+        return documents
+
+    @classmethod
+    def get_model(cls) -> Type[BaseModel]:
+        return IPTCMapperParameters
+
+
+Mediatopic = namedtuple("Mediatopic", ["label", "code", "path"])
+
+
+@lru_cache(maxsize=None)
+def get_mediatopics():
+    iptc = Path(__file__).parent / "IPTC-MediaTopic-NewsCodes.xlsx"
+    topics = {}
+    iptc_codes = pd.read_excel(iptc, header=1).fillna(value="")
+    levels = [None] * 6
+    for index, row in iptc_codes.iterrows():
+        topic_url = row["NewsCode-QCode (flat)"]
+        topic_code = topic_url[len("medtop:"):]
+        for lev in range(0, 6):
+            level = f"Level{lev + 1}/NewsCode"
+            level_url = row[level]
+            if level_url:
+                level_code = level_url[len("medtop:"):]
+                levels[lev] = level_code
+                break
+        path = "_".join(levels[0: lev + 1])
+        topics[topic_code] = Mediatopic(
+            label=row["Name (en-GB)"], code=topic_code, path=path
+        )
+    return topics

pyprocessors_iptc_mapper-0.5.7.dist-info/METADATA

@@ -0,0 +1,95 @@
+Metadata-Version: 2.4
+Name: pyprocessors-iptc_mapper
+Version: 0.5.7
+Summary: Sherpa IPTC category mapper
+Home-page: https://github.com/oterrier/pyprocessors_iptc_mapper/
+Keywords: 
+Author: Olivier Terrier
+Author-email: olivier.terrier@kairntech.com
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: System Administrators
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python :: 3.8
+License-File: LICENSE
+Requires-Dist: pymultirole-plugins>=0.5.152,<0.6.0
+Requires-Dist: collections-extended
+Requires-Dist: pandas>=1.2.3,<=1.3.5
+Requires-Dist: openpyxl==3.0.7
+Requires-Dist: log-with-context
+Requires-Dist: flit ; extra == "dev"
+Requires-Dist: pre-commit ; extra == "dev"
+Requires-Dist: bump2version ; extra == "dev"
+Requires-Dist: sphinx ; extra == "docs"
+Requires-Dist: sphinx-rtd-theme ; extra == "docs"
+Requires-Dist: m2r2 ; extra == "docs"
+Requires-Dist: sphinxcontrib.apidoc ; extra == "docs"
+Requires-Dist: jupyter_sphinx ; extra == "docs"
+Requires-Dist: pytest ; extra == "test"
+Requires-Dist: pytest-cov ; extra == "test"
+Requires-Dist: pytest-flake8 ; extra == "test"
+Requires-Dist: pytest-black ; extra == "test"
+Requires-Dist: flake8==3.9.2 ; extra == "test"
+Requires-Dist: tox ; extra == "test"
+Requires-Dist: dirty-equals ; extra == "test"
+Provides-Extra: dev
+Provides-Extra: docs
+Provides-Extra: test
+
+# pyprocessors_iptc_mapper
+
+[![license](https://img.shields.io/github/license/oterrier/pyprocessors_iptc_mapper)](https://github.com/oterrier/pyprocessors_iptc_mapper/blob/master/LICENSE)
+[![tests](https://github.com/oterrier/pyprocessors_iptc_mapper/workflows/tests/badge.svg)](https://github.com/oterrier/pyprocessors_iptc_mapper/actions?query=workflow%3Atests)
+[![codecov](https://img.shields.io/codecov/c/github/oterrier/pyprocessors_iptc_mapper)](https://codecov.io/gh/oterrier/pyprocessors_iptc_mapper)
+[![docs](https://img.shields.io/readthedocs/pyprocessors_iptc_mapper)](https://pyprocessors_iptc_mapper.readthedocs.io)
+[![version](https://img.shields.io/pypi/v/pyprocessors_iptc_mapper)](https://pypi.org/project/pyprocessors_iptc_mapper/)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/pyprocessors_iptc_mapper)](https://pypi.org/project/pyprocessors_iptc_mapper/)
+
+CategoriesFromAnnotations annotations coming from different annotators
+
+## Installation
+
+You can simply `pip install pyprocessors_iptc_mapper`.
+
+## Developing
+
+### Pre-requesites
+
+You will need to install `flit` (for building the package) and `tox` (for orchestrating testing and documentation building):
+
+```
+python3 -m pip install flit tox
+```
+
+Clone the repository:
+
+```
+git clone https://github.com/oterrier/pyprocessors_iptc_mapper
+```
+
+### Running the test suite
+
+You can run the full test suite against all supported versions of Python (3.8) with:
+
+```
+tox
+```
+
+### Building the documentation
+
+You can build the HTML documentation with:
+
+```
+tox -e docs
+```
+
+The built documentation is available at `docs/_build/index.html.
+

pyprocessors_iptc_mapper-0.5.7.dist-info/RECORD

@@ -0,0 +1,8 @@
+pyprocessors_iptc_mapper/IPTC-MediaTopic-NewsCodes.xlsx,sha256=yM-qPrJHyqEg9zelQsQJfvMpJjvHJt_isR8YdoRHw_4,703544
+pyprocessors_iptc_mapper/__init__.py,sha256=A-4L20hcpPg7KfTApbYFFhXUV4Lc47dsQkUFCz_2RoU,56
+pyprocessors_iptc_mapper/iptc_mapper.py,sha256=5wVZ1XqpIDer6o5w-piW1G_ECt9Mxd4xeOI72NmcLXI,3469
+pyprocessors_iptc_mapper-0.5.7.dist-info/entry_points.txt,sha256=MnrwTzmr_PKkxRlg6MaCfM5oIVFXUa1P-2F0JWxWh4A,84
+pyprocessors_iptc_mapper-0.5.7.dist-info/licenses/LICENSE,sha256=QPypQEap7i1K1jF4X_JOx3MvFyVupJe4-rr6h2DuXME,1082
+pyprocessors_iptc_mapper-0.5.7.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+pyprocessors_iptc_mapper-0.5.7.dist-info/METADATA,sha256=fGs7459R3Zm8iGnT4uvmBpt811ZrVn26MXTuor_ipy0,3470
+pyprocessors_iptc_mapper-0.5.7.dist-info/RECORD,,

pyprocessors_iptc_mapper-0.5.7.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: flit 3.12.0
+Root-Is-Purelib: true
+Tag: py3-none-any

pyprocessors_iptc_mapper-0.5.7.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[pyprocessors.plugins]
+iptc_mapper=pyprocessors_iptc_mapper.iptc_mapper:IPTCMapper
+

pyprocessors_iptc_mapper-0.5.7.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 Olivier Terrier
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.