txpyfind 0.1.0__tar.gz

txpyfind-0.1.0/.github/workflows/pypi.yml

@@ -0,0 +1,33 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [created]
+  workflow_dispatch:
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.8'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel build twine
+        pip install -r requirements.txt
+    - name: Build and publish
+      env:
+        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+      run: |
+        python -m build .
+        twine upload dist/*

txpyfind-0.1.0/.github/workflows/test.yml

@@ -0,0 +1,38 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Tests
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version:  ${{ matrix.python-version }}
+    - name: Lint with flake8
+      run: |
+        pip install flake8
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 txpyfind --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 txpyfind --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Install package
+      run: pip install .
+    - name: Install dependencies for test
+      run: pip install pytest coverage
+    - name: Run test and coverage
+      run: |
+        coverage erase
+        coverage run -m pytest test
+        coverage report
+        coverage html

txpyfind-0.1.0/.gitignore

@@ -0,0 +1,151 @@
+# generated by setuptools-scm
+txpyfind/_version.py
+
+# Development files / directories
+_/
+dev/
+*-dev
+*-dev.*
+dev-*
+dev.*
+
+# Editor
+*.swp
+*.swo
+*~
+.vscode
+
+# Data
+*.json
+
+# <https://github.com/github/gitignore/blob/master/Python.gitignore>
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/

txpyfind-0.1.0/CHANGELOG

@@ -0,0 +1,5 @@
+0.1.0 [2025-05-31]
+- first published release
+
+0.0.0 [2022-08-20]
+- dummy release

txpyfind-0.1.0/PKG-INFO

@@ -0,0 +1,69 @@
+Metadata-Version: 2.1
+Name: txpyfind
+Version: 0.1.0
+Summary: enables Pythonic access to data exports from TYPO3-find
+Author: Donatus Herre
+Author-email: donatus.herre@slub-dresden.de
+License: GPLv3
+Project-URL: homepage, https://github.com/slub/txpyfind
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.7
+Description-Content-Type: text/x-rst
+License-File: LICENSE
+
+========
+txpyfind
+========
+
+``txpyfind`` enables access to data exports from `TYPO3-find <https://github.com/subugoe/typo3-find>`_
+in Python. Details on the TYPO3-find setup required for data exports can be found in the section
+`Data export <https://github.com/subugoe/typo3-find#data-export>`_ in the README file of that repository.
+
+The three JSON formats ``json-all``, ``json-solr-results`` and ``raw-solr-response`` are already available
+in the TYPO3 extension, see the
+`partials <https://github.com/subugoe/typo3-find/tree/main/Resources/Private/Partials/Formats>`_ used
+to create the three formats.
+
+You can use the client class available in this Python package to query these exports. A simple parser
+for the returned JSON objects is also available.
+
+Installation
+============
+
+... via PyPI
+~~~~~~~~~~~~
+
+.. code-block:: bash
+
+   pip install txpyfind
+
+... or from Github source
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+   pip install git+https://github.com/herreio/txpyfind.git
+
+
+Usage Example
+=============
+
+.. code-block:: python
+
+   from txpyfind.client import Find
+   # create Find instance
+   slub_find = Find("https://katalog.slub-dresden.de", document_path="id", export_format="json-ld")
+   # retrieve JSON-LD data (detail view)
+   slub_ld_doc = slub_find.get_document("0-1132486122")
+   # retrieve JSON-LD data (query view)
+   slub_ld_q_default = slub_find.get_query("manfred bonitz")
+   # ...
+
+See `slubfind <https://github.com/slub/slubfind>`_ for a full setup example.

txpyfind-0.1.0/README.rst

@@ -0,0 +1,49 @@
+========
+txpyfind
+========
+
+``txpyfind`` enables access to data exports from `TYPO3-find <https://github.com/subugoe/typo3-find>`_
+in Python. Details on the TYPO3-find setup required for data exports can be found in the section
+`Data export <https://github.com/subugoe/typo3-find#data-export>`_ in the README file of that repository.
+
+The three JSON formats ``json-all``, ``json-solr-results`` and ``raw-solr-response`` are already available
+in the TYPO3 extension, see the
+`partials <https://github.com/subugoe/typo3-find/tree/main/Resources/Private/Partials/Formats>`_ used
+to create the three formats.
+
+You can use the client class available in this Python package to query these exports. A simple parser
+for the returned JSON objects is also available.
+
+Installation
+============
+
+... via PyPI
+~~~~~~~~~~~~
+
+.. code-block:: bash
+
+   pip install txpyfind
+
+... or from Github source
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+   pip install git+https://github.com/herreio/txpyfind.git
+
+
+Usage Example
+=============
+
+.. code-block:: python
+
+   from txpyfind.client import Find
+   # create Find instance
+   slub_find = Find("https://katalog.slub-dresden.de", document_path="id", export_format="json-ld")
+   # retrieve JSON-LD data (detail view)
+   slub_ld_doc = slub_find.get_document("0-1132486122")
+   # retrieve JSON-LD data (query view)
+   slub_ld_q_default = slub_find.get_query("manfred bonitz")
+   # ...
+
+See `slubfind <https://github.com/slub/slubfind>`_ for a full setup example.

txpyfind-0.1.0/pyproject.toml

@@ -0,0 +1,39 @@
+[build-system]
+requires = [
+         "setuptools>=42",
+         "setuptools_scm[toml]",
+         "wheel",
+]  # PEP 508 specifications.
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "txpyfind"
+description = "enables Pythonic access to data exports from TYPO3-find"
+readme = "README.rst"
+authors = [
+        {name = "Donatus Herre"},
+        {email = "donatus.herre@slub-dresden.de"}
+]
+license = {text = "GPLv3"}
+keywords = []
+classifiers = [
+            "Programming Language :: Python :: 3",
+            "Operating System :: OS Independent",
+            "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+            "Development Status :: 3 - Alpha",
+            "Intended Audience :: Developers",
+            "Intended Audience :: Education",
+            "Intended Audience :: Science/Research",
+            "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dynamic = ["version", "dependencies"]
+requires-python = ">=3.7"
+
+[project.urls]
+homepage = "https://github.com/slub/txpyfind"
+
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt"]}
+
+[tool.setuptools_scm]
+write_to = "txpyfind/_version.py"

txpyfind-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

txpyfind-0.1.0/test/test_slub.py

@@ -0,0 +1,46 @@
+from txpyfind.client import Find
+
+def test_get_document():
+    # create Find instance
+    slub_find = Find("https://katalog.slub-dresden.de", document_path="id", export_format="json-ld")
+    # retrieve JSON-LD data (detail view)
+    slub_ld_doc = slub_find.get_document("0-1132486122")
+    assert slub_ld_doc is not None
+    assert '@graph' in slub_ld_doc.raw
+    assert any(slub_ld_doc.raw['@graph'])
+    assert '@id' in slub_ld_doc.raw['@graph'][0]
+    assert 'katalog.slub-dresden.de' in slub_ld_doc.raw['@graph'][0]['@id']
+
+def test_get_query():
+    # create Find instance
+    slub_find = Find("https://katalog.slub-dresden.de", document_path="id", export_format="json-ld")
+    # retrieve JSON-LD data (query view)
+    slub_ld_doc = slub_find.get_query("manfred bonitz")
+    assert slub_ld_doc is not None
+    assert '@graph' in slub_ld_doc.raw
+    assert any(slub_ld_doc.raw['@graph'])
+    assert '@id' in slub_ld_doc.raw['@graph'][0]
+
+def test_get_query_via_url():
+    # create Find instance
+    slub_find = Find("https://katalog.slub-dresden.de", document_path="id", export_format="json-ld")
+    # retrieve JSON-LD data (query view)
+    slub_ld_doc = slub_find.get_query_via_url("https://katalog.slub-dresden.de/?tx_find_find%5Bq%5D%5Bdefault%5D=manfred+bonitz")
+    assert slub_ld_doc is not None
+    assert '@graph' in slub_ld_doc.raw
+    assert any(slub_ld_doc.raw['@graph'])
+    assert '@id' in slub_ld_doc.raw['@graph'][0]
+
+def test_pagination():
+    # create Find instance
+    slub_find = Find("https://katalog.slub-dresden.de", document_path="id")
+    # scroll SOLR JSON data (query view + pagination)
+    slub_solr_docs = slub_find.scroll_get_query("manfred bonitz", batch=10)
+    assert slub_solr_docs is not None
+    assert len(slub_solr_docs) > 0
+    # stream SOLR JSON data (query view + pagination)
+    slub_solr_docs2 = slub_find.stream_get_query("manfred bonitz", batch=10)
+    assert slub_solr_docs2 is not None
+    slub_solr_docs2 = list(slub_solr_docs2)
+    assert len(slub_solr_docs2) > 0
+    assert len(slub_solr_docs) == len(slub_solr_docs2)

txpyfind-0.1.0/txpyfind/__init__.py

@@ -0,0 +1,7 @@
+"""
+With ``txpyfind`` you can access data exports from TYPO3-find.
+"""
+from ._version import __version__, version, version_tuple
+from . import client, parser
+
+__all__ = ["__version__", "version", "version_tuple", "client", "parser"]

txpyfind-0.1.0/txpyfind/_version.py

@@ -0,0 +1,21 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)

txpyfind-0.1.0/txpyfind/client.py

@@ -0,0 +1,333 @@
+"""
+client module of ``txpyfind`` package
+"""
+import re
+import logging
+from . import utils
+from .parser import JSONResponse
+from .urlparse import URLParser
+
+
+class Find:
+    """
+    ``Find`` class from ``txpyfind.client`` module
+    """
+
+    def __init__(
+            self,
+            base_url,
+            document_path=None,
+            query_types=None,
+            facets=None,
+            count_limit=100,
+            sort_pattern=None,
+            export_format="raw-solr-response",
+            export_page=1369315139,
+            parser_class=JSONResponse):
+        self.base_url = base_url
+        self.document_path = document_path
+        if query_types is None:
+            query_types = ["default"]
+        self.query_types = query_types
+        self.facets = facets
+        self.count_limit = count_limit
+        self.sort_pattern = sort_pattern
+        self.document_url = None
+        if document_path is not None:
+            self.document_url = f"{self.base_url}/{self.document_path}"
+        self.export_page = export_page
+        self.export_format = export_format
+        self.parser_class = parser_class
+        self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
+
+    def set_data_params(
+            self,
+            url,
+            data_format=None,
+            type_num=None):
+        """
+        add data exports parameters as initial parameters to given URL
+        """
+        if data_format is None:
+            data_format = self.export_format
+        if type_num is None:
+            type_num = self.export_page
+        return utils.set_tx_param_data(
+            url,
+            data_format=data_format,
+            type_num=type_num)
+
+    def add_data_params(
+            self,
+            url,
+            data_format=None,
+            type_num=None):
+        """
+        add data exports parameters as subsequent parameters to given URL
+        """
+        if data_format is None:
+            data_format = self.export_format
+        if type_num is None:
+            type_num = self.export_page
+        return utils.add_tx_param_data(
+            url,
+            data_format=data_format,
+            type_num=type_num)
+
+    def add_facet_params(
+            self,
+            url,
+            facet=None):
+        """
+        add facet parameters as subsequent parameters to given URL
+        """
+        if facet is not None:
+            for fct in facet:
+                if isinstance(fct, str):
+                    if isinstance(self.facets, list) and fct not in self.facets:
+                        self.logger.warning("Unknown facet type %s!", fct)
+                        continue
+                    url = utils.add_tx_param(url, ["facet", fct, utils.url_encode(facet[fct])], 1)
+                elif isinstance(fct, dict):
+                    for k in fct:
+                        if isinstance(self.facets, list) and k not in self.facets:
+                            self.logger.warning("Unknown facet type %s!", k)
+                            continue
+                        url = utils.add_tx_param(url, ["facet", k, utils.url_encode(fct[k])], 1)
+        return url
+
+    def url_parser(self, url):
+        return URLParser(url)
+
+    def url_document(
+            self,
+            doc_id,
+            data_format=None,
+            type_num=None):
+        """
+        get the URL for the detail view of the document given by id
+        """
+        if self.document_url is not None:
+            doc_url = f"{self.document_url}/{doc_id}"
+            return self.set_data_params(
+                doc_url,
+                data_format=data_format,
+                type_num=type_num)
+        return None
+
+    def get_document(
+            self,
+            document_id,
+            data_format=None,
+            type_num=None,
+            parser_class=None):
+        """
+        get the detail view of the document given by id
+        """
+        url = self.url_document(
+            document_id,
+            data_format=data_format,
+            type_num=type_num)
+        if url is not None:
+            doc = utils.plain_request(url)
+            if doc is not None:
+                if parser_class is not None:
+                    return parser_class(doc)
+                if self.parser_class is not None:
+                    return self.parser_class(doc)
+                return doc
+        return None
+
+    def url_query(
+            self,
+            query,
+            qtype="default",
+            facet=None,
+            page=0,
+            count=0,
+            sort="",
+            data_format=None,
+            type_num=None):
+        """
+        get the URL for the query view
+        """
+        if qtype not in self.query_types:
+            self.logger.warning("Unknown query type!")
+            qtype = "default"
+        url = utils.set_tx_param(self.base_url, ["q", qtype], utils.url_encode(query))
+        url = self.add_data_params(url, data_format=data_format, type_num=type_num)
+        url = self.add_facet_params(url, facet=facet)
+        if page:
+            url = utils.add_tx_param(url, "page", page)
+        if count:
+            if count > self.count_limit:
+                self.logger.warning("Count %d exceeds limit!", count)
+                count = self.count_limit
+            url = utils.add_tx_param(url, "count", count)
+        if sort != "" and self.sort_pattern is not None and not isinstance(
+                self.sort_pattern.match(sort), re.Match):
+            self.logger.warning("Sort instruction %s is unknown!", sort)
+            sort = ""
+        if sort != "":
+            url = utils.add_tx_param(url, "sort", utils.url_encode(sort))
+        return url
+
+    def get_query(
+            self,
+            query,
+            qtype="default",
+            facet=None,
+            page=0,
+            count=0,
+            sort="",
+            data_format=None,
+            type_num=None,
+            parser_class=None):
+        """
+        get query view
+        """
+        url = self.url_query(
+            query,
+            qtype=qtype,
+            facet=facet,
+            page=page,
+            count=count,
+            sort=sort,
+            data_format=data_format,
+            type_num=type_num)
+        response = utils.plain_request(url)
+        if response is not None:
+            if parser_class is not None:
+                return parser_class(response)
+            if self.parser_class is not None:
+                return self.parser_class(response)
+            return response
+        return None
+
+    def get_query_via_url(
+            self,
+            url,
+            data_format=None,
+            type_num=None,
+            parser_class=None):
+        """
+        get query view via url
+        """
+        url = self.url_parser(url)
+        if url.is_ok:
+            return self.get_query(
+                url.query,
+                qtype=url.qtype,
+                facet=url.facets,
+                page=url.page,
+                count=url.count,
+                sort=url.sort,
+                data_format=data_format,
+                type_num=type_num,
+                parser_class=parser_class)
+        return None
+
+    def scroll_get_query(
+            self,
+            query,
+            qtype="default",
+            facet=None,
+            batch=20,
+            sort="",
+            data_format="raw-solr-response",
+            type_num=None,
+            parser_class=None):
+        """
+        get all pages of a query view
+        """
+        if data_format is None:
+            data_format = self.export_format
+        if data_format != "raw-solr-response":
+            self.logger.warning(
+                "Scrolling only supports data format of type 'raw-solr-response'!")
+            data_format = "raw-solr-response"
+        response = self.get_query(
+            query,
+            qtype=qtype,
+            facet=facet,
+            sort=sort,
+            data_format=data_format,
+            type_num=type_num,
+            parser_class=parser_class)
+        if hasattr(response, "raw") and isinstance(
+                response.raw, dict) and "response" in response.raw:
+            data = response.raw["response"]
+            total = data["numFound"]
+            docs = []
+            pages = int(total / batch) + (total % batch > 0)
+            for i in range(1, pages+1):
+                response_i = self.get_query(
+                    query,
+                    qtype=qtype,
+                    facet=facet,
+                    page=i,
+                    count=batch,
+                    sort=sort,
+                    data_format=data_format,
+                    type_num=type_num,
+                    parser_class=parser_class)
+                if hasattr(response_i, "raw") and isinstance(
+                        response_i.raw, dict) and "response" in response_i.raw:
+                    data_i = response_i.raw["response"]
+                    if "docs" in data_i:
+                        for doc in data_i["docs"]:
+                            docs.append(doc)
+            found = len(docs)
+            if total != found:
+                self.logger.warning(
+                    "Expected %d record%s for query %s. Got %d record%s.",
+                    total, 's' if total != 1 else '', query, found, 's' if found != 1 else '')
+            return docs
+        return None
+
+    def stream_get_query(
+            self,
+            query,
+            qtype="default",
+            facet=None,
+            batch=20,
+            sort="",
+            data_format="raw-solr-response",
+            type_num=None,
+            parser_class=None):
+        """
+        iterate all pages of a query view
+        """
+        if data_format is None:
+            data_format = self.export_format
+        if data_format != "raw-solr-response":
+            self.logger.warning(
+                "Streaming only supports data format of type 'raw-solr-response'!")
+            data_format = "raw-solr-response"
+        response = self.get_query(
+            query,
+            qtype=qtype,
+            facet=facet,
+            sort=sort,
+            data_format=data_format,
+            type_num=type_num,
+            parser_class=parser_class)
+        if hasattr(response, "raw") and isinstance(
+                response.raw, dict) and "response" in response.raw:
+            data = response.raw["response"]
+            total = data["numFound"]
+            pages = int(total / batch) + (total % batch > 0)
+            for i in range(1, pages+1):
+                response_i = self.get_query(
+                    query,
+                    qtype=qtype,
+                    facet=facet,
+                    page=i,
+                    count=batch,
+                    sort=sort,
+                    data_format=data_format)
+                if hasattr(response_i, "raw") and isinstance(
+                        response_i.raw, dict) and "response" in response_i.raw:
+                    data_i = response_i.raw["response"]
+                    if "docs" in data_i:
+                        yield from data_i["docs"]

txpyfind-0.1.0/txpyfind/parser.py

@@ -0,0 +1,44 @@
+"""
+parser module of ``txpyfind`` package
+"""
+import html
+import json
+import logging
+
+
+class JSONResponse:
+    """
+    ``JSONResponse`` class from ``txpyfind.parser`` module
+    """
+
+    def __init__(self, plain):
+        self.plain = plain
+        self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
+        try:
+            self.raw = json.loads(plain)
+        except json.decoder.JSONDecodeError as err:
+            self.logger.error(err)
+            self.raw = None
+        self.fields = self._names(raw=self.raw)
+
+    def _names(self, raw=None):
+        if raw is None:
+            raw = self.raw
+        if isinstance(raw, dict):
+            return list(raw.keys())
+        return []
+
+    def _field(self, name, raw=None):
+        if raw is None:
+            raw = self.raw
+        if isinstance(raw, dict) and name in raw:
+            return self._unescape(raw[name])
+        return None
+
+    def _unescape(self, value):
+        if isinstance(value, str):
+            return html.unescape(value.strip())
+        if isinstance(value, list) and len(value) > 0 and \
+                all(isinstance(v, str) and len(v.strip()) > 0 for v in value):
+            return [html.unescape(v.strip()) for v in value]
+        return value

txpyfind-0.1.0/txpyfind/urlparse.py

@@ -0,0 +1,141 @@
+"""
+urlparse module of ``txpyfind`` package
+"""
+import re
+from urllib.parse import unquote_plus as unquote
+
+QUERY = re.compile(r"tx_find_find\[q\]\[([^]]*)\]=([^&]*)")
+QUERY_AMP = re.compile(r"=([^&]*%26[^&]*)")
+SUBSTITUTE = "%#"
+FACET = re.compile(r"tx_find_find\[facet\]\[([^]]*)\]\[([^]]*)\]=1&?")
+PAGE = re.compile(r"tx_find_find\[page\]=(\d*)&?")
+COUNT = re.compile(r"tx_find_find\[count\]=(\d*)&?")
+SORT = re.compile(r"tx_find_find\[sort\]=([a-zA-Z]*)[+ ]([a-zA-Z]*)&?")
+
+
+class URLParser:
+    """
+    ``URLParser`` class from ``txpyfind.urlparse`` module
+    """
+
+    def __init__(self, url):
+        self.url = url
+        query_details = get_query(url)
+        self.is_ok = False
+        if len(query_details) > 1:
+            self.query = query_details[1]
+            self.qtype = query_details[0]
+            self.is_ok = True
+        self.facets = get_facets(url)
+        self.page = get_page(url)
+        self.count = get_count(url)
+        self.sort = get_sort(url)
+
+
+def preserve_ampersand(url):
+    """
+    preserve ampersand (``&``) in given URL
+    """
+    amps = QUERY_AMP.findall(url)
+    if len(amps) == 1:
+        url = url.replace(amps[0], amps[0].replace("%", SUBSTITUTE))
+        return url, True
+    return url, False
+
+
+def find_query(url):
+    """
+    find query parameter in given URL
+    """
+    url, ampersand = preserve_ampersand(url)
+    url = unquote(url)
+    if ampersand:
+        return [tuple(unquote(e.replace(SUBSTITUTE, "%"))
+                      for e in q)
+                for q in QUERY.findall(url)]
+    return QUERY.findall(url)
+
+
+def find_facets(url):
+    """
+    find facet parameters in given URL
+    """
+    return FACET.findall(unquote(url))
+
+
+def find_page(url):
+    """
+    find page parameter in given URL
+    """
+    return PAGE.findall(unquote(url))
+
+
+def find_count(url):
+    """
+    find count parameter in given URL
+    """
+    return COUNT.findall(unquote(url))
+
+
+def find_sort(url):
+    """
+    find sort parameter in given URL
+    """
+    return SORT.findall(unquote(url))
+
+
+def get_query(url):
+    """
+    get query parameter from given URL
+    """
+    query = find_query(url)
+    if len(query) == 1:
+        qtype = query[0][0]
+        qval = query[0][1]
+        qval = re.sub(" +", " ", qval)
+        qval = re.sub("^ *| *$", "", qval)
+        return qtype, qval
+    return ()
+
+
+def get_facets(url):
+    """
+    get facet parameters from given URL
+    """
+    facets = find_facets(url)
+    if len(facets) > 0:
+        fct = []
+        for facet in facets:
+            fct.append({facet[0]: facet[1]})
+        return fct
+    return {}
+
+
+def get_page(url):
+    """
+    get page parameter from given URL
+    """
+    page = find_page(url)
+    if len(page) == 1:
+        return int(page[0])
+    return 0
+
+
+def get_count(url):
+    """
+    get count parameter from given URL
+    """
+    count = find_count(url)
+    if len(count) > 0:
+        return int(count[0])
+    return 0
+
+
+def get_sort(url):
+    """
+    get sort parameter from given URL
+    """
+    sort = find_sort(url)
+    if len(sort) > 0:
+        return f"{sort[0][0]} {sort[0][1]}"
+    return ""

txpyfind-0.1.0/txpyfind/utils.py

@@ -0,0 +1,131 @@
+"""
+utils module of ``txpyfind`` package
+"""
+import json
+import logging
+from urllib.parse import quote_plus
+from urllib.request import Request, urlopen
+
+from ._version import __version__
+
+
+logger = logging.getLogger(__name__)
+
+
+def get_request(url):
+    """
+    send HTTP GET request to given URL
+    """
+    req = Request(url)
+    req.add_header("User-Agent", f"txpyfind {__version__}")
+    try:
+        with urlopen(req) as response:
+            if response.code == 200:
+                return response.read()
+            logger.error("HTTP %d GET %s", response.code, url)
+    except Exception as exc:
+        logger.error(exc)
+    return None
+
+
+def plain_request(url):
+    """
+    request data in plain text format from given URL
+    """
+    payload = get_request(url)
+    if isinstance(payload, bytes):
+        try:
+            return payload.decode()
+        except Exception as exc:
+            logger.error(exc)
+    return None
+
+
+def json_request(url):
+    """
+    request data in JSON format from given URL
+    """
+    plain = plain_request(url)
+    if isinstance(plain, str):
+        try:
+            return json.loads(plain)
+        except json.decoder.JSONDecodeError:
+            logger.error("Got faulty JSON from URL %s", url)
+    return None
+
+
+def url_encode(url):
+    """
+    encode given URL
+    """
+    return quote_plus(url)
+
+
+def set_param(url, key, value=None):
+    """
+    add initial parameter to given URL
+    """
+    url = f"{url}?{key}"
+    if value is not None:
+        url = f"{url}={value}"
+    return url
+
+
+def add_param(url, key, value=None):
+    """
+    add subsequent parameter to given URL
+    """
+    url = f"{url}&{key}"
+    if value is not None:
+        url = f"{url}={value}"
+    return url
+
+
+def tx_param(key, index=None):
+    """
+    create URL parameter for TYPO3-find
+    """
+    if isinstance(key, str):
+        k = f"[{key}]"
+    else:
+        k = "".join(f"[{k}]" for k in key)
+    if isinstance(index, int):
+        k += f"[{index}]"
+    return f"tx_find_find{k}"
+
+
+def set_tx_param(url, key, value, index=None):
+    """
+    add TYPO3-find parameter as initial parameter to given URL
+    """
+    return set_param(url, tx_param(key, index=index), value)
+
+
+def add_tx_param(url, key, value, index=None):
+    """
+    add TYPO3-find parameter as subsequent parameter to given URL
+    """
+    return add_param(url, tx_param(key,  index=index), value)
+
+
+def tx_param_data(data_format, type_num=1369315139):
+    """
+    create parameters for TYPO3-find data exports
+    """
+    param = f"{tx_param('format')}=data"
+    param = add_tx_param(param, "data-format", data_format)
+    return add_param(param, "type", type_num)
+
+
+def set_tx_param_data(url, data_format, type_num=1369315139):
+    """
+    add parameters for TYPO3-find data exports as initial parameters to given URL
+    """
+    return set_param(url, tx_param_data(data_format, type_num=type_num))
+
+
+def add_tx_param_data(url, data_format, type_num=1369315139):
+    """
+    add parameters for TYPO3-find data exports as subsequent parameters to given URL
+    """
+    return add_param(url, tx_param_data(data_format, type_num=type_num))

txpyfind-0.1.0/txpyfind.egg-info/PKG-INFO

@@ -0,0 +1,69 @@
+Metadata-Version: 2.1
+Name: txpyfind
+Version: 0.1.0
+Summary: enables Pythonic access to data exports from TYPO3-find
+Author: Donatus Herre
+Author-email: donatus.herre@slub-dresden.de
+License: GPLv3
+Project-URL: homepage, https://github.com/slub/txpyfind
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.7
+Description-Content-Type: text/x-rst
+License-File: LICENSE
+
+========
+txpyfind
+========
+
+``txpyfind`` enables access to data exports from `TYPO3-find <https://github.com/subugoe/typo3-find>`_
+in Python. Details on the TYPO3-find setup required for data exports can be found in the section
+`Data export <https://github.com/subugoe/typo3-find#data-export>`_ in the README file of that repository.
+
+The three JSON formats ``json-all``, ``json-solr-results`` and ``raw-solr-response`` are already available
+in the TYPO3 extension, see the
+`partials <https://github.com/subugoe/typo3-find/tree/main/Resources/Private/Partials/Formats>`_ used
+to create the three formats.
+
+You can use the client class available in this Python package to query these exports. A simple parser
+for the returned JSON objects is also available.
+
+Installation
+============
+
+... via PyPI
+~~~~~~~~~~~~
+
+.. code-block:: bash
+
+   pip install txpyfind
+
+... or from Github source
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+   pip install git+https://github.com/herreio/txpyfind.git
+
+
+Usage Example
+=============
+
+.. code-block:: python
+
+   from txpyfind.client import Find
+   # create Find instance
+   slub_find = Find("https://katalog.slub-dresden.de", document_path="id", export_format="json-ld")
+   # retrieve JSON-LD data (detail view)
+   slub_ld_doc = slub_find.get_document("0-1132486122")
+   # retrieve JSON-LD data (query view)
+   slub_ld_q_default = slub_find.get_query("manfred bonitz")
+   # ...
+
+See `slubfind <https://github.com/slub/slubfind>`_ for a full setup example.

txpyfind-0.1.0/txpyfind.egg-info/SOURCES.txt

@@ -0,0 +1,20 @@
+.gitignore
+CHANGELOG
+LICENSE
+README.rst
+pyproject.toml
+requirements.txt
+.github/workflows/pypi.yml
+.github/workflows/test.yml
+test/__init__.py
+test/test_slub.py
+txpyfind/__init__.py
+txpyfind/_version.py
+txpyfind/client.py
+txpyfind/parser.py
+txpyfind/urlparse.py
+txpyfind/utils.py
+txpyfind.egg-info/PKG-INFO
+txpyfind.egg-info/SOURCES.txt
+txpyfind.egg-info/dependency_links.txt
+txpyfind.egg-info/top_level.txt

txpyfind-0.1.0/txpyfind.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

txpyfind-0.1.0/txpyfind.egg-info/top_level.txt

@@ -0,0 +1 @@
+txpyfind