ckanext-csvwmapandtransform 1.0.2__tar.gz → 1.1.0__tar.gz

ckanext_csvwmapandtransform-1.1.0/.github/workflows/pypi.yml

@@ -0,0 +1,105 @@
+name: upload-pypi
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: write
+  id-token: write
+
+
+jobs:
+  release-build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set version from release tag
+        run: echo "VERSION=${{ github.event.release.tag_name }}" >> $GITHUB_ENV
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+
+      - name: Build release distributions
+        run: |
+          # NOTE: put your own distribution build steps here.
+          python -m pip install build
+          python -m build
+
+      - name: Upload distributions
+        uses: actions/upload-artifact@v4
+        with:
+          name: release-dists
+          path: dist/
+
+  pypi-publish:
+    runs-on: ubuntu-latest
+    needs:
+      - release-build
+    permissions:
+      # IMPORTANT: this permission is mandatory for trusted publishing
+      id-token: write
+
+    # Dedicated environments with protections for publishing are strongly recommended.
+    # For more information, see: https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment#deployment-protection-rules
+    environment:
+      name: pypi
+      # OPTIONAL: uncomment and update to include your PyPI project URL in the deployment status:
+      #url: https://pypi.org/p/${GITHUB_REPOSITORY#*/}
+      #
+      # ALTERNATIVE: if your GitHub Release name is the PyPI project version string
+      # ALTERNATIVE: exactly, uncomment the following line instead:
+      url: https://pypi.org/project/${GITHUB_REPOSITORY#*/}/${{ github.event.release.name }}
+
+    steps:
+      - name: Retrieve release distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: release-dists
+          path: dist/
+
+      - name: Publish release distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist/
+  sign-package:
+    runs-on: ubuntu-latest
+    needs:
+    - release-build
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: release-dists
+        path: dist/
+    - name: Sign the dists with Sigstore
+      uses: sigstore/gh-action-sigstore-python@v3.0.0
+      with:
+        inputs: >-
+          ./dist/*.tar.gz
+          ./dist/*.whl
+  publish-to-testpypi:
+    name: Publish Python 🐍 distribution 📦 to TestPyPI
+    needs:
+    - release-build
+    runs-on: ubuntu-latest
+
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/${GITHUB_REPOSITORY#*/}
+
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: release-dists
+        path: dist/
+    - name: Publish distribution 📦 to TestPyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        repository-url: https://test.pypi.org/legacy/

ckanext_csvwmapandtransform-1.1.0/.github/workflows/test.yml

@@ -0,0 +1,78 @@
+name: Tests
+on: [push, pull_request]
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+      - name: Install requirements
+        run: pip install flake8 pycodestyle
+      - name: Check syntax
+        run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan
+
+  test:
+    needs: lint
+    strategy:
+      matrix:
+        include:
+          - ckan-version: "2.11"
+            ckan-image: "ckan/ckan-dev:2.11-py3.10"
+            solr-image: "2.11-solr9-spatial"
+            harvester-version: 'master'
+          - ckan-version: "2.10"
+            ckan-image: "ckan/ckan-dev:2.10-py3.10"
+            solr-image: "2.10-solr9-spatial"
+            harvester-version: 'master'
+      fail-fast: false
+
+    name: CKAN ${{ matrix.ckan-version }}, Solr ${{ matrix.solr-image }}
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ matrix.ckan-image }}
+      options: --user root
+    services:
+      solr:
+        image: ckan/ckan-solr:${{ matrix.solr-image }}
+      postgres:
+        image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+      redis:
+        image: redis:7
+    env:
+      CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
+      CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
+      CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test
+      CKAN_SOLR_URL: http://solr:8983/solr/ckan
+      CKAN_REDIS_URL: redis://redis:6379/1
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Fix git safe directory
+      run: git config --global --add safe.directory '*'
+    - name: Install requirements
+      run: |
+        pip install -r requirements.txt
+        pip install -r dev-requirements.txt
+    - name: Install requirements (common)
+      run: |
+        pip install -e .
+        # Replace default path to CKAN core config file with the one on the container
+        sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
+    - name: Setup extension
+      run: |
+        ckan -c test.ini db init
+    - name: Run tests
+      run: pytest --ckan-ini=test.ini --cov=ckanext.csvwmapandtransform --disable-warnings ckanext/csvwmapandtransform || ([ $? = 5 ] && exit 0 || exit $?)
+    - name: Upload coverage report to codecov
+      uses: codecov/codecov-action@v1
+      with:
+        file: ./coverage.xml
+
+

ckanext_csvwmapandtransform-1.1.0/.gitignore

@@ -0,0 +1,157 @@
+# Byte-compiled / optimized / DLL files
+ckanect/csvwmapandtransform/__pycache__/
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

{ckanext_csvwmapandtransform-1.0.2/ckanext_csvwmapandtransform.egg-info → ckanext_csvwmapandtransform-1.1.0}/PKG-INFO

@@ -1,33 +1,28 @@
 Metadata-Version: 2.4
 Name: ckanext-csvwmapandtransform
-Version: 1.0.2
-Summary: Extension automatically generating csvw metadata for uploaded textual tabular data.
+Version: 1.1.0
+Summary: CKAN extension facilitating YARRRML mappings in conjunction with RDFConverter to run automatic transforms matching best fitting maps as a background job.
 Home-page: https://github.com/Mat-O-Lab/ckanext-csvwmapandtransform
 Author: Thomas Hanke
-Author-email: thomas.hanke@iwm.fraunhofer.de
+Author-email: Thomas Hanke <thomas.hanke@iwm.fraunhofer.de>
 License: AGPL
-Keywords: CKAN CSVW CSV JSON-LD
+Project-URL: Homepage, https://github.com/Mat-O-Lab/ckanext-csvwmapandtransform
+Keywords: CKAN,CSVW,CSV,JSON-LD
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)
 Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
+Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: ckanapi
 Requires-Dist: requests>=2.4.2
-Requires-Dist: pytest
-Requires-Dist: pytest-ckan
-Dynamic: author
-Dynamic: author-email
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: license
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-ckan; extra == "dev"
 Dynamic: license-file
-Dynamic: requires-dist
-Dynamic: summary
 
 [![Tests](https://github.com/Mat-O-Lab/ckanext-csvwmapandtransform/actions/workflows/test.yml/badge.svg)](https://github.com/Mat-O-Lab/ckanext-csvwmapandtransform/actions/workflows/test.yml)
 

ckanext_csvwmapandtransform-1.1.0/ckanext/csvwmapandtransform/__init__.py

@@ -0,0 +1,22 @@
+# encoding: utf-8
+"""
+ckanext-csvwmapandtransform
+
+A CKAN extension to automate mapping of CSVW metadata documents to knowledge graphs
+and run automatic pipelines.
+"""
+
+# Expose package version from setuptools-scm
+try:
+    from importlib.metadata import version, PackageNotFoundError
+except ImportError:
+    # Python < 3.8 fallback
+    from importlib_metadata import version, PackageNotFoundError
+
+try:
+    __version__ = version("ckanext-csvwmapandtransform")
+except PackageNotFoundError:
+    # Package is not installed
+    __version__ = "unknown"
+
+__all__ = ["__version__"]

{ckanext_csvwmapandtransform-1.0.2 → ckanext_csvwmapandtransform-1.1.0}/ckanext/csvwmapandtransform/action.py

@@ -10,7 +10,6 @@ else:
 
 
 import datetime
-import itertools
 import json
 import os
 from typing import Any
@@ -37,33 +36,21 @@ def find_first_matching_id(dicts: list, key: str, value: str):
 
 
 def csvwmapandtransform_find_mappings(context: Context, data_dict):
-    mapping_group_id = find_first_matching_id(
-        toolkit.get_action("group_list")({}, {"all_fields": True}),
-        key="name",
-        value=MAPPING_GROUP,
+    result = toolkit.get_action("package_search")(
+        {"ignore_auth": True},
+        {"fq": f"groups:{MAPPING_GROUP}", "rows": 1000, "include_private": False},
     )
-    if mapping_group_id:
-        mapping_group = toolkit.get_action("group_show")(
-            {"ignore_auth": True}, {"id": mapping_group_id, "include_datasets": True}
-        )
-    else:
-        log.warn("group with name mappings not found!")
-        mapping_group = create_group(MAPPING_GROUP)
-        log.info("created group mappings")
-    packages = mapping_group.get("packages", None)
-
-    if packages:
-        packages = [
-            toolkit.get_action("package_show")({}, {"id": package["id"]})
-            for package in packages
-        ]
-        resources = list(
-            itertools.chain.from_iterable(
-                [package["resources"] for package in packages]
-            )
-        )
-    else:
-        resources = list()
+    packages = result.get("results", [])
+    if not packages:
+        log.warn("group with name mappings not found or empty!")
+    resources = [
+        r
+        for pkg in packages
+        for r in pkg.get("resources", [])
+        if r.get("format", "").upper() == "YAML"
+    ]
+    if packages and not resources:
+        log.warning("Packages found in group '%s' but 0 YAML resources", MAPPING_GROUP)
     return resources
 
 
@@ -135,7 +122,8 @@ def csvwmapandtransform_transform_status(
                 "key": "csvwmapandtransform",
             },
         )
-    except:
+    except toolkit.ObjectNotFound:
+        log.debug("No task found for resource %s", res_id)
         status = None
     else:
         value = json.loads(task["value"])
@@ -160,11 +148,11 @@ def csvwmapandtransform_transform_status(
         db_job = db.get_job(job_id)
 
         if db_job and db_job.get("logs"):
-            for log in db_job["logs"]:
-                if "timestamp" in log and isinstance(
-                    log["timestamp"], datetime.datetime
+            for entry in db_job["logs"]:
+                if "timestamp" in entry and isinstance(
+                    entry["timestamp"], datetime.datetime
                 ):
-                    log["timestamp"] = log["timestamp"].isoformat()
+                    entry["timestamp"] = entry["timestamp"].isoformat()
         status = dict(status, **db_job)
         # status['task_info']=db_job
     return status
@@ -217,25 +205,8 @@ def enqueue_transform(res_id, res_name, res_url, dataset_id, operation):
         assume_task_stale_after = datetime.timedelta(seconds=3600)
         assume_task_stillborn_after = datetime.timedelta(seconds=int(5))
         if existing_task.get("state") == "pending":
-            # queued_res_ids = [
-            #     re.search(r"'resource_id': u?'([^']+)'",
-            #               job.description).groups()[0]
-            #     for job in get_queue().get_jobs()
-            #     if 'xloader_to_datastore' in str(job)  # filter out test_job etc
-            # ]
             updated = parse_iso_date(existing_task["last_updated"])
             time_since_last_updated = datetime.datetime.utcnow() - updated
-            # if (res_id not in queued_res_ids
-            #         and time_since_last_updated > assume_task_stillborn_after):
-            #     # it's not on the queue (and if it had just been started then
-            #     # its taken too long to update the task_status from pending -
-            #     # the first thing it should do in the xloader job).
-            #     # Let it be restarted.
-            #     log.info('A pending task was found %r, but its not found in '
-            #              'the queue %r and is %s hours old',
-            #              existing_task['id'], queued_res_ids,
-            #              time_since_last_updated)
-            # elif time_since_last_updated > assume_task_stale_after:
             if time_since_last_updated > assume_task_stale_after:
                 # it's been a while since the job was last updated - it's more
                 # likely something went wrong with it and the state wasn't
@@ -347,7 +318,13 @@ def csvwmapandtransform_hook(context, data_dict):
     # task['task_info'] = job_info
     resubmit = False
 
-    if status in ("complete", "running_but_viewable"):
+    if status == "error":
+        log.debug("job errored now update job db at: {}".format(task))
+        db.init()
+        job_id = json.loads(task["value"])["job_id"]
+        db.mark_job_as_errored(job_id, data_dict.get("error"))
+
+    elif status in ("complete", "running_but_viewable"):
         # Create default views for resource if necessary (only the ones that
         # require data to be in the DataStore)
         resource_dict = toolkit.get_action("resource_show")(context, {"id": res_id})

{ckanext_csvwmapandtransform-1.0.2 → ckanext_csvwmapandtransform-1.1.0}/ckanext/csvwmapandtransform/assets/script.js

@@ -52,7 +52,7 @@ ckan.module('csvwmapandtransform', function (jQuery) {
                       + value.icon +
                       "'></i><div class='alert alert-"
                       + value.alertlevel +
-                      " mb-0 mt-3' role='alert'>"
+                      " mb-0 mt-1 py-1' role='alert'>"
                       + value.message +
                       "</div><span class='date' title='timestamp'>"
                       + value.timestamp +

ckanext_csvwmapandtransform-1.1.0/ckanext/csvwmapandtransform/i18n/ckanext-csvwmapandtransform.pot

@@ -0,0 +1,108 @@
+# Translations template for ckanext-csvwmapandtransform.
+# Copyright (C) 2025 ORGANIZATION
+# This file is distributed under the same license as the
+# ckanext-csvwmapandtransform project.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2025.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: ckanext-csvwmapandtransform 0.0.0\n"
+"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
+"POT-Creation-Date: 2025-05-14 07:19+0000\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.15.0\n"
+
+#: ckanext/csvwmapandtransform/helpers.py:17
+msgid "Complete"
+msgstr ""
+
+#: ckanext/csvwmapandtransform/helpers.py:18
+msgid "Pending"
+msgstr ""
+
+#: ckanext/csvwmapandtransform/helpers.py:19
+msgid "Submitting"
+msgstr ""
+
+#: ckanext/csvwmapandtransform/helpers.py:20
+msgid "Error"
+msgstr ""
+
+#: ckanext/csvwmapandtransform/helpers.py:25
+msgid "Not Uploaded Yet"
+msgstr ""
+
+#: ckanext/csvwmapandtransform/views.py:29 ckanext/csvwmapandtransform/views.py:52
+#: ckanext/csvwmapandtransform/views.py:89 ckanext/csvwmapandtransform/views.py:163
+msgid "Not authorized to see this page"
+msgstr ""
+
+#: ckanext/csvwmapandtransform/templates/csvwmapandtransform/create_mapping.html:13
+msgid "Map"
+msgstr ""
+
+#: ckanext/csvwmapandtransform/templates/csvwmapandtransform/create_mapping.html:19
+#: ckanext/csvwmapandtransform/templates/csvwmapandtransform/transform.html:24
+msgid "View resource"
+msgstr ""
+
+#: ckanext/csvwmapandtransform/templates/csvwmapandtransform/create_mapping.html:28
+msgid "Map resource"
+msgstr ""
+
+#: ckanext/csvwmapandtransform/templates/csvwmapandtransform/transform.html:18
+#: ckanext/csvwmapandtransform/templates/package/resource_read.html:6
+#: ckanext/csvwmapandtransform/templates/package/snippets/resource_item.html:17
+msgid "Transform"
+msgstr ""
+
+#: ckanext/csvwmapandtransform/templates/csvwmapandtransform/transform.html:30
+msgid "Edit resource"
+msgstr ""
+
+#: ckanext/csvwmapandtransform/templates/csvwmapandtransform/transform.html:32
+msgid "Views"
+msgstr ""
+
+#: ckanext/csvwmapandtransform/templates/csvwmapandtransform/transform.html:36
+msgid "Transform Status"
+msgstr ""
+
+#: ckanext/csvwmapandtransform/templates/csvwmapandtransform/transform.html:45
+msgid "Run Transformation"
+msgstr ""
+
+#: ckanext/csvwmapandtransform/templates/csvwmapandtransform/transform.html:48
+msgid "Delete"
+msgstr ""
+
+#: ckanext/csvwmapandtransform/templates/csvwmapandtransform/transform.html:63
+msgid "The status of the service (Green means available, Red means unavailable)"
+msgstr ""
+
+#: ckanext/csvwmapandtransform/templates/csvwmapandtransform/transform.html:78
+msgid "Status"
+msgstr ""
+
+#: ckanext/csvwmapandtransform/templates/csvwmapandtransform/transform.html:82
+msgid "Last updated"
+msgstr ""
+
+#: ckanext/csvwmapandtransform/templates/csvwmapandtransform/transform.html:87
+msgid "Never"
+msgstr ""
+
+#: ckanext/csvwmapandtransform/templates/csvwmapandtransform/transform.html:92
+msgid "Graph Update Log"
+msgstr ""
+
+#: ckanext/csvwmapandtransform/templates/package/snippets/resource_item.html:10
+msgid "Create Mapping"
+msgstr ""
+