gn2pg_client 1.9.0.dev0__tar.gz → 1.10.0__tar.gz

gn2pg_client-1.10.0/PKG-INFO

@@ -0,0 +1,107 @@
+Metadata-Version: 2.4
+Name: gn2pg_client
+Version: 1.10.0
+Summary: Import tool from GeoNature to a PostgreSQL database through Export module API (client side)
+License: AGPL-3.0-or-later
+License-File: LICENSE
+Keywords: GeoNature,Export,SINP,opendata,biodiversity
+Author: Collectif
+Maintainer: lpofredc
+Maintainer-email: frederic.cloitre@lpo.fr
+Requires-Python: >=3.10,<4.0
+Classifier: Development Status :: 1 - Planning
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: GNU Affero General Public License v3
+Classifier: License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)
+Classifier: Natural Language :: French
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: Topic :: Documentation
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: System :: Installation/Setup
+Classifier: Topic :: System :: Software Distribution
+Provides-Extra: dashboard
+Requires-Dist: alembic (>=1.16,<2.0)
+Requires-Dist: coloredlogs (>=0.0.0)
+Requires-Dist: flask (>=2,<3) ; extra == "dashboard"
+Requires-Dist: flask-admin (>=1,<2) ; extra == "dashboard"
+Requires-Dist: flask-sqlalchemy (>=3,<4) ; extra == "dashboard"
+Requires-Dist: gunicorn (>=0.0.0) ; extra == "dashboard"
+Requires-Dist: psycopg2-binary (>=2,<3)
+Requires-Dist: python-decouple (>=3.8,<4.0) ; extra == "dashboard"
+Requires-Dist: python-dotenv (>=1,<2) ; extra == "dashboard"
+Requires-Dist: requests (>=2,<3)
+Requires-Dist: schema (>=0,<1)
+Requires-Dist: sqlalchemy (>=2.0,<3.0)
+Requires-Dist: toml (>=0,<1)
+Project-URL: Documentation, https://lpoaura.github.io/GN2PG/
+Project-URL: Homepage, https://github.com/lpoaura/gn2gn_client/
+Description-Content-Type: text/markdown
+
+# GN2PG Client
+
+![https://www.python.org/](https://img.shields.io/badge/python-3.10+-yellowgreen)
+![https://www.postgresql.org/](https://img.shields.io/badge/PostgreSQL-10+-blue)
+![https://python-poetry.org/](https://img.shields.io/badge/packaging%20tool-poetry-important)
+![https://github.com/psf/black](https://img.shields.io/badge/code%20style-black-black)
+![https://opensource.org/licenses/AGPL-3.0](https://img.shields.io/badge/licence-AGPL--3.0-blue)
+
+This project provides an import data from [GeoNature] instances to a PostgreSQL database (client side).
+Widely inspired from [ClientApiVN](https://framagit.org/lpo/Client_API_VN/)
+
+> [!WARNING]
+> The minimum version of the source GeoNature instance required for the incremental update must be a version 2.12.0
+
+![Project logo](./docs/_static/src_gn2pg.png)
+
+## Project Setup
+
+GN2PG Client can be installed by running `pip`. It requires Python 3.10 or above to run.
+
+```bash
+pip install gn2pg-client
+```
+
+## Issues
+
+Please report any bugs or requests that you have using the [GitHub issue tracker](https://github.com/lpoaura/gn2pg_client/issues)!
+
+## [HowTo](https://lpoaura.github.io/GN2PG/usage/howto.html)
+
+## [Contributing](https://lpoaura.github.io/GN2PG/development/contribute.html)
+
+## Licence
+
+[GNU AGPLv3](https://www.gnu.org/licenses/gpl.html)
+
+## Team
+
+<a href="">
+<img height="100px" src="https://auvergne-rhone-alpes.lpo.fr/wp-content/uploads/LPO_AuRA.svg" title="DREAL AuRA">
+</a>
+
+[@lpofredc](https://github.com/lpofredc/) ([LPO Auvergne-Rhône-Alpes](https://github.com/lpoaura/)), lead developer
+
+---
+
+<a href="https://github.com/lpoaura/GN2PG/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=lpoaura/GN2PG" />
+</a>
+
+---
+
+With the financial support of the [DREAL Auvergne-Rhône-Alpes](http://www.auvergne-rhone-alpes.developpement-durable.gouv.fr/) and the [Office français de la biodiversité](https://www.ofb.gouv.fr/).
+
+
+<img height="100px" src="https://data.lpo-aura.org/web/images/blocmarque_pref_region_auvergne_rhone_alpes_rvb_web.png" title="DREAL AuRA"> <img height="100px" src="https://www.ofb.gouv.fr/sites/default/files/logo-ofb.png" title="OFB" style="padding-left:10px;">
+
+[geonature]: https://geonature.fr
+

gn2pg_client-1.10.0/README.md

@@ -0,0 +1,58 @@
+# GN2PG Client
+
+![https://www.python.org/](https://img.shields.io/badge/python-3.10+-yellowgreen)
+![https://www.postgresql.org/](https://img.shields.io/badge/PostgreSQL-10+-blue)
+![https://python-poetry.org/](https://img.shields.io/badge/packaging%20tool-poetry-important)
+![https://github.com/psf/black](https://img.shields.io/badge/code%20style-black-black)
+![https://opensource.org/licenses/AGPL-3.0](https://img.shields.io/badge/licence-AGPL--3.0-blue)
+
+This project provides an import data from [GeoNature] instances to a PostgreSQL database (client side).
+Widely inspired from [ClientApiVN](https://framagit.org/lpo/Client_API_VN/)
+
+> [!WARNING]
+> The minimum version of the source GeoNature instance required for the incremental update must be a version 2.12.0
+
+![Project logo](./docs/_static/src_gn2pg.png)
+
+## Project Setup
+
+GN2PG Client can be installed by running `pip`. It requires Python 3.10 or above to run.
+
+```bash
+pip install gn2pg-client
+```
+
+## Issues
+
+Please report any bugs or requests that you have using the [GitHub issue tracker](https://github.com/lpoaura/gn2pg_client/issues)!
+
+## [HowTo](https://lpoaura.github.io/GN2PG/usage/howto.html)
+
+## [Contributing](https://lpoaura.github.io/GN2PG/development/contribute.html)
+
+## Licence
+
+[GNU AGPLv3](https://www.gnu.org/licenses/gpl.html)
+
+## Team
+
+<a href="">
+<img height="100px" src="https://auvergne-rhone-alpes.lpo.fr/wp-content/uploads/LPO_AuRA.svg" title="DREAL AuRA">
+</a>
+
+[@lpofredc](https://github.com/lpofredc/) ([LPO Auvergne-Rhône-Alpes](https://github.com/lpoaura/)), lead developer
+
+---
+
+<a href="https://github.com/lpoaura/GN2PG/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=lpoaura/GN2PG" />
+</a>
+
+---
+
+With the financial support of the [DREAL Auvergne-Rhône-Alpes](http://www.auvergne-rhone-alpes.developpement-durable.gouv.fr/) and the [Office français de la biodiversité](https://www.ofb.gouv.fr/).
+
+
+<img height="100px" src="https://data.lpo-aura.org/web/images/blocmarque_pref_region_auvergne_rhone_alpes_rvb_web.png" title="DREAL AuRA"> <img height="100px" src="https://www.ofb.gouv.fr/sites/default/files/logo-ofb.png" title="OFB" style="padding-left:10px;">
+
+[geonature]: https://geonature.fr

gn2pg_client-1.10.0/gn2pg/alembic/env.py

@@ -0,0 +1,72 @@
+"""Alembic environment for GN2PG's configurable import schema."""
+
+# Alembic exposes context operations dynamically at migration runtime.
+# pylint: disable=no-member
+
+from alembic import context
+from sqlalchemy import engine_from_config, pool, text
+from sqlalchemy.dialects import postgresql
+
+from gn2pg.database import build_metadata
+
+config = context.config
+schema = config.attributes.get("schema", "gn2pg_import")
+target_metadata = build_metadata(schema)
+
+
+def include_name(name, object_type, parent_names) -> bool:
+    """Restrict reflection to the configured GN2PG schema."""
+    if object_type == "schema":
+        return name == schema
+    if object_type == "table":
+        return parent_names.get("schema_name") == schema
+    return True
+
+
+def run_migrations_offline() -> None:
+    """Run migrations without creating an Engine."""
+    context.configure(
+        url=config.get_main_option("sqlalchemy.url"),
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+        include_schemas=True,
+        include_name=include_name,
+        version_table="alembic_version",
+        version_table_schema=schema,
+        schema=schema,
+    )
+    quoted_schema = postgresql.dialect().identifier_preparer.quote(schema)
+    context.execute(f"CREATE SCHEMA IF NOT EXISTS {quoted_schema}")
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations with a live database connection."""
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+    with connectable.connect() as connection:
+        quoted_schema = connection.dialect.identifier_preparer.quote(schema)
+        connection.execute(text(f"CREATE SCHEMA IF NOT EXISTS {quoted_schema}"))
+        connection.commit()
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            include_schemas=True,
+            include_name=include_name,
+            version_table="alembic_version",
+            version_table_schema=schema,
+            schema=schema,
+        )
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

gn2pg_client-1.10.0/gn2pg/alembic/script.py.mako

@@ -0,0 +1,25 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+"""
+
+from typing import Optional, Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+revision: str = ${repr(up_revision)}
+down_revision: Optional[str] = ${repr(down_revision)}
+branch_labels: Optional[Union[str, Sequence[str]]] = ${repr(branch_labels)}
+depends_on: Optional[Union[str, Sequence[str]]] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

gn2pg_client-1.10.0/gn2pg/alembic/versions/20260818_01_initial_schema.py

@@ -0,0 +1,134 @@
+"""Create the initial GN2PG import schema.
+
+Revision ID: 20260818_01
+Revises: None
+"""
+
+# pylint: disable=invalid-name,no-member
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+revision = "20260818_01"
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def _schema() -> str:
+    return op.get_context().opts.get("schema", "gn2pg_import")
+
+
+def _index(schema: str, table: str, column: str) -> str:
+    return f"ix_{schema}_{table}_{column}"
+
+
+def upgrade() -> None:
+    """Create extensions and an immutable snapshot of the initial schema."""
+    schema = _schema()
+    op.execute('CREATE EXTENSION IF NOT EXISTS "pgcrypto"')
+    op.execute('CREATE EXTENSION IF NOT EXISTS "uuid-ossp"')
+    op.create_table(
+        "import_log",
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column("source", sa.String(), nullable=False),
+        sa.Column("controler", sa.String(), nullable=False),
+        sa.Column("xfer_type", sa.String(), nullable=True),
+        sa.Column("xfer_status", sa.String(), nullable=True),
+        sa.Column("xfer_start_ts", sa.DateTime(), nullable=False),
+        sa.Column("xfer_end_ts", sa.DateTime(), nullable=True),
+        sa.Column("api_count_items", sa.Integer(), server_default=sa.text("0"), nullable=False),
+        sa.Column("api_count_errors", sa.Integer(), server_default=sa.text("0"), nullable=False),
+        sa.Column("data_count_upserts", sa.Integer(), server_default=sa.text("0"), nullable=False),
+        sa.Column("data_count_delete", sa.Integer(), server_default=sa.text("0"), nullable=False),
+        sa.Column("data_count_errors", sa.Integer(), server_default=sa.text("0"), nullable=False),
+        sa.Column(
+            "metadata_count_upserts", sa.Integer(), server_default=sa.text("0"), nullable=False
+        ),
+        sa.Column(
+            "metadata_count_errors", sa.Integer(), server_default=sa.text("0"), nullable=False
+        ),
+        sa.Column(
+            "xfer_filters",
+            postgresql.JSONB(astext_type=sa.Text()),
+            server_default=sa.text("'{}'::jsonb"),
+            nullable=True,
+        ),
+        sa.Column("comment", sa.Text(), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+        schema=schema,
+    )
+    op.create_index(
+        _index(schema, "import_log", "source"), "import_log", ["source"], schema=schema
+    )
+    op.create_index(
+        _index(schema, "import_log", "xfer_type"), "import_log", ["xfer_type"], schema=schema
+    )
+    op.create_table(
+        "error_log",
+        sa.Column("source", sa.String(), nullable=False),
+        sa.Column("uuid", postgresql.UUID(), nullable=False),
+        sa.Column("controler", sa.String(), nullable=False),
+        sa.Column("last_ts", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
+        sa.Column("item", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+        sa.Column("error", sa.String(), nullable=True),
+        sa.Column("import_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["import_id"],
+            [f"{schema}.import_log.id"],
+            onupdate="CASCADE",
+            ondelete="CASCADE",
+        ),
+        schema=schema,
+    )
+    op.create_index(
+        _index(schema, "error_log", "import_id"), "error_log", ["import_id"], schema=schema
+    )
+    op.create_index(_index(schema, "error_log", "uuid"), "error_log", ["uuid"], schema=schema)
+    op.create_table(
+        "data_json",
+        sa.Column("source", sa.String(), nullable=False),
+        sa.Column("controler", sa.String(), nullable=False),
+        sa.Column("type", sa.String(), nullable=False),
+        sa.Column("id_data", sa.Integer(), nullable=False),
+        sa.Column("uuid", postgresql.UUID(), nullable=True),
+        sa.Column("item", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
+        sa.Column("update_ts", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
+        sa.Column("import_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(["import_id"], [f"{schema}.import_log.id"], onupdate="CASCADE"),
+        sa.PrimaryKeyConstraint("id_data", "source", "type", name="pk_source_data"),
+        sa.UniqueConstraint("uuid", name="unique_uuid"),
+        schema=schema,
+    )
+    op.create_index(
+        _index(schema, "data_json", "id_data"), "data_json", ["id_data"], schema=schema
+    )
+    op.create_index(_index(schema, "data_json", "uuid"), "data_json", ["uuid"], schema=schema)
+    op.create_table(
+        "metadata_json",
+        sa.Column("source", sa.String(), nullable=False),
+        sa.Column("controler", sa.String(), nullable=False),
+        sa.Column("type", sa.String(), nullable=False),
+        sa.Column("level", sa.String(), nullable=False),
+        sa.Column("uuid", postgresql.UUID(), nullable=False),
+        sa.Column("item", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
+        sa.Column("update_ts", sa.DateTime(), server_default=sa.text("now()"), nullable=False),
+        sa.Column("import_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(["import_id"], [f"{schema}.import_log.id"], onupdate="CASCADE"),
+        sa.PrimaryKeyConstraint("uuid", "source", name="pk_source_metadata"),
+        sa.UniqueConstraint("uuid", name="metadata_unique_uuid"),
+        schema=schema,
+    )
+    op.create_index(
+        _index(schema, "metadata_json", "uuid"), "metadata_json", ["uuid"], schema=schema
+    )
+
+
+def downgrade() -> None:
+    """Drop GN2PG tables while preserving the schema and shared extensions."""
+    schema = _schema()
+    op.drop_table("metadata_json", schema=schema)
+    op.drop_table("data_json", schema=schema)
+    op.drop_table("error_log", schema=schema)
+    op.drop_table("import_log", schema=schema)