spinta 0.2.dev25__tar.gz → 0.2.dev27__tar.gz

spinta-0.2.dev27/PKG-INFO

@@ -0,0 +1,309 @@
+Metadata-Version: 2.4
+Name: spinta
+Version: 0.2.dev27
+Summary: A platform for describing, extracting, transforming, loading and serving open data.
+License: MIT
+License-File: LICENSE
+Author: Mantas Zimnickas
+Author-email: sirexas@gmail.com
+Requires-Python: >=3.10,<4.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Environment :: Web Environment
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: System Administrators
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: POSIX
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Database
+Classifier: Topic :: Database :: Database Engines/Servers
+Classifier: Topic :: Database :: Front-Ends
+Provides-Extra: all
+Provides-Extra: http
+Provides-Extra: migrations
+Requires-Dist: GeoAlchemy2
+Requires-Dist: Shapely
+Requires-Dist: XlsxWriter
+Requires-Dist: aiofiles
+Requires-Dist: aiohttp
+Requires-Dist: alembic (>=1.11.0,<1.12.0) ; extra == "migrations" or extra == "all"
+Requires-Dist: asyncpg
+Requires-Dist: authlib (>=1.0.0)
+Requires-Dist: cachetools
+Requires-Dist: cbor2 (>=5.9.0,<6.0.0)
+Requires-Dist: click
+Requires-Dist: dask[dataframe]
+Requires-Dist: docutils (>=0.21.2,<0.22.0)
+Requires-Dist: fsspec
+Requires-Dist: gunicorn ; extra == "http" or extra == "all"
+Requires-Dist: jinja2
+Requires-Dist: jsonpatch
+Requires-Dist: lark-parser
+Requires-Dist: lxml
+Requires-Dist: markdown
+Requires-Dist: msgpack
+Requires-Dist: multipledispatch
+Requires-Dist: nh3
+Requires-Dist: numpy
+Requires-Dist: openpyxl
+Requires-Dist: pandas (>=2.3.3,<3)
+Requires-Dist: phonenumbers
+Requires-Dist: pprintpp
+Requires-Dist: psutil
+Requires-Dist: psycopg2-binary
+Requires-Dist: pydantic
+Requires-Dist: pymongo (<=4.8.0)
+Requires-Dist: pyproj (==3.6.1) ; python_version < "3.13"
+Requires-Dist: pyproj (>=3.7.0) ; python_version >= "3.13"
+Requires-Dist: python-multipart
+Requires-Dist: pytz
+Requires-Dist: rdflib
+Requires-Dist: redis (>=6.4.0,<7.0.0)
+Requires-Dist: requests
+Requires-Dist: ruamel.yaml
+Requires-Dist: setuptools
+Requires-Dist: setuptools-scm
+Requires-Dist: sqlalchemy (>=1.4,<1.5)
+Requires-Dist: sqlalchemy-utils (<=0.41.2)
+Requires-Dist: sqlean-py (<=3.49.1)
+Requires-Dist: sqlparse
+Requires-Dist: starlette (>=0.52.1,<1.0.0)
+Requires-Dist: tabulate
+Requires-Dist: toposort
+Requires-Dist: tqdm
+Requires-Dist: typer[all]
+Requires-Dist: ujson
+Requires-Dist: unidecode
+Requires-Dist: uvicorn ; extra == "http" or extra == "all"
+Requires-Dist: xlrd
+Requires-Dist: xmltodict
+Requires-Dist: zeep (>=4.3.1,<5.0.0)
+Project-URL: Bug Tracker, https://github.com/atviriduomenys/spinta/issues
+Project-URL: Documentation, https://spinta.readthedocs.io/
+Project-URL: Homepage, https://github.com/atviriduomenys/spinta
+Project-URL: Repository, https://github.com/atviriduomenys/spinta
+Description-Content-Type: text/x-rst
+
+.. default-role:: literal
+
+Spinta is a command line tool and REST JSON API service for publishing and
+mapping data between different physical data models, JSON API and semantic data
+models. It supports a great deal of data schemes and formats.
+
+.. image:: https://gitlab.com/atviriduomenys/spinta/badges/master/pipeline.svg
+   :target: https://gitlab.com/atviriduomenys/spinta/commits/master
+
+.. image:: https://gitlab.com/atviriduomenys/spinta/badges/master/coverage.svg
+   :target: https://gitlab.com/atviriduomenys/spinta/commits/master
+
+::
+
+    Physical data        Different       Real time         REST JSON
+       sources            formats      transformation         API
+                    
+                          +-----+
+                     +--> | SQL | --------->|
+       +------+      |    +-----+           |
+       | file | ---->|                      |
+       +------+      |    +-----+           |
+                     +--> | CSV | --------->|
+      +--------+     |    +-----+           |         +---------------+
+      | DB API | --->|                      | ------->| REST JSON API |
+      +--------+     |    +------+          |         +---------------+
+                     |    | JSON | -------->|
+     +----------+    |    +------+          |
+     | REST API | -->|                      |
+     +----------+    |    +-----+           |
+                     +--> | XML | --------->|
+                          +-----+
+
+
+
+
+Purpose
+=======
+
+- **Describe your data**: You can automatically generate data structure
+  description table (*Manifest*) from many different data sources.
+
+- **Extract your data**: Once you have your data structure in *Manifest* tables,
+  you can extract data from multiple external data sources. Extracted data are
+  validated and transformed using rules defined in *Manifest* table. Finally,
+  data can be stored into internal database in order to provide fast and
+  flexible access to data.
+  
+- **Transform your data**: Data transformations are applied in real time, when
+  reading data from source. This puts some limitations on transformation side, but
+  allows data to be streamed in real time.
+
+- **Publish your data**: Once you have your data loaded into internal
+  database, you can publish data using API. API is generated automatically using
+  *Manifest* tables and provides extracted data in many different formats. For
+  example if original data source was a CSV file, now you have a flexible API,
+  that can talk JSON, RDF, SQL, CSV and other formats.
+
+
+Features
+========
+
+- Simple 15 column table format for describing data structures (you can use
+  any spreadsheet software to manage metadata of your data)
+
+- Internal data storage with pluggable backends (PostgreSQL or Mongo)
+
+- Build-in async API server built on top of Starlette_ for data publishing
+
+- Simple web based data browser.
+
+- Convenient command-line interface
+
+- Public or restricted API access via OAuth protocol using build-in access
+  management.
+
+- Simple DSL_ for querying, transforming and validating data.
+
+- Low memory consumption for data of any size
+
+- Support for many different data sources
+
+- Advanced data extraction even from dynamic API.
+
+- Compatible with DCAT_ and `Frictionless Data Specifications`_.
+
+.. _Starlette: https://www.starlette.io/
+.. _DSL: https://en.wikipedia.org/wiki/Domain-specific_language
+.. _DCAT: https://www.w3.org/TR/vocab-dcat-2/
+.. _Frictionless Data Specifications: https://specs.frictionlessdata.io/
+
+
+.. note::
+
+    Currently this project is under heavy development and is in pre-alpha stage.
+    So many things might not work and many things can change. Use it at your own
+    risk.
+
+
+Example
+=======
+
+If you have an SQLite database:
+
+.. code-block:: sh
+
+    $ sqlite3 sqlite.db <<EOF
+    CREATE TABLE COUNTRY (
+        NAME TEXT
+    );
+    EOF
+
+You can get a limited API and simple web based data browser with a single
+command:
+
+.. code-block:: sh
+
+    $ spinta run -r sql sqlite:///sqlite.db
+
+Then you can generate metadata table (*manifest*) like this:
+
+.. code-block:: sh
+
+    $ spinta inspect -r sql sqlite:///sqlite.db
+    d | r | b | m | property | type   | ref | source              | prepare | level | access | uri | title | description
+    dataset                  |        |     |                     |         |       |        |     |       |
+      | sql                  | sql    |     | sqlite:///sqlite.db |         |       |        |     |       |
+                             |        |     |                     |         |       |        |     |       |
+      |   |   | Country      |        |     | COUNTRY             |         |       |        |     |       |
+      |   |   |   | name     | string |     | NAME                |         | 3     | open   |     |       |
+
+Generated data structure table can be saved into a CSV file:
+
+.. code-block:: sh
+
+    $ spinta inspect -r sql sqlite:///sqlite.db -o manifest.csv
+
+Missing peaces in metadata can be filled using any Spreadsheet software.
+
+Once you done editing metadata, you can test it via web based data browser or
+API:
+
+.. code-block:: sh
+
+    $ spinta run --mode external manifest.csv
+
+Once you are satisfied with metadata, you can generate a new metadata table for
+publishing, removing all traces of original data source:
+
+.. code-block:: sh
+
+    $ spinta copy --no-source --access open manifest.csv manifest-public.csv
+
+Now you have matadata for publishing, but all things about original data
+source are gone. In order to publish data, you need to copy external data to
+internal data store. To do that, first you need to initialize internal data
+store:
+
+.. code-block:: sh
+
+    $ spinta config add backend my_backend postgresql postgresql://localhost/db
+    $ spinta config add manifest my_manifest tabular manifest-public.csv
+    $ spinta migrate
+
+Once internal database is initialized, you can push external data into it:
+
+.. code-block:: sh
+
+    $ spinta push --access open manifest.csv
+
+And now you can publish data via full featured API with a web based data
+browser:
+
+.. code-block:: json
+
+    $ spinta run
+
+You can access your data like this:
+
+.. code-block:: json
+
+    $ http :8000/dataset/sql/Country
+    HTTP/1.1 200 OK
+    content-type: application/json
+
+    {
+        "_data": [
+            {
+                "_type": "dataset/sql/Country",
+                "_id": "abdd1245-bbf9-4085-9366-f11c0f737c1d",
+                "_rev": "16dabe62-61e9-4549-a6bd-07cecfbc3508",
+                "_txn": "792a5029-63c9-4c07-995c-cbc063aaac2c",
+                "name": "Vilnius"
+            }
+        ]
+    }
+
+    $ http :8000/dataset/sql/Country/abdd1245-bbf9-4085-9366-f11c0f737c1d
+    HTTP/1.1 200 OK
+    content-type: application/json
+
+    {
+        "_type": "dataset/sql/Country",
+        "_id": "abdd1245-bbf9-4085-9366-f11c0f737c1d",
+        "_rev": "16dabe62-61e9-4549-a6bd-07cecfbc3508",
+        "_txn": "792a5029-63c9-4c07-995c-cbc063aaac2c",
+        "name": "Vilnius"
+    }
+
+    $ http :8000/dataset/sql/Country/abdd1245-bbf9-4085-9366-f11c0f737c1d?select(name)
+    HTTP/1.1 200 OK
+    content-type: application/json
+
+    {
+        "name": "Vilnius"
+    }
+

spinta-0.2.dev27/pyproject.toml

@@ -0,0 +1,184 @@
+[tool.poetry]
+name = "spinta"
+version = "0.2dev27"
+description = "A platform for describing, extracting, transforming, loading and serving open data."
+authors = ["Mantas Zimnickas <sirexas@gmail.com>"]
+license = "MIT"
+readme = "README.rst"
+homepage = "https://github.com/atviriduomenys/spinta"
+repository = "https://github.com/atviriduomenys/spinta"
+documentation = "https://spinta.readthedocs.io/"
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Environment :: Web Environment",
+    "Intended Audience :: Developers",
+    "Intended Audience :: System Administrators",
+    "Operating System :: POSIX",
+    "Operating System :: POSIX :: Linux",
+    "Topic :: Database",
+    "Topic :: Database :: Database Engines/Servers",
+    "Topic :: Database :: Front-Ends",
+]
+packages = [
+    { include = "spinta" },
+]
+
+
+[tool.poetry.urls]
+"Bug Tracker" = "https://github.com/atviriduomenys/spinta/issues"
+
+
+[tool.poetry.scripts]
+spinta = "spinta.cli.main:app"
+
+
+[tool.poetry.dependencies]
+python = "^3.10"
+pandas = ">=2.3.3,<3"
+aiofiles = "*"
+# https://github.com/atviriduomenys/spinta/issues/675
+# 0.x -> 1.x upgrade changed their implementation API, so we can no longer support 0.x.
+authlib = ">=1.0.0"
+nh3 = "*"
+markdown = "*"
+jinja2 = "*"
+jsonpatch = "*"
+lark-parser = "*"
+msgpack = "*"
+multipledispatch = "*"
+python-multipart = "*"
+pytz = "*"
+requests = "*"
+"ruamel.yaml" = "*"
+# FIXME: https://setuptools.readthedocs.io/en/latest/history.html#v58-0-0
+#        simpleeval package uses use_2to3 thus it is not compatbile with
+#        setuptools>=58
+setuptools = "*"
+setuptools-scm = "*"
+# https://github.com/encode/starlette/security/advisories/GHSA-f96h-pmfr-66vw
+starlette = ">=0.52.1,<1.0.0"
+pydantic = "*"
+toposort = "*"
+tqdm = "*"
+ujson = "*"
+unidecode = "*"
+sqlparse = "*"
+pprintpp = "*"
+rdflib = "*"
+numpy = "*"
+
+# CLI tool
+click = "*"
+typer = { version = "*", extras = ["all"] }
+
+# API server dependencies
+gunicorn = { version = "*", optional = true }
+uvicorn = { version = "*", optional = true }
+
+# PostgreSQL backend dependencies
+alembic = { version = "~1.11.0", optional = true }
+asyncpg = "*"
+psycopg2-binary = "*"
+# FIXME: https://github.com/python-poetry/poetry/issues/4402
+# sqlalchemy = "~1.4"
+sqlalchemy = "~1.4"
+# https://github.com/kvesteri/sqlalchemy-utils/issues/472
+# https://github.com/kvesteri/sqlalchemy-utils/issues/791
+sqlalchemy-utils = "<=0.41.2"
+
+# Mongo backend dependendencies
+# https://github.com/atviriduomenys/spinta/issues/806
+pymongo = "<=4.8.0"
+
+# Excel dependencies
+xlrd = "*"
+
+# XML format dependencies
+lxml = "*"
+
+# PII (Person Identifiable Information) dependencies
+phonenumbers = "*"
+
+# Other dependencies
+XlsxWriter = "*"
+openpyxl = "*"
+GeoAlchemy2 = "*"
+Shapely = "*"
+aiohttp = "*"
+fsspec = "*"
+dask = { version = "*", extras = ["dataframe"] }
+psutil = "*"
+tabulate = "*"
+pyproj = [{version = ">=3.7.0", python = ">=3.13"}, {version = "3.6.1", python = "<3.13"}]
+xmltodict = "*"
+# Error when trying to install >=3.50
+sqlean-py = "<=3.49.1"
+cachetools = "*"
+docutils = "^0.21.2"
+zeep = "^4.3.1"
+redis = "^6.4.0"
+cbor2 = "^5.9.0"
+
+[tool.poetry.extras]
+http = ["gunicorn", "uvicorn"]
+migrations = ["alembic"]
+all = ["gunicorn", "uvicorn", "alembic"]
+
+
+[tool.poetry.group.dev.dependencies]
+ipdb = "*"
+ipython = "*"
+pp-ez = "*"
+pytest = "~7"
+pytest-asyncio = "*"
+pytest-cov = "*"
+pytest-mock = "*"
+responses = "*"
+snoop = "*"
+python-dotenv = "*"
+docutils = "*"
+requests-mock = "^1.11.0"
+ruff = "^0.12.10"
+
+# Starlette
+httpx = "*"
+
+# Docs
+sphinx = "*"
+sphinx-autobuild = "*"
+sphinxcontrib-httpdomain = "*"
+memory-profiler = "*"
+mypy = "*"
+cssselect = "*"
+objprint = "*"
+sphinx-rtd-theme = "*"
+sqlalchemy-stubs = "*"
+types-requests = "^2.32.4.20250809"
+fakeredis = "^2.32.0"
+
+
+[build-system]
+requires = ["poetry_core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+# https://mypy.readthedocs.io/en/stable/config_file.html#using-a-pyproject-toml-file
+[tool.mypy]
+disallow_any_unimported = true
+show_error_codes = true
+check_untyped_defs = true
+plugins = "sqlmypy"
+
+
+[tool.ruff]
+line-length = 120
+
+
+[tool.ruff.format]
+quote-style = "double"
+
+
+[tool.ruff.lint]
+# I (isort) checks import sorting and grouping.
+extend-select = ["I"]
+ignore = ["F811"]

spinta-0.2.dev27/spinta/accesslog/__init__.py

@@ -0,0 +1,204 @@
+import datetime
+import os
+import time
+from typing import Any, AsyncIterator, Dict, Iterable, Iterator, List, TypeVar, overload
+
+import psutil
+from starlette.requests import Request
+
+from spinta import commands
+from spinta.auth import Token, get_default_auth_client_id
+from spinta.components import Config, Context, UrlParams
+
+
+class AccessLog:
+    client: str = None
+    method: str = None
+    reason: str = None
+    url: str = None
+    buffer_size: int = 100
+    format: str = None  # response format
+    content_type: str = None  # request content-type header
+    agent: str = None  # request user-agent header
+    txn: str = None  # request transaction id
+    start: float = None  # request start time in seconds
+    memory: int = None  # memory used in bytes at the start of request
+    scopes: List[str] = None  # list of scopes
+    token: str = None  # token used for request
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        return None
+
+    def log(self, message: Dict[str, Any]) -> None:
+        raise NotImplementedError
+
+    def request(
+        self,
+        *,
+        txn: str,
+        ns: str = None,
+        model: str = None,
+        prop: str = None,
+        action: str = None,
+        method: str = None,
+        reason: str = None,
+        id_: str = None,
+        rev: str = None,
+        scopes: List[str] = None,
+    ):
+        self.txn = txn
+        self.start = self._get_time()
+        self.memory = self._get_memory()
+        message = {
+            "time": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+            "pid": os.getpid(),
+            "type": "request",
+            "method": method or self.method,
+            "action": action,
+            "ns": ns,
+            "model": model,
+            "prop": prop,
+            "id": id_,
+            "rev": rev,
+            "txn": txn,
+            "rctype": self.content_type,
+            "format": self.format,
+            "url": self.url,
+            "client": self.client,
+            "token": self.token,
+            "reason": reason or self.reason,
+            "agent": self.agent,
+        }
+        message = {k: v for k, v in message.items() if v is not None}
+        self.log(message)
+
+    def response(
+        self,
+        *,
+        objects: int,
+    ):
+        delta = self._get_time() - self.start
+        memory = self._get_memory() - self.memory
+        message = {
+            "time": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+            "type": "response",
+            "delta": delta,
+            "memory": memory,
+            "objects": objects,
+            "txn": self.txn,
+        }
+
+        self.log(message)
+
+    def auth(
+        self,
+        *,
+        reason: str = None,
+    ):
+        message = {
+            "time": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+            "type": "auth",
+            "client": self.client,
+            "token": self.token,
+            "scope": self.scopes,
+            "reason": reason or self.reason,
+            "method": self.method,
+            "rctype": self.content_type,
+            "format": self.format,
+            "url": self.url,
+            "agent": self.agent,
+        }
+        message = {k: v for k, v in message.items() if v is not None}
+        self.log(message)
+
+    def _get_time(self) -> float:
+        "Get current time in seconds"
+        return time.monotonic()
+
+    def _get_memory(self) -> float:
+        "Get currently used memory in bytes"
+        return psutil.Process().memory_info().rss
+
+
+@overload
+@commands.load.register(Context, AccessLog, Config)
+def load(context: Context, accesslog: AccessLog, config: Config):
+    accesslog.buffer_size = config.rc.get(
+        "accesslog",
+        "buffer_size",
+        required=True,
+    )
+
+
+@overload
+@commands.load.register(Context, AccessLog, Token)
+def load(context: Context, accesslog: AccessLog, token: Token):  # noqa
+    accesslog.client = token.get_sub()
+
+    client_id = token.get_client_id()
+    config = context.get("config")
+    if client_id != get_default_auth_client_id(context):
+        if config.scope_log:
+            accesslog.scopes = token.get_scope()
+        accesslog.token = token.get_jti()
+
+
+@overload
+@commands.load.register(Context, AccessLog, Request)
+def load(context: Context, accesslog: AccessLog, request: Request):  # noqa
+    accesslog.method = request.method
+    accesslog.url = str(request.url)
+    accesslog.content_type = request.headers.get("content-type")
+    accesslog.agent = request.headers.get("user-agent")
+
+
+@overload
+@commands.load.register(Context, AccessLog, UrlParams)
+def load(context: Context, accesslog: AccessLog, params: UrlParams):  # noqa
+    accesslog.format = params.format if params.format else "json"
+
+
+def create_accesslog(
+    context: Context,
+    *,
+    method: str = None,
+    loaders: List[Any],
+) -> AccessLog:
+    config: Config = context.get("config")
+    # XXX: Probably we should clone store.accesslog here, instead of creating
+    #      completely new AccessLog instance.
+    accesslog: AccessLog = config.AccessLog()
+    accesslog.method = method
+    for loader in loaders:
+        commands.load(context, accesslog, loader)
+    return accesslog
+
+
+TRow = TypeVar("TRow")
+
+
+def log_response(
+    context: Context,
+    rows: Iterable[TRow],
+) -> Iterator[TRow]:
+    objects = 0
+    for row in rows:
+        objects += 1
+        yield row
+    accesslog: AccessLog = context.get("accesslog")
+    accesslog.response(objects=objects)
+
+
+async def log_async_response(
+    context: Context,
+    rows: AsyncIterator[TRow],
+) -> AsyncIterator[TRow]:
+    objects = 0
+    async for row in rows:
+        objects += 1
+        yield row
+    accesslog: AccessLog = context.get("accesslog")
+    accesslog.response(objects=objects)