flatdata-py 0.4.11__tar.gz → 0.4.12__tar.gz

{flatdata_py-0.4.11 → flatdata_py-0.4.12}/.gitignore

@@ -4,10 +4,6 @@ build
 venv*
 **/.vscode/**
 **/.idea/**
-**/*_generated.go
-**/coverage.out
-**/flatdata-fuzz.zip
-**/corpus/**
 **/dist/**
 *.egg-info
 .DS_Store

{flatdata_py-0.4.11 → flatdata_py-0.4.12}/PKG-INFO

@@ -1,13 +1,14 @@
 Metadata-Version: 2.4
 Name: flatdata-py
-Version: 0.4.11
+Version: 0.4.12
 Summary: Python 3 implementation of Flatdata
 Project-URL: Homepage, https://github.com/heremaps/flatdata
 Author: Flatdata Developers
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Requires-Dist: flatdata-generator==0.4.11
+Requires-Python: >=3.8
+Requires-Dist: flatdata-generator==0.4.12
 Requires-Dist: numpy
 Requires-Dist: pandas
 Provides-Extra: inspector

{flatdata_py-0.4.11 → flatdata_py-0.4.12}/flatdata/lib/archive.py

@@ -3,17 +3,27 @@
  See the LICENSE file in the root of this project for license details.
 '''
 
-from collections import namedtuple
+from __future__ import annotations
+
+from typing import Any, NamedTuple, TYPE_CHECKING
 
 import pandas as pd
 
 from .errors import MissingResourceError, SchemaMismatchError
 
-ResourceSignature = namedtuple("ResourceSignature",
-                               ["container", "initializer", "schema", "is_optional", "doc"])
+if TYPE_CHECKING:
+    from .resources import ReadStorage, ResourceBase
+
+
+class ResourceSignature(NamedTuple):
+    container: type[ResourceBase] | type[Archive]
+    initializer: Any
+    schema: str
+    is_optional: bool
+    doc: str
 
-def _is_archive_signature(resource_signature):
-    return resource_signature.container == Archive
+def _is_archive_signature(resource_signature: ResourceSignature) -> bool:
+    return bool(resource_signature.container == Archive)
 
 _SCHEMA_EXT = ".schema"
 
@@ -23,35 +33,38 @@ class Archive:
     Archive class. Entry point to Flatdata.
     Provides access to flatdata resources and verifies archive/resource schemas on opening.
     """
+    _NAME: str
+    _SCHEMA: str
+    _RESOURCES: dict[str, ResourceSignature]
 
-    def __init__(self, resource_storage):
+    def __init__(self, resource_storage: ReadStorage) -> None:
         """
         Opens archive from a given resource storage.
         :raises flatdata.errors.CorruptArchiveError
         :raises flatdata.errors.SchemaMismatchError
         :param resource_storage: Resource storage to use.
         """
-        self._resource_storage = resource_storage
-        self._loaded_resources = {}
+        self._resource_storage: ReadStorage = resource_storage
+        self._loaded_resources: dict[str, Any] = {}
 
         # Preload resources and check their schemas
         for name, _ in sorted(list(self._RESOURCES.items())):
             self.__getattr__(name)
 
-    def __getattr__(self, name):
+    def __getattr__(self, name: str) -> Any:
         if name not in self._RESOURCES:
             raise AttributeError("Resource %s not defined in archive." % name)
         if name not in self._loaded_resources:
             self._loaded_resources[name] = self._open_resource(name)
         return self._loaded_resources[name]
 
-    def __dir__(self):
+    def __dir__(self) -> list[str]:
         return list(self._RESOURCES.keys()) + ['schema']
 
-    def __repr__(self):
-        return self.to_data_frame().__repr__()
+    def __repr__(self) -> str:
+        return repr(self.to_data_frame())
 
-    def to_data_frame(self):
+    def to_data_frame(self) -> pd.DataFrame:
         result = []
         for name, signature in self._RESOURCES.items():
             resource = self.__getattr__(name)
@@ -62,34 +75,34 @@ class Archive:
                             columns=["Name", "Type", "Optional", "SizeInBytes", "Size"])
 
     @classmethod
-    def name(cls):
+    def name(cls) -> str:
         return cls._NAME
 
     @classmethod
-    def schema(cls):
+    def schema(cls) -> str:
         return cls._SCHEMA
 
     @classmethod
-    def resource_schema(cls, resource):
-        return cls._RESOURCES[resource].schema
+    def resource_schema(cls, resource: str) -> str:
+        return str(cls._RESOURCES[resource].schema)
 
     @classmethod
-    def open(cls, storage, name, initializer, is_optional=False):
+    def open(cls, storage: ReadStorage, name: str, initializer: type[Archive], is_optional: bool = False) -> Archive | None:
         nested_storage = storage.get(name, is_optional)
         assert nested_storage is not None or is_optional
         if nested_storage is None:
             return None
         return initializer(nested_storage)
 
-    def size_in_bytes(self):
+    def size_in_bytes(self) -> int:
         return sum(resource_value.size_in_bytes() for resource_value in
                    (self.__getattr__(resource) for resource in self._RESOURCES.keys())
                    if resource_value)
 
-    def __len__(self):
+    def __len__(self) -> int:
         return len(self._RESOURCES)
 
-    def _schema_validated_resource_signature(self, name):
+    def _schema_validated_resource_signature(self, name: str) -> ResourceSignature | None:
         resource_signature = self._RESOURCES[name]
         # We check only schema for non-subarchives, since the subarchives schema is checked,
         # when it is initialized.
@@ -103,7 +116,7 @@ class Archive:
                 return None
         return resource_signature
 
-    def _open_resource(self, name):
+    def _open_resource(self, name: str) -> Any:
         resource_signature = self._schema_validated_resource_signature(name)
         if resource_signature:
             resource = resource_signature.container.open(storage=self._resource_storage,
@@ -116,7 +129,7 @@ class Archive:
         return None
 
     @staticmethod
-    def _check_non_subarchive_schema(name, resource_signature, storage):
+    def _check_non_subarchive_schema(name: str, resource_signature: ResourceSignature, storage: Any) -> None:
         actual_schema = bytes(storage).decode()
         if actual_schema != resource_signature.schema:
             raise SchemaMismatchError(

{flatdata_py-0.4.11 → flatdata_py-0.4.12}/flatdata/lib/archive_builder.py

@@ -3,8 +3,10 @@
  See the LICENSE file in the root of this project for license details.
 '''
 
-from collections import namedtuple
+from __future__ import annotations
+
 import os
+from typing import Any, NamedTuple, Protocol, TYPE_CHECKING
 
 from .errors import IndexWriterError, MissingFieldError, UnknownFieldError, \
     UnknownStructureError, UnknownResourceError, ResourceAlreadySetError
@@ -12,10 +14,24 @@ from .errors import IndexWriterError, MissingFieldError, UnknownFieldError, \
 from .resources import Instance, Vector, Multivector, RawData
 from .data_access import write_value
 
+if TYPE_CHECKING:
+    from .resource_storage import _Resource
+    from .structure import Structure
+
 _SCHEMA_EXT = ".schema"
 
-ResourceSignature = namedtuple("ResourceSignature",
-                               ["container", "initializer", "schema", "is_optional", "doc"])
+
+class ResourceSignature(NamedTuple):
+    container: type
+    initializer: Any
+    schema: str
+    is_optional: bool
+    doc: str
+
+
+class WriteStorage(Protocol):
+    def get(self, resource_name: str, is_subarchive: bool = False) -> Any: ...
+    def close(self) -> None: ...
 
 
 class IndexWriter:
@@ -23,7 +39,7 @@ class IndexWriter:
     IndexWriter class. Only applicable when multivector is present in archive schema.
     """
 
-    def __init__(self, name, size, resource_storage):
+    def __init__(self, name: str, size: int, resource_storage: WriteStorage) -> None:
         """
         Create IndexWriter class.
 
@@ -36,9 +52,9 @@ class IndexWriter:
 
         self._name = name
         self._index_size = size
-        self._fout = resource_storage.get(f'{self._name}_index', False)
+        self._fout: _Resource = resource_storage.get(f'{self._name}_index', False)
 
-    def add(self, index):
+    def add(self, index: int) -> None:
         """
         Convert index(number) to bytearray and add to in memory store
         """
@@ -46,7 +62,7 @@ class IndexWriter:
                                           byteorder="little", signed=False)
         self._fout.write(index_bytes)
 
-    def finish(self):
+    def finish(self) -> None:
         """
         Complete index resource by adding size and padding followed by writing to file
         """
@@ -60,30 +76,33 @@ class ArchiveBuilder:
     ArchiveBuilder class. Entry point to writing Flatdata.
     Provides methods to create flatdata archives.
     """
+    _NAME: str
+    _SCHEMA: str
+    _RESOURCES: dict[str, ResourceSignature]
 
-    def __init__(self, resource_storage, path=""):
+    def __init__(self, resource_storage: WriteStorage, path: str = "") -> None:
         """
         Opens archive from a given resource writer.
         :param resource_storage: storage manager to store and write to disc
         :param path: file path where archive is created
         """
         self._path = os.path.join(path, self._NAME)
-        self._resource_storage = resource_storage
+        self._resource_storage: WriteStorage = resource_storage
         self._write_archive_signature()
         self._write_archive_schema()
         self._resources_written = [f"{self._NAME}.archive"]
 
     @classmethod
-    def name(cls):
+    def name(cls) -> str:
         '''Returns archive name'''
         return cls._NAME
 
     @classmethod
-    def schema(cls):
+    def schema(cls) -> str:
         '''Returns archive schema'''
         return cls._SCHEMA
 
-    def _write_raw_data(self, name, data):
+    def _write_raw_data(self, name: str, data: bytes | bytearray) -> None:
         '''
         Helper function to write data
 
@@ -94,7 +113,7 @@ class ArchiveBuilder:
         storage.write(data)
         storage.close()
 
-    def _write_schema(self, name):
+    def _write_schema(self, name: str) -> None:
         '''
         Writes resource schema
 
@@ -103,29 +122,29 @@ class ArchiveBuilder:
         self._write_raw_data(f"{name}.schema", bytes(
             self._RESOURCES[name].schema, 'utf-8'))
 
-    def _write_archive_signature(self):
+    def _write_archive_signature(self) -> None:
         '''Writes archive's signature'''
         self._write_raw_data(f"{self._NAME}.archive", b'\x00' * 16)
 
-    def _write_archive_schema(self):
+    def _write_archive_schema(self) -> None:
         '''Writes archive schema'''
         self._write_raw_data(
             f"{self._NAME}.archive.schema", bytes(self._SCHEMA, 'utf-8'))
 
-    def _write_index_schema(self, resource_name, schema):
+    def _write_index_schema(self, resource_name: str, schema: str) -> None:
         self._write_raw_data(
             f"{resource_name}_index.schema", bytes(schema, 'utf-8'))
 
-    def subarchive(self, name):
+    def subarchive(self, name: str) -> 'ArchiveBuilder':
         """
         Returns an archive builder for the sub-archive `name`.
         :raises $name_not_subarchive_error
         :param name: name of the sub-archive
         """
-        NotImplemented
+        raise NotImplementedError(f"subarchive '{name}' is not implemented")
 
     @classmethod
-    def __validate_structure_fields(cls, name, struct, initializer):
+    def __validate_structure_fields(cls, name: str, struct: dict[str, Any], initializer: type[Structure]) -> None:
         '''
         Validates whether passed object has all required fields
 
@@ -142,7 +161,7 @@ class ArchiveBuilder:
             if key not in initializer._FIELD_KEYS:
                 raise UnknownFieldError(key, name)
 
-    def __set_instance(self, storage, name, value):
+    def __set_instance(self, storage: _Resource, name: str, value: dict[str, Any]) -> None:
         '''
         Creates and writes instance type resource
 
@@ -160,7 +179,7 @@ class ArchiveBuilder:
 
         storage.write(bout)
 
-    def __set_vector(self, storage, name, vector):
+    def __set_vector(self, storage: _Resource, name: str, vector: list[dict[str, Any]]) -> None:
         '''
         Creates and writes vector resource
 
@@ -179,7 +198,7 @@ class ArchiveBuilder:
                             field.is_signed, value[key])
             storage.write(bout)
 
-    def __set_multivector(self, storage, name, value):
+    def __set_multivector(self, storage: _Resource, name: str, value: list[list[dict[str, Any]]]) -> None:
         '''
         Creates and writes multivector resource
 
@@ -193,10 +212,10 @@ class ArchiveBuilder:
         for index, obj_type in enumerate(initializer_list[1:]):
             initializers[obj_type._NAME] = (index, obj_type)
 
-        def valid_structure_name(_obj):
+        def valid_structure_name(_obj: dict[str, Any]) -> bool:
             return _obj['name'] in [_initializer._NAME for _initializer in initializer_list[1:]]
 
-        def validate_fields(_obj):
+        def validate_fields(_obj: dict[str, Any]) -> None:
             matched_obj_list = [
                 _initializer for _initializer in initializer_list[1:] \
                     if _initializer._NAME == _obj['name']]
@@ -248,7 +267,7 @@ class ArchiveBuilder:
         self._resources_written.append(name)
         self._resources_written.append(f'{name}_index')
 
-    def set(self, name, value):
+    def set(self, name: str, value: Any) -> None:
         """
         Write a resource for this archive at once.
         Can only be done once. `set` and `start` can't be used for the same resource.
@@ -284,7 +303,7 @@ class ArchiveBuilder:
 
         self._resources_written.append(name)
 
-    def finish(self):
+    def finish(self) -> None:
         """
         Closes the storage manager
         """

{flatdata_py-0.4.11 → flatdata_py-0.4.12}/flatdata/lib/data_access.py

@@ -3,13 +3,22 @@
  See the LICENSE file in the root of this project for license details.
 '''
 
+from __future__ import annotations
+
+import mmap
+from collections.abc import Callable
+from typing import Union
+
 import numpy as np
+from numpy.typing import NDArray
+
+ReadableBuffer = Union[bytes, bytearray, memoryview, mmap.mmap]
 
 # Sign bits cache for the value reading.
 _SIGN_BITS = [0] + [(1 << (bits - 1)) for bits in range(1, 65)]
 
 
-def make_field_reader(offset_bits, num_bits, is_signed):
+def make_field_reader(offset_bits: int, num_bits: int, is_signed: bool) -> Callable[[ReadableBuffer, int], int]:
     """Build a specialized closure for reading a single field from a structure.
 
     Returns a function reader(data, pos_bytes) that reads the field value
@@ -26,7 +35,7 @@ def make_field_reader(offset_bits, num_bits, is_signed):
 
     if num_bits == 1:
         bit_mask = 1 << offset_extra
-        def reader(data, pos):
+        def reader(data: ReadableBuffer, pos: int) -> int:
             return int((data[pos + offset_bytes] & bit_mask) != 0)
         return reader
 
@@ -34,21 +43,21 @@ def make_field_reader(offset_bits, num_bits, is_signed):
         sign_bit = _SIGN_BITS[num_bits]
         sign_mask = sign_bit - 1
         if needs_extra:
-            def reader(data, pos):
+            def reader(data: ReadableBuffer, pos: int) -> int:
                 result = int.from_bytes(
                     data[pos + offset_bytes: pos + end_byte], byteorder="little")
                 result >>= offset_extra
                 result |= data[pos + end_byte] << extra_shift
                 result &= mask
-                return (result & sign_mask) - (result & sign_bit)
+                return int((result & sign_mask) - (result & sign_bit))
         elif offset_extra:
-            def reader(data, pos):
+            def reader(data: ReadableBuffer, pos: int) -> int:
                 result = (int.from_bytes(
                     data[pos + offset_bytes: pos + end_byte],
                     byteorder="little") >> offset_extra) & mask
                 return (result & sign_mask) - (result & sign_bit)
         else:
-            def reader(data, pos):
+            def reader(data: ReadableBuffer, pos: int) -> int:
                 result = int.from_bytes(
                     data[pos + offset_bytes: pos + end_byte],
                     byteorder="little") & mask
@@ -57,26 +66,26 @@ def make_field_reader(offset_bits, num_bits, is_signed):
 
     # Unsigned paths
     if needs_extra:
-        def reader(data, pos):
+        def reader(data: ReadableBuffer, pos: int) -> int:
             result = int.from_bytes(
                 data[pos + offset_bytes: pos + end_byte], byteorder="little")
             result >>= offset_extra
             result |= data[pos + end_byte] << extra_shift
-            return result & mask
+            return int(result & mask)
     elif offset_extra:
-        def reader(data, pos):
+        def reader(data: ReadableBuffer, pos: int) -> int:
             return (int.from_bytes(
                 data[pos + offset_bytes: pos + end_byte],
                 byteorder="little") >> offset_extra) & mask
     else:
-        def reader(data, pos):
+        def reader(data: ReadableBuffer, pos: int) -> int:
             return int.from_bytes(
                 data[pos + offset_bytes: pos + end_byte],
                 byteorder="little") & mask
     return reader
 
 
-def read_field_vectorized(raw_bytes_2d, field_offset_bits, field_width_bits, is_signed):
+def read_field_vectorized(raw_bytes_2d: NDArray[np.uint8], field_offset_bits: int, field_width_bits: int, is_signed: bool) -> NDArray[np.uint64] | NDArray[np.int64]:
     """Read a bit-packed field from all elements at once, returning a numpy array.
 
     :param raw_bytes_2d: numpy uint8 array shaped (num_elements, struct_size_bytes)
@@ -117,12 +126,12 @@ def read_field_vectorized(raw_bytes_2d, field_offset_bits, field_width_bits, is_
         sign_bit = np.uint64(1 << (field_width_bits - 1))
         offset = -(1 << field_width_bits)
         signed = result.astype(np.int64) + np.int64(offset)
-        result = np.where(result & sign_bit, signed, result.astype(np.int64))
+        result = np.where(result & sign_bit, signed, result.astype(np.int64))  # type: ignore[assignment, unused-ignore]
 
     return result
 
 
-def read_value(data, offset_bits, num_bits, is_signed):
+def read_value(data: ReadableBuffer, offset_bits: int, num_bits: int, is_signed: bool) -> int:
     """Read a bit-packed value from data at the given bit offset.
 
     This is a convenience wrapper around :func:`make_field_reader` for one-off
@@ -133,7 +142,7 @@ def read_value(data, offset_bits, num_bits, is_signed):
     return reader(data, 0)
 
 
-def write_value(data, offset_bits, num_bits, is_signed, value):
+def write_value(data: bytearray, offset_bits: int, num_bits: int, is_signed: bool, value: int) -> None:
     assert num_bits <= 64, f'Number of bits to write is greater than 64'
 
     offset_bytes, offset_extra_bits = divmod(offset_bits, 8)

{flatdata_py-0.4.11 → flatdata_py-0.4.12}/flatdata/lib/errors.py

@@ -3,6 +3,8 @@
  See the LICENSE file in the root of this project for license details.
 """
 
+from __future__ import annotations
+
 import difflib
 
 
@@ -11,7 +13,7 @@ class SchemaMismatchError(RuntimeError):
     Schema mismatch: archive does not match software expectations.
     """
 
-    def __init__(self, name, expected_schema, actual_schema):
+    def __init__(self, name: str, expected_schema: list[str], actual_schema: list[str]) -> None:
         diff = '\n'.join([l for l in difflib.unified_diff(expected_schema, actual_schema)])
         message = "Schema mismatch for resource {name}. Expected: \n{expected}\n\nActual:{actual}\n\nDiff:{diff}"
         RuntimeError.__init__(self,
@@ -36,7 +38,7 @@ class MissingResourceError(KeyError, CorruptArchiveError):
     """
     Resource or schema is missing.
     """
-    def __init__(self, key):
+    def __init__(self, key: str) -> None:
         super().__init__("Resource {key} not found".format(key=key))
 
 
@@ -50,7 +52,7 @@ class MissingFieldError(RuntimeError):
     """
     Fields missing in provided dictionary object
     """
-    def __init__(self, key, name):
+    def __init__(self, key: str, name: str) -> None:
         super().__init__(f'Missing "{key}" is required for "{name}"')
 
 
@@ -58,21 +60,21 @@ class UnknownFieldError(RuntimeError):
     """
     Field provided is not present in resource schema
     """
-    def __init__(self, key, name):
+    def __init__(self, key: str, name: str) -> None:
         super().__init__(f'Field "{key}" is not specified for "{name}"')
 
 class FileExistsError(RuntimeError):
     """
     Provided file name is already present.
     """
-    def __init__(self, key):
+    def __init__(self, key: str) -> None:
         super().__init__(f'File "{key}" exists already')
 
 class DirExistsError(RuntimeError):
     """
     Directory with given path is already present
     """
-    def __init__(self, path):
+    def __init__(self, path: str) -> None:
         super().__init__(f'Directory "{path}" exists already')
 
 class UnknownStructureError(RuntimeError):
@@ -80,47 +82,47 @@ class UnknownStructureError(RuntimeError):
     Provided structure/dictionary is not part of any initializer defined
     in multivector schema
     """
-    def __init__(self, name):
+    def __init__(self, name: str) -> None:
         super().__init__(f'"{name}" structure is not part of the multivector')
 
 class IndexWriterError(RuntimeError):
     """
     Error while creating instance of IndexWriter needed for multivector
     """
-    def __init__(self, error_str="Error initializing IndexWritter Class"):
+    def __init__(self, error_str: str = "Error initializing IndexWritter Class") -> None:
         super().__init__(f'{error_str}')
 
 class ArchivePathNotProvidedError(RuntimeError):
     """
     Path where archive will be created is missing
     """
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__("File path is not provided")
 
 class MissingResourceName(RuntimeError):
     """
     Resource name is not provided
     """
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__("Resource name is not provided")
 
 class FileNameNotProvided(RuntimeError):
     """
     File name is not provided
     """
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__("File name is not provided")
 
 class ResourceAlreadySetError(RuntimeError):
     """
     Provided resource name is already set for the archive
     """
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__("Resource is already set")
 
 class UnknownResourceError(RuntimeError):
     """
     Provided resource name is not in archive schema
     """
-    def __init__(self, name):
+    def __init__(self, name: str) -> None:
         super().__init__(f"Resource {name} is not part of provided schema")

{flatdata_py-0.4.11 → flatdata_py-0.4.12}/flatdata/lib/file_resource_storage.py

@@ -3,6 +3,8 @@
  See the LICENSE file in the root of this project for license details.
 '''
 
+from __future__ import annotations
+
 import mmap
 import os
 
@@ -15,7 +17,7 @@ class FileResourceStorage:
     """
 
     @staticmethod
-    def memory_map(filename):
+    def memory_map(filename: str) -> mmap.mmap:
         """
         Memory maps given file. Introduced to be able to swap mmap implementations.
         :param filename:
@@ -24,10 +26,10 @@ class FileResourceStorage:
         opened_file = open(filename, 'r')
         return mmap.mmap(opened_file.fileno(), 0, access=mmap.ACCESS_READ)
 
-    def __init__(self, path):
-        self.path = path
+    def __init__(self, path: str) -> None:
+        self.path: str = path
 
-    def get(self, key, is_optional=False):
+    def get(self, key: str, is_optional: bool = False) -> mmap.mmap | 'FileResourceStorage' | None:
         filename = os.path.join(self.path, key)
         if not os.path.exists(filename):
             if not is_optional:
@@ -40,5 +42,5 @@ class FileResourceStorage:
 
         return FileResourceStorage(filename)
 
-    def ls(self):
+    def ls(self) -> list[str]:
         return os.listdir(self.path)