vtlengine 1.0.4__tar.gz → 1.1rc1__tar.gz

{vtlengine-1.0.4 → vtlengine-1.1rc1}/PKG-INFO

@@ -1,32 +1,31 @@
 Metadata-Version: 2.3
 Name: vtlengine
-Version: 1.0.4
+Version: 1.1rc1
 Summary: Run and Validate VTL Scripts
 License: AGPL-3.0
 Keywords: vtl,sdmx,vtlengine,Validation and Transformation Language
 Author: MeaningfulData
 Author-email: info@meaningfuldata.eu
-Requires-Python: >=3.9,<4.0
+Maintainer: Francisco Javier Hernandez del Caño
+Maintainer-email: javier.hernandez@meaningfuldata.eu
+Requires-Python: >=3.9,<4
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Information Technology
 Classifier: Intended Audience :: Science/Research
-Classifier: License :: OSI Approved :: GNU Affero General Public License v3
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
 Classifier: Typing :: Typed
-Requires-Dist: antlr4-python3-runtime (==4.9.2)
-Requires-Dist: duckdb (>=1.1,<2.0)
-Requires-Dist: jsonschema (>=4.23.0,<5.0.0)
-Requires-Dist: networkx (>=2.8.8,<3.0.0)
-Requires-Dist: numexpr (>=2.9.0,<3.0.0)
-Requires-Dist: pandas (>=2.1.4,<3.0.0)
-Requires-Dist: s3fs (>=2025.2.0,<2026.0.0)
-Requires-Dist: sqlglot (>=22.2.0,<23.0.0)
+Provides-Extra: all
+Provides-Extra: s3
+Requires-Dist: antlr4-python3-runtime (>=4.9.2,<4.10)
+Requires-Dist: duckdb (>=1.1,<1.2)
+Requires-Dist: fsspec (>=2022.11.0,<2023.0) ; extra == "all"
+Requires-Dist: fsspec (>=2022.11.0,<2023.0) ; extra == "s3"
+Requires-Dist: jsonschema (>=3.2.0,<5.0)
+Requires-Dist: networkx (>=2.8,<3.0)
+Requires-Dist: pandas (>=2.1,<3.0)
+Requires-Dist: s3fs (>=2022.11.0,<2023.0) ; extra == "all"
+Requires-Dist: s3fs (>=2022.11.0,<2023.0) ; extra == "s3"
+Requires-Dist: sqlglot (>=22.2.0,<23.0)
 Project-URL: Authors, https://github.com/Meaningful-Data/vtlengine/graphs/contributors
 Project-URL: Documentation, https://docs.vtlengine.meaningfuldata.eu
 Project-URL: IssueTracker, https://github.com/Meaningful-Data/vtlengine/issues

{vtlengine-1.0.4 → vtlengine-1.1rc1}/pyproject.toml

@@ -1,11 +1,18 @@
-[tool.poetry]
+[project]
 name = "vtlengine"
-version = "1.0.4"
+version = "1.1rc1"
 description = "Run and Validate VTL Scripts"
-authors = ["MeaningfulData <info@meaningfuldata.eu>"]
 license = "AGPL-3.0"
 readme = "README.md"
-
+requires-python = ">=3.9,<4"
+authors = [
+    {name = "MeaningfulData", email = "info@meaningfuldata.eu"},
+]
+maintainers = [
+    {name = "Francisco Javier Hernandez del Caño", email = "javier.hernandez@meaningfuldata.eu"},
+    {name = "Alberto Hernandez del Caño", email = "alberto.hernandez@meaningfuldata.eu"},
+    {name = "Mateo de Lorenzo Argeles", email = "mateo.delorenzo@meaningfuldata.eu"}
+]
 classifiers = [
     "Development Status :: 5 - Production/Stable",
     "Intended Audience :: Developers",
@@ -13,43 +20,45 @@ classifiers = [
     "Intended Audience :: Science/Research",
     "Typing :: Typed"
 ]
-
 keywords = ['vtl', 'sdmx', 'vtlengine', 'Validation and Transformation Language']
 
-[tool.poetry.urls]
+dependencies = [
+    # PyPi dependencies
+    "duckdb>=1.1,<1.2",
+
+    # APT-supported dependencies
+    "jsonschema>=3.2.0,<5.0",
+    "sqlglot>=22.2.0,<23.0",
+    "antlr4-python3-runtime>=4.9.2,<4.10",
+    "pandas>=2.1,<3.0",
+    "networkx>=2.8,<3.0",
+]
+
+[project.optional-dependencies]
+s3 = ["s3fs>=2022.11.0,<2023.0", "fsspec>=2022.11.0,<2023.0"]
+all = ["s3fs>=2022.11.0,<2023.0", "fsspec>=2022.11.0,<2023.0"]
+
+[project.urls]
 Repository = 'https://github.com/Meaningful-Data/vtlengine'
 Documentation = 'https://docs.vtlengine.meaningfuldata.eu'
 MeaningfulData = 'https://www.meaningfuldata.eu/'
 IssueTracker = 'https://github.com/Meaningful-Data/vtlengine/issues'
 Authors = 'https://github.com/Meaningful-Data/vtlengine/graphs/contributors'
 
-[tool.poetry.dependencies]
-python = "^3.9"
-# PyPi dependencies
-duckdb = "^1.1"
-#numba = "^0.60.0"
-s3fs = "^2025.2.0"
-jsonschema = "^4.23.0"
-
-# APT dependencies
-antlr4-python3-runtime = "4.9.2"
-networkx = "^2.8.8"
-numexpr = "^2.9.0"
-pandas = "^2.1.4"
-sqlglot = "^22.2.0"
-
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.3"
 pytest-cov = "^6.0.0"
 line-profiler-pycharm = "^1.2.0"
-sphinx = "^7.4.7"
-sphinx-rtd-theme = "^3.0.2"
 mypy = "1.14.1"
 pandas-stubs = "^2.1.4.231227"
-toml = "^0.10.2"
 ruff = "^0.9.4"
 types-jsonschema = "4.23.0.20241208"
 
+[tool.poetry.group.docs.dependencies]
+sphinx = "^7.4.7"
+sphinx-rtd-theme = "^3.0.2"
+toml = "^0.10.2"
+
 [tool.ruff]
 line-length = 100
 lint.mccabe.max-complexity = 20

{vtlengine-1.0.4 → vtlengine-1.1rc1}/src/vtlengine/API/_InternalApi.py

@@ -5,8 +5,8 @@ from typing import Any, Dict, List, Optional, Union
 
 import jsonschema
 import pandas as pd
-from s3fs import S3FileSystem  # type: ignore[import-untyped]
 
+from vtlengine.__extras_check import __check_s3_extra
 from vtlengine.AST import PersistentAssignment, Start
 from vtlengine.DataTypes import SCALAR_TYPES
 from vtlengine.Exceptions import InputValidationException, check_key
@@ -105,38 +105,16 @@ def _load_single_datapoint(datapoint: Union[str, Path]) -> Dict[str, Any]:
         raise Exception("Invalid datapoint. Input must be a Path or an S3 URI")
     if isinstance(datapoint, str):
         if "s3://" in datapoint:
-            # Handling S3 URI
-            s3fs_obj = S3FileSystem()
-
-            # Check if the S3 URI is valid
-            if not s3fs_obj.exists(datapoint):
-                raise Exception(
-                    f"Invalid datapoint. S3 URI does not exist or it is not accessible: {datapoint}"
-                )
-
-            # Check if the S3 URI is a directory
-            if s3fs_obj.isdir(datapoint):
-                datapoints: Dict[str, Any] = {}
-                for f in s3fs_obj.ls(datapoint):
-                    if f.endswith(".csv"):
-                        dataset_name = f.split("/")[-1].removesuffix(".csv")
-                        dict_data = {dataset_name: f"s3://{f}"}
-                        datapoints = {**datapoints, **dict_data}
-                return datapoints
-
-            # Check if the S3 URI is a csv file
-            if s3fs_obj.isfile(datapoint) and not datapoint.endswith(".csv"):
-                raise Exception(f"Invalid datapoint. S3 URI must refer to a csv file: {datapoint}")
+            __check_s3_extra()
             dataset_name = datapoint.split("/")[-1].removesuffix(".csv")
             dict_data = {dataset_name: datapoint}
             return dict_data
-
         try:
             datapoint = Path(datapoint)
         except Exception:
             raise Exception("Invalid datapoint. Input must refer to a Path or an S3 URI")
     if datapoint.is_dir():
-        datapoints = {}
+        datapoints: Dict[str, Any] = {}
         for f in datapoint.iterdir():
             if f.suffix != ".csv":
                 continue
@@ -400,25 +378,18 @@ def _check_output_folder(output_folder: Union[str, Path]) -> None:
     """
     if isinstance(output_folder, str):
         if "s3://" in output_folder:
-            s3fs_obj = S3FileSystem()
-            # Check if the S3 URI is valid
-            if not s3fs_obj.exists(output_folder):
-                try:
-                    s3fs_obj.mkdir(output_folder)
-                except Exception:
-                    raise Exception(
-                        f"Invalid output folder. S3 URI is invalid or "
-                        f"it is not accessible: {output_folder}"
-                    )
+            __check_s3_extra()
+            if not output_folder.endswith("/"):
+                raise ValueError("Output folder must be a Path or S3 URI to a directory")
             return
         try:
             output_folder = Path(output_folder)
         except Exception:
-            raise Exception("Output folder must be a Path or S3 URI to a directory")
+            raise ValueError("Output folder must be a Path or S3 URI to a directory")
 
     if not isinstance(output_folder, Path):
-        raise Exception("Output folder must be a Path or S3 URI to a directory")
+        raise ValueError("Output folder must be a Path or S3 URI to a directory")
     if not output_folder.exists():
         if output_folder.suffix != "":
-            raise Exception("Output folder must be a Path or S3 URI to a directory")
+            raise ValueError("Output folder must be a Path or S3 URI to a directory")
         os.mkdir(output_folder)

vtlengine-1.1rc1/src/vtlengine/__extras_check.py

@@ -0,0 +1,17 @@
+import importlib.util
+
+EXTRAS_DOCS = "https://docs.vtlengine.meaningfuldata.eu/#installation"
+ERROR_MESSAGE = (
+    "The '{extra_name}' extra is required to run {extra_desc}. "
+    "Please install it using 'pip install vtlengine[{extra_name}]' or "
+    "install all extras with 'pip install vtlengine[all]'. "
+    f"Check the documentation at: {EXTRAS_DOCS}"
+)
+
+
+def __check_s3_extra() -> None:
+    package_loc = importlib.util.find_spec("s3fs")
+    if package_loc is None:
+        raise ImportError(
+            ERROR_MESSAGE.format(extra_name="s3", extra_desc="over csv files using S3 URIs")
+        ) from None

{vtlengine-1.0.4 → vtlengine-1.1rc1}/src/vtlengine/files/output/__init__.py

@@ -3,6 +3,7 @@ from typing import Optional, Union
 
 import pandas as pd
 
+from vtlengine.__extras_check import __check_s3_extra
 from vtlengine.files.output._time_period_representation import (
     TimePeriodRepresentation,
     format_time_period_external_representation,
@@ -20,6 +21,7 @@ def save_datapoints(
     if time_period_representation is not None:
         format_time_period_external_representation(dataset, time_period_representation)
     if isinstance(output_path, str):
+        __check_s3_extra()
         if output_path.endswith("/"):
             s3_file_output = output_path + f"{dataset.name}.csv"
         else:

{vtlengine-1.0.4 → vtlengine-1.1rc1}/src/vtlengine/files/parser/__init__.py

@@ -109,7 +109,7 @@ def _sanitize_pandas_columns(
     return data
 
 
-def _pandas_load_csv(components: Dict[str, Component], csv_path: Path) -> pd.DataFrame:
+def _pandas_load_csv(components: Dict[str, Component], csv_path: Union[str, Path]) -> pd.DataFrame:
     obj_dtypes = {comp_name: np.object_ for comp_name, comp in components.items()}
 
     try:
@@ -121,31 +121,14 @@ def _pandas_load_csv(components: Dict[str, Component], csv_path: Path) -> pd.Dat
             na_values=[""],
         )
     except UnicodeDecodeError:
-        raise InputValidationException(code="0-1-2-5", file=csv_path.name)
+        if isinstance(csv_path, Path):
+            raise InputValidationException(code="0-1-2-5", file=csv_path.name)
+        else:
+            raise InputValidationException(code="0-1-2-5", file=csv_path)
 
     return _sanitize_pandas_columns(components, csv_path, data)
 
 
-def _pandas_load_s3_csv(components: Dict[str, Component], csv_path: str) -> pd.DataFrame:
-    obj_dtypes = {comp_name: np.object_ for comp_name, comp in components.items()}
-
-    # start = time()
-    try:
-        data = pd.read_csv(
-            csv_path,
-            dtype=obj_dtypes,
-            engine="c",
-            keep_default_na=False,
-            na_values=[""],
-        )
-
-    except UnicodeDecodeError:
-        raise InputValidationException(code="0-1-2-5", file=csv_path)
-    except Exception as e:
-        raise InputValidationException(f"ERROR: {str(e)}, review file {str(csv_path)}")
-    return _sanitize_pandas_columns(components, csv_path, data)
-
-
 def _parse_boolean(value: str) -> bool:
     if isinstance(value, bool):
         return value
@@ -240,10 +223,9 @@ def load_datapoints(
 ) -> pd.DataFrame:
     if csv_path is None or (isinstance(csv_path, Path) and not csv_path.exists()):
         return pd.DataFrame(columns=list(components.keys()))
-    elif isinstance(csv_path, str):
-        data = _pandas_load_s3_csv(components, csv_path)
-    elif isinstance(csv_path, Path):
-        _validate_csv_path(components, csv_path)
+    elif isinstance(csv_path, (str, Path)):
+        if isinstance(csv_path, Path):
+            _validate_csv_path(components, csv_path)
         data = _pandas_load_csv(components, csv_path)
     else:
         raise Exception("Invalid csv_path type")