vtlengine 1.0.4__py3-none-any.whl → 1.1rc1__py3-none-any.whl

vtlengine/API/_InternalApi.py

@@ -5,8 +5,8 @@ from typing import Any, Dict, List, Optional, Union
 
 import jsonschema
 import pandas as pd
-from s3fs import S3FileSystem  # type: ignore[import-untyped]
 
+from vtlengine.__extras_check import __check_s3_extra
 from vtlengine.AST import PersistentAssignment, Start
 from vtlengine.DataTypes import SCALAR_TYPES
 from vtlengine.Exceptions import InputValidationException, check_key
@@ -105,38 +105,16 @@ def _load_single_datapoint(datapoint: Union[str, Path]) -> Dict[str, Any]:
         raise Exception("Invalid datapoint. Input must be a Path or an S3 URI")
     if isinstance(datapoint, str):
         if "s3://" in datapoint:
-            # Handling S3 URI
-            s3fs_obj = S3FileSystem()
-
-            # Check if the S3 URI is valid
-            if not s3fs_obj.exists(datapoint):
-                raise Exception(
-                    f"Invalid datapoint. S3 URI does not exist or it is not accessible: {datapoint}"
-                )
-
-            # Check if the S3 URI is a directory
-            if s3fs_obj.isdir(datapoint):
-                datapoints: Dict[str, Any] = {}
-                for f in s3fs_obj.ls(datapoint):
-                    if f.endswith(".csv"):
-                        dataset_name = f.split("/")[-1].removesuffix(".csv")
-                        dict_data = {dataset_name: f"s3://{f}"}
-                        datapoints = {**datapoints, **dict_data}
-                return datapoints
-
-            # Check if the S3 URI is a csv file
-            if s3fs_obj.isfile(datapoint) and not datapoint.endswith(".csv"):
-                raise Exception(f"Invalid datapoint. S3 URI must refer to a csv file: {datapoint}")
+            __check_s3_extra()
             dataset_name = datapoint.split("/")[-1].removesuffix(".csv")
             dict_data = {dataset_name: datapoint}
             return dict_data
-
         try:
             datapoint = Path(datapoint)
         except Exception:
             raise Exception("Invalid datapoint. Input must refer to a Path or an S3 URI")
     if datapoint.is_dir():
-        datapoints = {}
+        datapoints: Dict[str, Any] = {}
         for f in datapoint.iterdir():
             if f.suffix != ".csv":
                 continue
@@ -400,25 +378,18 @@ def _check_output_folder(output_folder: Union[str, Path]) -> None:
     """
     if isinstance(output_folder, str):
         if "s3://" in output_folder:
-            s3fs_obj = S3FileSystem()
-            # Check if the S3 URI is valid
-            if not s3fs_obj.exists(output_folder):
-                try:
-                    s3fs_obj.mkdir(output_folder)
-                except Exception:
-                    raise Exception(
-                        f"Invalid output folder. S3 URI is invalid or "
-                        f"it is not accessible: {output_folder}"
-                    )
+            __check_s3_extra()
+            if not output_folder.endswith("/"):
+                raise ValueError("Output folder must be a Path or S3 URI to a directory")
             return
         try:
             output_folder = Path(output_folder)
         except Exception:
-            raise Exception("Output folder must be a Path or S3 URI to a directory")
+            raise ValueError("Output folder must be a Path or S3 URI to a directory")
 
     if not isinstance(output_folder, Path):
-        raise Exception("Output folder must be a Path or S3 URI to a directory")
+        raise ValueError("Output folder must be a Path or S3 URI to a directory")
     if not output_folder.exists():
         if output_folder.suffix != "":
-            raise Exception("Output folder must be a Path or S3 URI to a directory")
+            raise ValueError("Output folder must be a Path or S3 URI to a directory")
         os.mkdir(output_folder)

vtlengine/__extras_check.py

@@ -0,0 +1,17 @@
+import importlib.util
+
+EXTRAS_DOCS = "https://docs.vtlengine.meaningfuldata.eu/#installation"
+ERROR_MESSAGE = (
+    "The '{extra_name}' extra is required to run {extra_desc}. "
+    "Please install it using 'pip install vtlengine[{extra_name}]' or "
+    "install all extras with 'pip install vtlengine[all]'. "
+    f"Check the documentation at: {EXTRAS_DOCS}"
+)
+
+
+def __check_s3_extra() -> None:
+    package_loc = importlib.util.find_spec("s3fs")
+    if package_loc is None:
+        raise ImportError(
+            ERROR_MESSAGE.format(extra_name="s3", extra_desc="over csv files using S3 URIs")
+        ) from None

vtlengine/files/output/__init__.py

@@ -3,6 +3,7 @@ from typing import Optional, Union
 
 import pandas as pd
 
+from vtlengine.__extras_check import __check_s3_extra
 from vtlengine.files.output._time_period_representation import (
     TimePeriodRepresentation,
     format_time_period_external_representation,
@@ -20,6 +21,7 @@ def save_datapoints(
     if time_period_representation is not None:
         format_time_period_external_representation(dataset, time_period_representation)
     if isinstance(output_path, str):
+        __check_s3_extra()
         if output_path.endswith("/"):
             s3_file_output = output_path + f"{dataset.name}.csv"
         else:

vtlengine/files/parser/__init__.py

@@ -109,7 +109,7 @@ def _sanitize_pandas_columns(
     return data
 
 
-def _pandas_load_csv(components: Dict[str, Component], csv_path: Path) -> pd.DataFrame:
+def _pandas_load_csv(components: Dict[str, Component], csv_path: Union[str, Path]) -> pd.DataFrame:
     obj_dtypes = {comp_name: np.object_ for comp_name, comp in components.items()}
 
     try:
@@ -121,31 +121,14 @@ def _pandas_load_csv(components: Dict[str, Component], csv_path: Path) -> pd.Dat
             na_values=[""],
         )
     except UnicodeDecodeError:
-        raise InputValidationException(code="0-1-2-5", file=csv_path.name)
+        if isinstance(csv_path, Path):
+            raise InputValidationException(code="0-1-2-5", file=csv_path.name)
+        else:
+            raise InputValidationException(code="0-1-2-5", file=csv_path)
 
     return _sanitize_pandas_columns(components, csv_path, data)
 
 
-def _pandas_load_s3_csv(components: Dict[str, Component], csv_path: str) -> pd.DataFrame:
-    obj_dtypes = {comp_name: np.object_ for comp_name, comp in components.items()}
-
-    # start = time()
-    try:
-        data = pd.read_csv(
-            csv_path,
-            dtype=obj_dtypes,
-            engine="c",
-            keep_default_na=False,
-            na_values=[""],
-        )
-
-    except UnicodeDecodeError:
-        raise InputValidationException(code="0-1-2-5", file=csv_path)
-    except Exception as e:
-        raise InputValidationException(f"ERROR: {str(e)}, review file {str(csv_path)}")
-    return _sanitize_pandas_columns(components, csv_path, data)
-
-
 def _parse_boolean(value: str) -> bool:
     if isinstance(value, bool):
         return value
@@ -240,10 +223,9 @@ def load_datapoints(
 ) -> pd.DataFrame:
     if csv_path is None or (isinstance(csv_path, Path) and not csv_path.exists()):
         return pd.DataFrame(columns=list(components.keys()))
-    elif isinstance(csv_path, str):
-        data = _pandas_load_s3_csv(components, csv_path)
-    elif isinstance(csv_path, Path):
-        _validate_csv_path(components, csv_path)
+    elif isinstance(csv_path, (str, Path)):
+        if isinstance(csv_path, Path):
+            _validate_csv_path(components, csv_path)
         data = _pandas_load_csv(components, csv_path)
     else:
         raise Exception("Invalid csv_path type")

{vtlengine-1.0.4.dist-info → vtlengine-1.1rc1.dist-info}/METADATA

@@ -1,32 +1,31 @@
 Metadata-Version: 2.3
 Name: vtlengine
-Version: 1.0.4
+Version: 1.1rc1
 Summary: Run and Validate VTL Scripts
 License: AGPL-3.0
 Keywords: vtl,sdmx,vtlengine,Validation and Transformation Language
 Author: MeaningfulData
 Author-email: info@meaningfuldata.eu
-Requires-Python: >=3.9,<4.0
+Maintainer: Francisco Javier Hernandez del Caño
+Maintainer-email: javier.hernandez@meaningfuldata.eu
+Requires-Python: >=3.9,<4
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Information Technology
 Classifier: Intended Audience :: Science/Research
-Classifier: License :: OSI Approved :: GNU Affero General Public License v3
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
 Classifier: Typing :: Typed
-Requires-Dist: antlr4-python3-runtime (==4.9.2)
-Requires-Dist: duckdb (>=1.1,<2.0)
-Requires-Dist: jsonschema (>=4.23.0,<5.0.0)
-Requires-Dist: networkx (>=2.8.8,<3.0.0)
-Requires-Dist: numexpr (>=2.9.0,<3.0.0)
-Requires-Dist: pandas (>=2.1.4,<3.0.0)
-Requires-Dist: s3fs (>=2025.2.0,<2026.0.0)
-Requires-Dist: sqlglot (>=22.2.0,<23.0.0)
+Provides-Extra: all
+Provides-Extra: s3
+Requires-Dist: antlr4-python3-runtime (>=4.9.2,<4.10)
+Requires-Dist: duckdb (>=1.1,<1.2)
+Requires-Dist: fsspec (>=2022.11.0,<2023.0) ; extra == "all"
+Requires-Dist: fsspec (>=2022.11.0,<2023.0) ; extra == "s3"
+Requires-Dist: jsonschema (>=3.2.0,<5.0)
+Requires-Dist: networkx (>=2.8,<3.0)
+Requires-Dist: pandas (>=2.1,<3.0)
+Requires-Dist: s3fs (>=2022.11.0,<2023.0) ; extra == "all"
+Requires-Dist: s3fs (>=2022.11.0,<2023.0) ; extra == "s3"
+Requires-Dist: sqlglot (>=22.2.0,<23.0)
 Project-URL: Authors, https://github.com/Meaningful-Data/vtlengine/graphs/contributors
 Project-URL: Documentation, https://docs.vtlengine.meaningfuldata.eu
 Project-URL: IssueTracker, https://github.com/Meaningful-Data/vtlengine/issues

{vtlengine-1.0.4.dist-info → vtlengine-1.1rc1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-vtlengine/API/_InternalApi.py,sha256=E3WbH-AfFAp0Dgk7TAx6T5FZb0EzHgVEwtVxTjcfTlA,16058
+vtlengine/API/_InternalApi.py,sha256=fz_JSIrLdXd29Tbg7FI6uZQE1wdOTlJPSZ27tPXHjeM,14825
 vtlengine/API/__init__.py,sha256=2IDUvvSJdbkL5It8JulhmPCgkEnNbw52_VbbWm_aRp0,11061
 vtlengine/API/data/schema/json_schema_2.1.json,sha256=v3-C0Xnq8qScJSPAtLgb3rjKMrd3nz-bIxgZdTSEUiU,4336
 vtlengine/AST/ASTConstructor.py,sha256=DdE0B6CyPt1RYb3he6L0tL-KhZ1UyHRxQisGC1GuKx8,19692
@@ -45,14 +45,15 @@ vtlengine/Operators/Time.py,sha256=9f2kQ6iAoA4YPvlfphJ_uQjM-ZuqjSnOs312ttWMhgg,4
 vtlengine/Operators/Validation.py,sha256=ev3HyU7e1XbeAtUQ1y6zY3fzBwMqetDPhG3NNveAGOE,9988
 vtlengine/Operators/__init__.py,sha256=GN5eaAwmzfYKD7JJRIaRqdIJzflGc3UMvrOC9mlYNVo,37227
 vtlengine/Utils/__init__.py,sha256=ZobqGLc4rpMrsmniexTD4J-VokQt3qLrBGdFEDHHT1M,7571
+vtlengine/__extras_check.py,sha256=Wr-lxGZhXJZEacVV5cUkvKt7XM-mry0kYAe3VxNrVcY,614
 vtlengine/__init__.py,sha256=L9tGzRGQ8HMDS23sVWIbBvj41sXR89pf0ZMzEidIEMM,89
 vtlengine/files/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vtlengine/files/output/__init__.py,sha256=W0P5E4Gnikp2cVIVtxPZ4BQazfUW7MTlMNjq7ltg5I8,1169
+vtlengine/files/output/__init__.py,sha256=4tmf-p1Y1u5Ohrwt3clQA-FMGaijKI3HC_iwn3H9J8c,1250
 vtlengine/files/output/_time_period_representation.py,sha256=D5XCSXyEuX_aBzTvBV3sZxACcgwXz2Uu_YH3loMP8q0,1610
-vtlengine/files/parser/__init__.py,sha256=5emul7mwgAEq3A4_y6gyIl-PxoUV_oMxEtKlmkzNux4,9621
+vtlengine/files/parser/__init__.py,sha256=Kt1hFk6El0B2Fpi3sSC34x4r9BfG6V2pwDv67D0Z3xg,9057
 vtlengine/files/parser/_rfc_dialect.py,sha256=0T8GshGA5z9ZgYStH7zz2ZwtdiGkj7B8jXcxsPkXfjs,488
 vtlengine/files/parser/_time_checking.py,sha256=UAC_Pv-eQJKrhgTguWb--xfqMMs6quyMeiAkGBt_vgI,4725
-vtlengine-1.0.4.dist-info/LICENSE.md,sha256=2xqHuoHohba7gpcZZKtOICRjzeKsQANXG8WoV9V35KM,33893
-vtlengine-1.0.4.dist-info/METADATA,sha256=6Mbusz1Tl8IKHID-GiuA0NEDg2uMu1ZXNf_X1lzhbHo,8816
-vtlengine-1.0.4.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-vtlengine-1.0.4.dist-info/RECORD,,
+vtlengine-1.1rc1.dist-info/LICENSE.md,sha256=2xqHuoHohba7gpcZZKtOICRjzeKsQANXG8WoV9V35KM,33893
+vtlengine-1.1rc1.dist-info/METADATA,sha256=M6qsKt4xzOOOiQ7TdfDnqcSxLeqgqGouytXDAdRR3o0,8726
+vtlengine-1.1rc1.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+vtlengine-1.1rc1.dist-info/RECORD,,