sinapsis-data-readers 0.1.14__tar.gz → 0.1.16__tar.gz

{sinapsis_data_readers-0.1.14/src/sinapsis_data_readers.egg-info → sinapsis_data_readers-0.1.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sinapsis-data-readers
-Version: 0.1.14
+Version: 0.1.16
 Summary: Templates to read data in different formats
 Author-email: SinapsisAI <dev@sinapsis.tech>
 Project-URL: Homepage, https://sinapsis.tech

{sinapsis_data_readers-0.1.14 → sinapsis_data_readers-0.1.16}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sinapsis-data-readers"
-version = "0.1.14"
+version = "0.1.16"
 description = "Templates to read data in different formats"
 authors = [{ name = "SinapsisAI", email = "dev@sinapsis.tech" }]
 

{sinapsis_data_readers-0.1.14 → sinapsis_data_readers-0.1.16}/src/sinapsis_data_readers/helpers/sktime_datasets_subset.py

@@ -1,12 +1,9 @@
 # -*- coding: utf-8 -*-
-""" Excluded sktime loaders"""
 from typing import Callable
 
 from sktime import datasets
 
-class_datasets = [
-"Airline",
-"Longley",
+class_datasets = ["Airline","Longley",
 "Lynx",
 "Macroeconomic",
 "ShampooSales",

{sinapsis_data_readers-0.1.14 → sinapsis_data_readers-0.1.16}/src/sinapsis_data_readers/templates/audio_readers/audio_reader_pydub.py

@@ -62,7 +62,7 @@ class AudioReaderPydub(_AudioBaseReader):
         """
 
         sample_rate_khz: int = 16
-        from_bytes: bool = True
+        from_bytes: bool = False
         audio_reader_format: Literal["wav", "raw", "pcm"] | None = None
 
     UIProperties = AudioReaderPydubUIProperties
@@ -84,7 +84,9 @@ class AudioReaderPydub(_AudioBaseReader):
             audio_segment = AudioSegment.from_file(io.BytesIO(audio_bytes))
 
         else:
-            audio_file_path: str = cast(str, self.attributes.audio_file_path)
+
+            audio_file_path = self.get_full_path()
+            #audio_file_path = os.path.join(self.attributes.root_dir, audio_file_path)
             if os.path.exists(audio_file_path):
                 audio_segment = AudioSegment.from_file(audio_file_path, format=self.attributes.audio_reader_format)
 

{sinapsis_data_readers-0.1.14 → sinapsis_data_readers-0.1.16}/src/sinapsis_data_readers/templates/audio_readers/audio_reader_soundfile.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 
 import os
-from typing import cast
 
 import soundfile as sf
 from sinapsis_core.data_containers.data_packet import AudioPacket, DataContainer
@@ -60,7 +59,7 @@ class AudioReaderSoundfile(_AudioBaseReader):
             AudioPacket|None: An AudioPacket containing the audio data and
             sample rate, or None if the file could not be read or was invalid.
         """
-        audio_path = cast(str, self.attributes.audio_file_path)
+        audio_path = self.get_full_path()
         if os.path.exists(audio_path):
             try:
                 audio_content, sample_rate = sf.read(audio_path)

{sinapsis_data_readers-0.1.14 → sinapsis_data_readers-0.1.16}/src/sinapsis_data_readers/templates/audio_readers/audio_reader_to_bytes.py

@@ -38,19 +38,19 @@ class AudioReaderToBytes(_AudioBaseReader):
             FileNotFoundError: If the specified audio file does not exist.
             IOError: If there is an error reading the audio file.
         """
-
+        full_path = self.get_full_path()
         try:
-            with open(self.attributes.audio_file_path, "rb") as audio_file:
+            with open(full_path, "rb") as audio_file:
                 audio_content = audio_file.read()
 
             audio_file.close()
             audio_packet = AudioPacket(
-                source=self.attributes.audio_file_path,
+                source=full_path,
                 content=audio_content,
             )
             return audio_packet
         except FileNotFoundError:
-            self.logger.error(f"Audio file not found: {self.attributes.audio_file_path}")
+            self.logger.error(f"Audio file not found: {full_path}")
             return None
         except IOError as e:
             self.logger.error(f"Error reading audio file: {e}")

{sinapsis_data_readers-0.1.14 → sinapsis_data_readers-0.1.16}/src/sinapsis_data_readers/templates/audio_readers/base_audio_reader.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 
 import abc
+import os
+from typing import cast
 from uuid import uuid4
 
 from sinapsis_core.data_containers.data_packet import AudioPacket, DataContainer
@@ -28,6 +30,8 @@ class _AudioBaseReader(_BaseDataReader):
             The source identifier, defaults to "streamlit".
         """
 
+
+        root_dir: str | None = None
         audio_file_path: str
         source: str = str(uuid4())
 
@@ -42,6 +46,10 @@ class _AudioBaseReader(_BaseDataReader):
         Returns:
             AudioPacket: The audio data wrapped in an AudioPacket.
         """
+    def get_full_path(self):
+        audio_file_path = cast(str, self.attributes.audio_file_path)
+        full_path = os.path.join(self.attributes.root_dir, audio_file_path)
+        return full_path
 
     def has_elements(self) -> bool:
         """Flag to indicate if there is still content to process"""

{sinapsis_data_readers-0.1.14 → sinapsis_data_readers-0.1.16}/src/sinapsis_data_readers/templates/base_file_data_loader.py

@@ -8,6 +8,7 @@ from sinapsis_core.template_base.base_models import (
     TemplateAttributes,
     TemplateAttributeType,
 )
+from sinapsis_core.utils.env_var_keys import SINAPSIS_CACHE_DIR
 
 
 def base_documentation() -> str:
@@ -80,7 +81,7 @@ class _BaseDataReader(Template, abc.ABC):
         __doc__ = f"""
         {base_attributes_documentation()}
         """
-
+        root_dir : str | None = None
         data_dir: str
         pattern: str = "**/*"
         batch_size: int = 1
@@ -93,8 +94,10 @@ class _BaseDataReader(Template, abc.ABC):
     def __init__(self, attributes: TemplateAttributeType) -> None:
         super().__init__(attributes)
         self.counter = 0
+        self.attributes.root_dir = self.attributes.root_dir or SINAPSIS_CACHE_DIR
         self.data_collection = self.make_data_entries()
 
+
     @abc.abstractmethod
     def make_data_entries(self) -> list[Packet]:
         """
@@ -181,4 +184,5 @@ class _BaseDataReader(Template, abc.ABC):
             self.append_packets_to_container(container)
         else:
             self.logger.debug(f"{self.class_name} has no more data to load.")
+
         return container

{sinapsis_data_readers-0.1.14 → sinapsis_data_readers-0.1.16}/src/sinapsis_data_readers/templates/datasets_readers/csv_datasets.py

@@ -1,26 +1,30 @@
 # -*- coding: utf-8 -*-
-from sinapsis_core.data_containers.data_packet import DataContainer, TextPacket, TimeSeriesPacket
+import os
+
+from sinapsis_core.data_containers.data_packet import DataContainer, TimeSeriesPacket
 from sinapsis_core.template_base.base_models import TemplateAttributes, TemplateAttributeType
 from sinapsis_core.template_base.template import Template
+from sinapsis_core.utils.env_var_keys import SINAPSIS_CACHE_DIR
 
 from sinapsis_data_readers.helpers.csv_reader import read_file
 
 
 class CSVDatasetReader(Template):
     class AttributesBaseModel(TemplateAttributes):
+        root_dir : str | None = None
         path_to_csv: str
         store_as_time_series: bool = False
-        store_as_text_packet: bool = True
 
     def __init__(self, attributes: TemplateAttributeType) -> None:
         super().__init__(attributes)
-        self.csv_file = read_file(self.attributes.path_to_csv)
+        self.attributes.root_dir = self.attributes.root_dir or SINAPSIS_CACHE_DIR
+        self.csv_file = read_file(os.path.join(self.attributes.root_dir, self.attributes.path_to_csv))
 
     def execute(self, container: DataContainer) -> DataContainer:
         if self.attributes.store_as_time_series:
             packet = TimeSeriesPacket(content=self.csv_file)
             container.time_series.append(packet)
-        if self.attributes.store_as_text_packet:
-            packet = TextPacket(content=self.csv_file)
-            container.texts.append(packet)
+        else:
+            self._set_generic_data(container, self.csv_file)
+
         return container

{sinapsis_data_readers-0.1.14 → sinapsis_data_readers-0.1.16}/src/sinapsis_data_readers/templates/image_readers/base_image_folder_data_loader.py

@@ -115,9 +115,10 @@ class ImageBaseDataReader(_BaseDataReader, abc.ABC):
         Returns:
             list[str| path]: the path as string or Path object
         """
+        full_path = os.path.join(self.attributes.root_dir, self.attributes.data_dir)
         data_items = [
             img_path if not return_as_str else str(img_path.resolve())
-            for img_path in Path(self.attributes.data_dir).glob(self.attributes.pattern)
+            for img_path in Path(full_path).glob(self.attributes.pattern)
             if img_path.suffix.lower() in SUPPORTED_IMAGE_TYPES
         ]
 

{sinapsis_data_readers-0.1.14 → sinapsis_data_readers-0.1.16}/src/sinapsis_data_readers/templates/image_readers/coco_dataset_reader.py

@@ -16,6 +16,7 @@ from sinapsis_core.data_containers.annotations import (
 )
 from sinapsis_core.data_containers.data_packet import ImagePacket
 from sinapsis_core.template_base.base_models import TemplateAttributeType
+from sinapsis_core.utils.env_var_keys import SINAPSIS_CACHE_DIR
 
 from sinapsis_data_readers.helpers.coco_dataclasses import (
     CocoAnnotationsKeys,
@@ -54,7 +55,8 @@ class CocoImageDatasetBaseCV2(FolderImageDatasetCV2):
         annotations_path: str
 
     def __init__(self, attributes: TemplateAttributeType) -> None:
-        self.annotations_file = os.path.join(attributes.get("data_dir"), attributes.get("annotations_path"))
+        self.annotations_file = os.path.join(attributes.get("root_dir", SINAPSIS_CACHE_DIR),
+                                             attributes.get("data_dir"), attributes.get("annotations_path"))
         self.raw_annotations_dict: list[dict[str, dict[str, Any]]] = self.read_annotations_file(self.annotations_file)
         self.annotations = self.images_annotations()
         super().__init__(attributes)

{sinapsis_data_readers-0.1.14 → sinapsis_data_readers-0.1.16}/src/sinapsis_data_readers/templates/image_readers/csv_dataset_reader.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 
-from typing import cast
+
+import os
 
 import numpy as np
 from sinapsis_core.data_containers.annotations import ImageAnnotations
@@ -10,6 +11,7 @@ from sinapsis_core.template_base.base_models import (
     TemplateAttributeType,
     UIPropertiesMetadata,
 )
+from sinapsis_core.utils.env_var_keys import SINAPSIS_CACHE_DIR
 
 from sinapsis_data_readers.helpers.csv_reader import read_file
 from sinapsis_data_readers.helpers.tags import Tags
@@ -75,11 +77,11 @@ class CSVImageDataset(_BaseDataReader):
         """
         # Ensure 'data_dir' is available and convert it to string if it's not None
         data_dir = getattr(self.attributes, "data_dir", None)
-
+        root_dir = getattr(self.attributes, "root_dir", SINAPSIS_CACHE_DIR)
         if data_dir is None:
             raise ValueError("The 'data_dir' attribute cannot be None.")
-
-        self.data_points = read_file(cast(str, data_dir))
+        full_path = os.path.join(root_dir, data_dir)
+        self.data_points = read_file(full_path)
         super().__init__(attributes)
 
     def read_packet_content(self, packet: ImagePacket) -> None:

{sinapsis_data_readers-0.1.14 → sinapsis_data_readers-0.1.16}/src/sinapsis_data_readers/templates/image_readers/image_folder_reader_cv2.py

@@ -21,7 +21,6 @@ def read_image_file(file_path: str | PosixPath | bytes) -> np.ndarray:
         np.ndarray: the image as a numpy array
     """
     np_image: np.ndarray
-
     if isinstance(file_path, bytes):
         image_arr = np.frombuffer(file_path, np.uint8)
         np_image = cv2.imdecode(image_arr, cv2.IMREAD_COLOR)

{sinapsis_data_readers-0.1.14 → sinapsis_data_readers-0.1.16}/src/sinapsis_data_readers/templates/video_readers/base_video_reader.py

@@ -22,6 +22,7 @@ from sinapsis_core.template_base.base_models import (
     TemplateAttributeType,
     UIPropertiesMetadata,
 )
+from sinapsis_core.utils.env_var_keys import SINAPSIS_CACHE_DIR
 
 from sinapsis_data_readers.helpers.file_path_helpers import parse_file_paths
 from sinapsis_data_readers.helpers.tags import Tags
@@ -40,7 +41,7 @@ class BaseVideoReaderAttributes(TemplateAttributes):
         device (Literal["cpu", "gpu"]): Device to be used for loading the video. Default is "cpu".
         loop_forever (bool): Whether to loop the video indefinitely. Default is False.
     """
-
+    root_dir: str | None = None
     video_file_path: str | list[str]
     batch_size: int = 1
     video_source: int | str | None = str(get_uuid())
@@ -59,6 +60,7 @@ class BaseVideoReader(Template):
 
     def __init__(self, attributes: TemplateAttributeType) -> None:
         super().__init__(attributes)
+        self.attributes.root_dir = self.attributes.root_dir or SINAPSIS_CACHE_DIR
         self.frame_count = 0
         self.video_reader: Any
         self.total_frames: int

{sinapsis_data_readers-0.1.14 → sinapsis_data_readers-0.1.16}/src/sinapsis_data_readers/templates/video_readers/video_reader_cv2.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 
 
+import os
+
 import cv2
 from sinapsis_core.data_containers.data_packet import ImagePacket
 
@@ -55,8 +57,8 @@ class VideoReaderCV2(BaseVideoReader):
                 - If successful, returns (video_reader, num_frames),
                   where video_reader is the OpenCV VideoCapture object.
         """
-
-        video_reader = cv2.VideoCapture(self.attributes.video_file_path)
+        full_path = os.path.join(self.attributes.root_dir, self.attributes.video_file_path)
+        video_reader = cv2.VideoCapture(full_path)
         num_frames = int(video_reader.get(cv2.CAP_PROP_FRAME_COUNT))
 
         if not video_reader.isOpened():

{sinapsis_data_readers-0.1.14 → sinapsis_data_readers-0.1.16}/src/sinapsis_data_readers/templates/video_readers/video_reader_dali.py

@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
-
-
+import os.path
 from typing import Literal, cast
 
 import nvidia.dali.fn as fn
@@ -111,13 +110,13 @@ class VideoReaderDali(BaseVideoReader):
             and the number of frames per epoch. If the pipeline cannot be created,
             it returns None and 0.
         """
-
+        full_path = os.path.join(self.attributes.root_dir, self.attributes.video_file_path)
         try:
             pipe: Pipeline = video_pipe(
                 batch_size=self.attributes.batch_size,
                 num_threads=self.attributes.num_threads,
                 device_id=0,
-                filenames=self.attributes.video_file_path,
+                filenames=full_path,
                 seed=12345,
                 device=self.attributes.device,
                 random_shuffle=self.attributes.random_shuffle,
@@ -154,6 +153,7 @@ class VideoReaderDali(BaseVideoReader):
         _ = template_name
         if self.attributes.device == "gpu":
             torch.cuda.empty_cache()
+            torch.cuda.ipc_collect()
         super().reset_state(template_name)
 
 

{sinapsis_data_readers-0.1.14 → sinapsis_data_readers-0.1.16}/src/sinapsis_data_readers/templates/video_readers/video_reader_ffmpeg.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-
+import os.path
 import subprocess
 
 import ffmpeg
@@ -54,8 +54,9 @@ class VideoReaderFFMPEG(BaseVideoReader):
         Returns:
             tuple[int, int, int]: the values for height, width and frames as integers
         """
+        full_path = os.path.join(self.attributes.root_dir, self.attributes.video_file_path)
         try:
-            probe = ffmpeg.probe(self.attributes.video_file_path)
+            probe = ffmpeg.probe(full_path)
         except ffmpeg.Error as e:
             self.logger.warning("ffmpeg error: %s", str(e))
             return (0, 0, 0)
@@ -72,8 +73,9 @@ class VideoReaderFFMPEG(BaseVideoReader):
 
     def make_video_reader(self) -> tuple[subprocess.Popen, int] | NotSetType:
         """This method asynchronously runs a subprocess to stream the video frames"""
+        full_path = os.path.join(self.attributes.root_dir, self.attributes.video_file_path)
         video_reader = (
-            ffmpeg.input(self.attributes.video_file_path)
+            ffmpeg.input(full_path)
             .output(
                 "pipe:",
                 format="rawvideo",

{sinapsis_data_readers-0.1.14 → sinapsis_data_readers-0.1.16}/src/sinapsis_data_readers/templates/video_readers/video_reader_torchcodec.py

@@ -1,4 +1,6 @@
 # -*- coding: utf-8 -*-
+import os.path
+
 import torch
 from sinapsis_core.data_containers.data_packet import ImagePacket
 from torchcodec.decoders import SimpleVideoDecoder
@@ -56,8 +58,9 @@ class VideoReaderTorchCodec(BaseVideoReader):
         Raises:
             ValueError: If there is an issue decoding the video file.
         """
+        full_path = os.path.join(self.attributes.root_dir, self.attributes.video_file_path)
         try:
-            video_reader = SimpleVideoDecoder(self.attributes.video_file_path)
+            video_reader = SimpleVideoDecoder(full_path)
         except ValueError as e:
             self.logger.warning(f"Was unable to decode video file: {e}")
             return NotSet

{sinapsis_data_readers-0.1.14 → sinapsis_data_readers-0.1.16/src/sinapsis_data_readers.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sinapsis-data-readers
-Version: 0.1.14
+Version: 0.1.16
 Summary: Templates to read data in different formats
 Author-email: SinapsisAI <dev@sinapsis.tech>
 Project-URL: Homepage, https://sinapsis.tech