sinapsis-generic-data-tools 0.1.7__tar.gz → 0.1.9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. {sinapsis_generic_data_tools-0.1.7/src/sinapsis_generic_data_tools.egg-info → sinapsis_generic_data_tools-0.1.9}/PKG-INFO +1 -1
  2. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.9}/pyproject.toml +1 -1
  3. sinapsis_generic_data_tools-0.1.9/src/sinapsis_generic_data_tools/helpers/audio_encoder.py +16 -0
  4. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.9}/src/sinapsis_generic_data_tools/helpers/encode_img_base64.py +14 -0
  5. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.9}/src/sinapsis_generic_data_tools/templates/__init__.py +1 -0
  6. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.9}/src/sinapsis_generic_data_tools/templates/image_color_conversion_np.py +11 -1
  7. sinapsis_generic_data_tools-0.1.9/src/sinapsis_generic_data_tools/templates/text_content_filter.py +74 -0
  8. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.9/src/sinapsis_generic_data_tools.egg-info}/PKG-INFO +1 -1
  9. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.9}/src/sinapsis_generic_data_tools.egg-info/SOURCES.txt +3 -1
  10. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.9}/LICENSE +0 -0
  11. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.9}/README.md +0 -0
  12. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.9}/setup.cfg +0 -0
  13. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.9}/src/sinapsis_generic_data_tools/__init__.py +0 -0
  14. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.9}/src/sinapsis_generic_data_tools/helpers/__init__.py +0 -0
  15. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.9}/src/sinapsis_generic_data_tools/helpers/file_downloader.py +0 -0
  16. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.9}/src/sinapsis_generic_data_tools/helpers/image_color_space_converter.py +0 -0
  17. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.9}/src/sinapsis_generic_data_tools/helpers/image_color_space_converter_cv.py +0 -0
  18. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.9}/src/sinapsis_generic_data_tools/helpers/image_color_space_converter_torch.py +0 -0
  19. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.9}/src/sinapsis_generic_data_tools/helpers/tags.py +0 -0
  20. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.9}/src/sinapsis_generic_data_tools/templates/image_color_conversion.py +0 -0
  21. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.9}/src/sinapsis_generic_data_tools/templates/mask_non_roi.py +0 -0
  22. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.9}/src/sinapsis_generic_data_tools/templates/packet_buffer_queue.py +0 -0
  23. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.9}/src/sinapsis_generic_data_tools/templates/source_history_aggregator.py +0 -0
  24. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.9}/src/sinapsis_generic_data_tools.egg-info/dependency_links.txt +0 -0
  25. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.9}/src/sinapsis_generic_data_tools.egg-info/requires.txt +0 -0
  26. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.9}/src/sinapsis_generic_data_tools.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: sinapsis-generic-data-tools
3
- Version: 0.1.7
3
+ Version: 0.1.9
4
4
  Summary: Collection of generic data tools
5
5
  Author-email: SinapsisAI <dev@sinapsis.tech>
6
6
  Project-URL: Homepage, https://sinapsis.tech
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "sinapsis-generic-data-tools"
3
- version = "0.1.7"
3
+ version = "0.1.9"
4
4
  description = "Collection of generic data tools"
5
5
  authors = [{ name = "SinapsisAI", email = "dev@sinapsis.tech" }]
6
6
 
@@ -0,0 +1,16 @@
1
+ # -*- coding: utf-8 -*-
2
+ from io import BytesIO
3
+
4
+ import numpy as np
5
+ from pydub import AudioSegment
6
+
7
+
8
+ def audio_bytes_to_numpy(audio_bytes: bytes) -> tuple[np.ndarray, int]:
9
+ audio = AudioSegment.from_file(BytesIO(audio_bytes), format="mp3")
10
+
11
+ samples = np.array(audio.get_array_of_samples())
12
+
13
+ if audio.channels == 2:
14
+ samples = samples.reshape((-1, 2))
15
+
16
+ return samples, audio.frame_rate
@@ -3,6 +3,7 @@ import base64
3
3
  from io import BytesIO
4
4
 
5
5
  import numpy as np
6
+ import requests # type:ignore[import-untyped]
6
7
  from PIL import Image
7
8
 
8
9
 
@@ -26,3 +27,16 @@ def convert_image_ndarray_to_base64(np_image: np.ndarray, image_format: str = "J
26
27
  base64_encoded_data = base64.b64encode(binary_data)
27
28
 
28
29
  return base64_encoded_data.decode("utf-8")
30
+
31
+
32
+ def decode_base64_to_numpy(base64_str: str) -> np.ndarray:
33
+ image_bytes = base64.b64decode(base64_str)
34
+ image = Image.open(BytesIO(image_bytes)).convert("RGB")
35
+ return np.array(image)
36
+
37
+
38
+ def fetch_url_to_numpy(url: str) -> np.ndarray:
39
+ response = requests.get(url)
40
+ response.raise_for_status()
41
+ image = Image.open(BytesIO(response.content)).convert("RGB")
42
+ return np.array(image)
@@ -9,6 +9,7 @@ _template_lookup = {
9
9
  "MaskNonROIs": f"{_root_lib_path}.mask_non_roi",
10
10
  "PacketBufferQueue": f"{_root_lib_path}.packet_buffer_queue",
11
11
  "SourceHistoryAggregator": f"{_root_lib_path}.source_history_aggregator",
12
+ "TextContentFilter": f"{_root_lib_path}.text_content_filter",
12
13
  }
13
14
 
14
15
 
@@ -4,9 +4,14 @@ from sinapsis_core.data_containers.data_packet import (
4
4
  ImageColor,
5
5
  )
6
6
  from sinapsis_core.template_base import Template
7
- from sinapsis_core.template_base.base_models import TemplateAttributes
7
+ from sinapsis_core.template_base.base_models import (
8
+ OutputTypes,
9
+ TemplateAttributes,
10
+ UIPropertiesMetadata,
11
+ )
8
12
 
9
13
  from sinapsis_generic_data_tools.helpers.image_color_space_converter_cv import convert_color_space_cv
14
+ from sinapsis_generic_data_tools.helpers.tags import Tags
10
15
 
11
16
 
12
17
  class ImageColorConversionNumpy(Template):
@@ -30,6 +35,11 @@ class ImageColorConversionNumpy(Template):
30
35
  target_color_space: 2
31
36
  """
32
37
 
38
+ UIProperties = UIPropertiesMetadata(
39
+ output_type=OutputTypes.IMAGE,
40
+ tags=[Tags.COLOR, Tags.CONVERSION, Tags.IMAGE, Tags.SPACE],
41
+ )
42
+
33
43
  class AttributesBaseModel(TemplateAttributes):
34
44
  """Defines the attributes required for the ColorConversion template.
35
45
 
@@ -0,0 +1,74 @@
1
+ # -*- coding: utf-8 -*-
2
+ import re
3
+ from collections import Counter
4
+
5
+ from sinapsis_core.data_containers.data_packet import DataContainer, TextPacket
6
+ from sinapsis_core.template_base.base_models import OutputTypes, TemplateAttributes, UIPropertiesMetadata
7
+ from sinapsis_core.template_base.template import Template
8
+
9
+ STOPWORDS = {"and", "the", "or", "is", "in", "to", "of", "a"}
10
+
11
+
12
+
13
+ def has_repetitive_chars(word: str) -> bool:
14
+ """Detect if a single character is repeated excessively in a word."""
15
+ count = 1
16
+ for i in range(1, len(word)):
17
+ if word[i] == word[i - 1]:
18
+ count += 1
19
+ if count >= 5:
20
+ return True
21
+ return False
22
+
23
+ def is_logical(
24
+ text: str, repeat_threshold: int = 10, unique_ratio_threshold: float = 0.1, window_size: int = 20
25
+ ) -> bool:
26
+ """Method to check whether a text string is logical, based on the number of repeated words in a
27
+ certain window size and a certain threshold.
28
+ Args:
29
+ text (str): text to be evaluated
30
+ repeat_threshold (int): The threshold for the number of repeated words
31
+ unique_ratio_threshold (float): the ratio between the number of words and that of repeated words
32
+ window_size (int): number of words in a single window size.
33
+ """
34
+ words = re.findall(r"\b\w+\b", text)
35
+ word_count = len(words)
36
+ unique_words = set(words)
37
+
38
+ if word_count == 0 or len(unique_words) / word_count < unique_ratio_threshold:
39
+ return False
40
+ for i in range(word_count - window_size + 1):
41
+ window = words[i : i + window_size]
42
+ counts = Counter(window)
43
+ for word, count in counts.items():
44
+ if word not in STOPWORDS and count > repeat_threshold:
45
+ return False
46
+ for word in words:
47
+ if has_repetitive_chars(word):
48
+ return False
49
+ return True
50
+
51
+
52
+ class TextContentFilter(Template):
53
+ UIProperties = UIPropertiesMetadata(output_types=OutputTypes.TEXT)
54
+
55
+ class AttributesBaseModel(TemplateAttributes):
56
+ repeated_word_threshold: int = 10
57
+ unique_words_ratio_threshold: float = 0.1
58
+ window_size: int = 20
59
+
60
+ def process_text_packet(self, packet: TextPacket) -> None:
61
+ """Evaluate a certain packet content for its logical meaning"""
62
+ is_text_logical = is_logical(
63
+ packet.content,
64
+ self.attributes.repeated_word_threshold,
65
+ self.attributes.unique_words_ratio_threshold,
66
+ self.attributes.window_size,
67
+ )
68
+ if not is_text_logical:
69
+ packet.content = "invalid string, please try again"
70
+
71
+ def execute(self, container: DataContainer) -> DataContainer:
72
+ for text in container.texts:
73
+ self.process_text_packet(text)
74
+ return container
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: sinapsis-generic-data-tools
3
- Version: 0.1.7
3
+ Version: 0.1.9
4
4
  Summary: Collection of generic data tools
5
5
  Author-email: SinapsisAI <dev@sinapsis.tech>
6
6
  Project-URL: Homepage, https://sinapsis.tech
@@ -8,6 +8,7 @@ src/sinapsis_generic_data_tools.egg-info/dependency_links.txt
8
8
  src/sinapsis_generic_data_tools.egg-info/requires.txt
9
9
  src/sinapsis_generic_data_tools.egg-info/top_level.txt
10
10
  src/sinapsis_generic_data_tools/helpers/__init__.py
11
+ src/sinapsis_generic_data_tools/helpers/audio_encoder.py
11
12
  src/sinapsis_generic_data_tools/helpers/encode_img_base64.py
12
13
  src/sinapsis_generic_data_tools/helpers/file_downloader.py
13
14
  src/sinapsis_generic_data_tools/helpers/image_color_space_converter.py
@@ -19,4 +20,5 @@ src/sinapsis_generic_data_tools/templates/image_color_conversion.py
19
20
  src/sinapsis_generic_data_tools/templates/image_color_conversion_np.py
20
21
  src/sinapsis_generic_data_tools/templates/mask_non_roi.py
21
22
  src/sinapsis_generic_data_tools/templates/packet_buffer_queue.py
22
- src/sinapsis_generic_data_tools/templates/source_history_aggregator.py
23
+ src/sinapsis_generic_data_tools/templates/source_history_aggregator.py
24
+ src/sinapsis_generic_data_tools/templates/text_content_filter.py