sinapsis-generic-data-tools 0.1.7__tar.gz → 0.1.8__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. {sinapsis_generic_data_tools-0.1.7/src/sinapsis_generic_data_tools.egg-info → sinapsis_generic_data_tools-0.1.8}/PKG-INFO +1 -1
  2. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.8}/pyproject.toml +1 -1
  3. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.8}/src/sinapsis_generic_data_tools/templates/__init__.py +1 -0
  4. sinapsis_generic_data_tools-0.1.8/src/sinapsis_generic_data_tools/templates/text_content_filter.py +60 -0
  5. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.8/src/sinapsis_generic_data_tools.egg-info}/PKG-INFO +1 -1
  6. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.8}/src/sinapsis_generic_data_tools.egg-info/SOURCES.txt +2 -1
  7. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.8}/LICENSE +0 -0
  8. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.8}/README.md +0 -0
  9. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.8}/setup.cfg +0 -0
  10. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.8}/src/sinapsis_generic_data_tools/__init__.py +0 -0
  11. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.8}/src/sinapsis_generic_data_tools/helpers/__init__.py +0 -0
  12. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.8}/src/sinapsis_generic_data_tools/helpers/encode_img_base64.py +0 -0
  13. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.8}/src/sinapsis_generic_data_tools/helpers/file_downloader.py +0 -0
  14. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.8}/src/sinapsis_generic_data_tools/helpers/image_color_space_converter.py +0 -0
  15. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.8}/src/sinapsis_generic_data_tools/helpers/image_color_space_converter_cv.py +0 -0
  16. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.8}/src/sinapsis_generic_data_tools/helpers/image_color_space_converter_torch.py +0 -0
  17. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.8}/src/sinapsis_generic_data_tools/helpers/tags.py +0 -0
  18. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.8}/src/sinapsis_generic_data_tools/templates/image_color_conversion.py +0 -0
  19. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.8}/src/sinapsis_generic_data_tools/templates/image_color_conversion_np.py +0 -0
  20. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.8}/src/sinapsis_generic_data_tools/templates/mask_non_roi.py +0 -0
  21. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.8}/src/sinapsis_generic_data_tools/templates/packet_buffer_queue.py +0 -0
  22. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.8}/src/sinapsis_generic_data_tools/templates/source_history_aggregator.py +0 -0
  23. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.8}/src/sinapsis_generic_data_tools.egg-info/dependency_links.txt +0 -0
  24. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.8}/src/sinapsis_generic_data_tools.egg-info/requires.txt +0 -0
  25. {sinapsis_generic_data_tools-0.1.7 → sinapsis_generic_data_tools-0.1.8}/src/sinapsis_generic_data_tools.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: sinapsis-generic-data-tools
3
- Version: 0.1.7
3
+ Version: 0.1.8
4
4
  Summary: Collection of generic data tools
5
5
  Author-email: SinapsisAI <dev@sinapsis.tech>
6
6
  Project-URL: Homepage, https://sinapsis.tech
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "sinapsis-generic-data-tools"
3
- version = "0.1.7"
3
+ version = "0.1.8"
4
4
  description = "Collection of generic data tools"
5
5
  authors = [{ name = "SinapsisAI", email = "dev@sinapsis.tech" }]
6
6
 
@@ -9,6 +9,7 @@ _template_lookup = {
9
9
  "MaskNonROIs": f"{_root_lib_path}.mask_non_roi",
10
10
  "PacketBufferQueue": f"{_root_lib_path}.packet_buffer_queue",
11
11
  "SourceHistoryAggregator": f"{_root_lib_path}.source_history_aggregator",
12
+ "TextContentFilter": f"{_root_lib_path}.text_content_filter",
12
13
  }
13
14
 
14
15
 
@@ -0,0 +1,60 @@
1
+ # -*- coding: utf-8 -*-
2
+ import re
3
+ from collections import Counter
4
+
5
+ from sinapsis_core.data_containers.data_packet import DataContainer, TextPacket
6
+ from sinapsis_core.template_base.base_models import OutputTypes, TemplateAttributes, UIPropertiesMetadata
7
+ from sinapsis_core.template_base.template import Template
8
+
9
+ STOPWORDS = {"and", "the", "or", "is", "in", "to", "of", "a"}
10
+
11
+
12
+ def is_logical(
13
+ text: str, repeat_threshold: int = 10, unique_ratio_threshold: float = 0.1, window_size: int = 20
14
+ ) -> bool:
15
+ """Method to check whether a text string is logical, based on the number of repeated words in a
16
+ certain window size and a certain threshold.
17
+ Args:
18
+ text (str): text to be evaluated
19
+ repeat_threshold (int): The threshold for the number of repeated words
20
+ unique_ratio_threshold (float): the ratio between the number of words and that of repeated words
21
+ window_size (int): number of words in a single window size.
22
+ """
23
+ words = re.findall(r"\b\w+\b", text)
24
+ word_count = len(words)
25
+ unique_words = set(words)
26
+
27
+ if word_count == 0 or len(unique_words) / word_count < unique_ratio_threshold:
28
+ return False
29
+ for i in range(word_count - window_size + 1):
30
+ window = words[i : i + window_size]
31
+ counts = Counter(window)
32
+ for word, count in counts.items():
33
+ if word not in STOPWORDS and count > repeat_threshold:
34
+ return False
35
+ return True
36
+
37
+
38
+ class TextContentFilter(Template):
39
+ UIProperties = UIPropertiesMetadata(output_types=OutputTypes.TEXT)
40
+
41
+ class AttributesBaseModel(TemplateAttributes):
42
+ repeated_word_threshold: int = 10
43
+ unique_words_ratio_threshold: float = 0.1
44
+ window_size: int = 20
45
+
46
+ def process_text_packet(self, packet: TextPacket) -> None:
47
+ """Evaluate a certain packet content for its logical meaning"""
48
+ is_text_logical = is_logical(
49
+ packet.content,
50
+ self.attributes.repeated_word_threshold,
51
+ self.attributes.unique_words_ratio_threshold,
52
+ self.attributes.window_size,
53
+ )
54
+ if not is_text_logical:
55
+ packet.content = "invalid string, please try again"
56
+
57
+ def execute(self, container: DataContainer) -> DataContainer:
58
+ for text in container.texts:
59
+ self.process_text_packet(text)
60
+ return container
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: sinapsis-generic-data-tools
3
- Version: 0.1.7
3
+ Version: 0.1.8
4
4
  Summary: Collection of generic data tools
5
5
  Author-email: SinapsisAI <dev@sinapsis.tech>
6
6
  Project-URL: Homepage, https://sinapsis.tech
@@ -19,4 +19,5 @@ src/sinapsis_generic_data_tools/templates/image_color_conversion.py
19
19
  src/sinapsis_generic_data_tools/templates/image_color_conversion_np.py
20
20
  src/sinapsis_generic_data_tools/templates/mask_non_roi.py
21
21
  src/sinapsis_generic_data_tools/templates/packet_buffer_queue.py
22
- src/sinapsis_generic_data_tools/templates/source_history_aggregator.py
22
+ src/sinapsis_generic_data_tools/templates/source_history_aggregator.py
23
+ src/sinapsis_generic_data_tools/templates/text_content_filter.py