datamule 2.2.0__tar.gz → 2.2.1__tar.gz

{datamule-2.2.0 → datamule-2.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 2.2.0
+Version: 2.2.1
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-2.2.0 → datamule-2.2.1}/datamule/document/document.py

@@ -64,11 +64,15 @@ class Tags:
         self.document = document
         self._tickers = None
         self.dictionaries = {}
+        self.processors = {}
         
-        # Load global dictionaries with their data
+        # Load global dictionaries with their data and processors
         active_dicts = _active_dictionaries
         for dict_name in active_dicts:
-            self.dictionaries[dict_name] = _loaded_dictionaries[dict_name]
+            dict_info = _loaded_dictionaries[dict_name]
+            self.dictionaries[dict_name] = dict_info['data']
+            if dict_info['processor'] is not None:
+                self.processors[dict_name] = dict_info['processor']
             
     
     def _check_support(self):
@@ -128,9 +132,9 @@ class Tags:
             return None
         
         if not hasattr(self, '_persons'):
-            if '8k_2024_persons' in self.dictionaries:
-                # Use FlashText dictionary lookup for 8K persons
-                self._persons = get_full_names_dictionary_lookup(self.document.text, self.dictionaries['8k_2024_persons'])
+            if '8k_2024_persons' in self.processors:
+                # Use pre-built processor
+                self._persons = get_full_names_dictionary_lookup(self.document.text, self.processors['8k_2024_persons'])
             elif 'ssa_baby_first_names' in self.dictionaries:
                 # Use regex with SSA names for validation
                 self._persons = get_full_names(self.document.text, self.dictionaries['ssa_baby_first_names'])

datamule-2.2.1/datamule/tags/config.py

@@ -0,0 +1,33 @@
+from ..utils.dictionaries import download_dictionary, load_dictionary
+
+_active_dictionaries = []
+_loaded_dictionaries = {}
+
+def set_dictionaries(dictionaries, overwrite=False):
+    """Set active dictionaries and load them into memory"""
+    global _active_dictionaries, _loaded_dictionaries
+    _active_dictionaries = dictionaries
+    _loaded_dictionaries = {}
+    
+    for dict_name in dictionaries:
+        # Download if needed
+        download_dictionary(dict_name, overwrite=overwrite)
+        # Load raw data
+        raw_data = load_dictionary(dict_name)
+        
+        # Create processor for dictionary lookup methods
+        if dict_name in ['8k_2024_persons']:  # Add other dict names as needed
+            from flashtext import KeywordProcessor
+            processor = KeywordProcessor(case_sensitive=True)
+            for key in raw_data.keys():
+                processor.add_keyword(key, key)
+            
+            _loaded_dictionaries[dict_name] = {
+                'data': raw_data,
+                'processor': processor
+            }
+        else:
+            _loaded_dictionaries[dict_name] = {
+                'data': raw_data,
+                'processor': None
+            }

{datamule-2.2.0 → datamule-2.2.1}/datamule/tags/utils.py

@@ -134,14 +134,10 @@ def get_full_names(text,keywords=None):
     return full_names
 
 # add dictionary lookup based on precomputed lists
-def get_full_names_dictionary_lookup(text, dictionary):
-    keyword_processor = KeywordProcessor(case_sensitive=True)
-    
-    for key in dictionary.keys():
-        keyword_processor.add_keyword(key, key)
-    
+def get_full_names_dictionary_lookup(text, processor):
+    """Use pre-built KeywordProcessor instead of creating new one"""
     matches = []
-    keywords_found = keyword_processor.extract_keywords(text, span_info=True)
+    keywords_found = processor.extract_keywords(text, span_info=True)
     
     for keyword, start_pos, end_pos in keywords_found:
         matches.append((keyword, start_pos, end_pos))

{datamule-2.2.0 → datamule-2.2.1}/datamule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 2.2.0
+Version: 2.2.1
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-2.2.0 → datamule-2.2.1}/setup.py

@@ -32,7 +32,7 @@ if not os.path.exists(file_path):
 setup(
     name="datamule",
     author="John Friedman",
-    version="2.2.0",
+    version="2.2.1",
     description="Work with SEC submissions at scale.",
     packages=find_packages(include=['datamule', 'datamule.*']),
     url="https://github.com/john-friedman/datamule-python",

datamule-2.2.0/datamule/tags/config.py

@@ -1,16 +0,0 @@
-from ..utils.dictionaries import download_dictionary, load_dictionary
-
-_active_dictionaries = []
-_loaded_dictionaries = {}
-
-def set_dictionaries(dictionaries, overwrite=False):
-    """Set active dictionaries and load them into memory"""
-    global _active_dictionaries, _loaded_dictionaries
-    _active_dictionaries = dictionaries
-    _loaded_dictionaries = {}
-    
-    for dict_name in dictionaries:
-        # Download if needed
-        download_dictionary(dict_name, overwrite=overwrite)
-        # Load into memory
-        _loaded_dictionaries[dict_name] = load_dictionary(dict_name)