datamule 2.2.0__py3-none-any.whl → 2.2.1__py3-none-any.whl

datamule/document/document.py

@@ -64,11 +64,15 @@ class Tags:
         self.document = document
         self._tickers = None
         self.dictionaries = {}
+        self.processors = {}
         
-        # Load global dictionaries with their data
+        # Load global dictionaries with their data and processors
         active_dicts = _active_dictionaries
         for dict_name in active_dicts:
-            self.dictionaries[dict_name] = _loaded_dictionaries[dict_name]
+            dict_info = _loaded_dictionaries[dict_name]
+            self.dictionaries[dict_name] = dict_info['data']
+            if dict_info['processor'] is not None:
+                self.processors[dict_name] = dict_info['processor']
             
     
     def _check_support(self):
@@ -128,9 +132,9 @@ class Tags:
             return None
         
         if not hasattr(self, '_persons'):
-            if '8k_2024_persons' in self.dictionaries:
-                # Use FlashText dictionary lookup for 8K persons
-                self._persons = get_full_names_dictionary_lookup(self.document.text, self.dictionaries['8k_2024_persons'])
+            if '8k_2024_persons' in self.processors:
+                # Use pre-built processor
+                self._persons = get_full_names_dictionary_lookup(self.document.text, self.processors['8k_2024_persons'])
             elif 'ssa_baby_first_names' in self.dictionaries:
                 # Use regex with SSA names for validation
                 self._persons = get_full_names(self.document.text, self.dictionaries['ssa_baby_first_names'])

datamule/tags/config.py

@@ -12,5 +12,22 @@ def set_dictionaries(dictionaries, overwrite=False):
     for dict_name in dictionaries:
         # Download if needed
         download_dictionary(dict_name, overwrite=overwrite)
-        # Load into memory
-        _loaded_dictionaries[dict_name] = load_dictionary(dict_name)
+        # Load raw data
+        raw_data = load_dictionary(dict_name)
+        
+        # Create processor for dictionary lookup methods
+        if dict_name in ['8k_2024_persons']:  # Add other dict names as needed
+            from flashtext import KeywordProcessor
+            processor = KeywordProcessor(case_sensitive=True)
+            for key in raw_data.keys():
+                processor.add_keyword(key, key)
+            
+            _loaded_dictionaries[dict_name] = {
+                'data': raw_data,
+                'processor': processor
+            }
+        else:
+            _loaded_dictionaries[dict_name] = {
+                'data': raw_data,
+                'processor': None
+            }

datamule/tags/utils.py

@@ -134,14 +134,10 @@ def get_full_names(text,keywords=None):
     return full_names
 
 # add dictionary lookup based on precomputed lists
-def get_full_names_dictionary_lookup(text, dictionary):
-    keyword_processor = KeywordProcessor(case_sensitive=True)
-    
-    for key in dictionary.keys():
-        keyword_processor.add_keyword(key, key)
-    
+def get_full_names_dictionary_lookup(text, processor):
+    """Use pre-built KeywordProcessor instead of creating new one"""
     matches = []
-    keywords_found = keyword_processor.extract_keywords(text, span_info=True)
+    keywords_found = processor.extract_keywords(text, span_info=True)
     
     for keyword, start_pos, end_pos in keywords_found:
         matches.append((keyword, start_pos, end_pos))

{datamule-2.2.0.dist-info → datamule-2.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 2.2.0
+Version: 2.2.1
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-2.2.0.dist-info → datamule-2.2.1.dist-info}/RECORD

@@ -15,7 +15,7 @@ datamule/datamule/datamule_mysql_rds.py,sha256=Q6_h24-SNECWK60RnM6UQjUIp5dhJmfn3
 datamule/datamule/downloader.py,sha256=mVg1SApfij_9-dTpcm_YB26Bxc_Yq1FR8xv2k50MHqU,18579
 datamule/datamule/sec_connector.py,sha256=VwOaODpHoAWy8JIky6kLR1-orW_PB61RHw7pIGRpkow,3288
 datamule/document/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/document/document.py,sha256=QjncYOIdf0Zf_0AONEOXu2KlPxMksGZzvwmHOpbM5N8,20450
+datamule/document/document.py,sha256=yiev4AYewjp8bPjWn9cuL43N2O11s9WUo4X2e7WUgiY,20628
 datamule/document/tables/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/document/tables/tables.py,sha256=8riSAof6o-Gxoo0SkiQAE61fw8NmzDnEhJe6dATzmvA,4487
 datamule/document/tables/tables_13fhr.py,sha256=-6tWcaTyNsb0XuW0WMBrYir9Zn1wLZL0laKxRYfPNyg,4265
@@ -49,14 +49,14 @@ datamule/sec/xbrl/xbrlmonitor.py,sha256=TKFVfSyyUUfUgFQw4WxEVs4g8Nh-2C0tygNIRmTq
 datamule/seclibrary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/seclibrary/bq.py,sha256=C8sb_rpXTvchprrFLcbRar4Qi0XWW25tnv1YsHSS5o4,18025
 datamule/tags/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/tags/config.py,sha256=JVdIkqu9rBEAadNLP-FiIbZ35TRORGIDCJvqDh0CuqE,585
+datamule/tags/config.py,sha256=RCYRw_voP2MrEx_iN7zjJiZ8YDa4QlzKPGpW5ZTij6U,1197
 datamule/tags/regex.py,sha256=Zr1dlnb8OfecDkI2DFCI8DUBr9LI50fapQyBAYNEZrg,4487
-datamule/tags/utils.py,sha256=k5fyjMjJNh6gZjj491sw_9rnMqYIlHHDBathkDcHD0A,5423
+datamule/tags/utils.py,sha256=hQpQBVAJPmys1UKVS2mqc8Z5-qO_zma5ecFXvW9DXoo,5329
 datamule/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/utils/construct_submissions_data.py,sha256=NB_hvfxlRXPyt4Fgc-5qA8vJRItkLhBedCSTaxwW7Jg,5887
 datamule/utils/dictionaries.py,sha256=VImvQWlP8IohB76rDd83bZcT184LBOpOaXPOH46fA6Y,2795
 datamule/utils/format_accession.py,sha256=60RtqoNqoT9zSKVb1DeOv1gncJxzPTFMNW4SNOVmC_g,476
-datamule-2.2.0.dist-info/METADATA,sha256=fuT_ABK8D6LhEi1_TjtVnIKobXdafBPiMSGy3XCWyRo,585
-datamule-2.2.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-datamule-2.2.0.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-2.2.0.dist-info/RECORD,,
+datamule-2.2.1.dist-info/METADATA,sha256=aINGZMWV34SclEt-2Ij2d2848PJA7cLF6ZoBL2LwpfY,585
+datamule-2.2.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+datamule-2.2.1.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-2.2.1.dist-info/RECORD,,