datamule 1.3.0__py3-none-any.whl → 1.4.0__py3-none-any.whl

datamule/document/document.py

@@ -3,11 +3,14 @@ import csv
 import re
 from doc2dict import xml2dict, txt2dict, dict2dict
 from doc2dict.mapping import flatten_hierarchy
+from doc2dict import html2dict, visualize_dict, get_title, unnest_dict
 from ..mapping_dicts.txt_mapping_dicts import dict_10k, dict_10q, dict_8k, dict_13d, dict_13g
 from ..mapping_dicts.xml_mapping_dicts import dict_345
+from ..mapping_dicts.html_mapping_dicts import dict_10k_html, dict_10q_html, dict_8k_html
 from selectolax.parser import HTMLParser
 from .processing import process_tabular_data
 from pathlib import Path
+import webbrowser
 
 class Document:
     def __init__(self, type, content, extension,accession,filing_date,path=None):
@@ -99,26 +102,10 @@ class Document:
         if self.data:
             return self.data
         
-        # preprocess content
-        if self.extension == '.txt':
-            self.content = self._preprocess_txt_content()
-        elif self.extension in ['.htm', '.html']:
-            self.content = self._preprocess_html_content()
-
         mapping_dict = None
-
-        if self.extension == '.xml':
-            if self.type in ['3', '4', '5', '3/A', '4/A', '5/A']:
-                mapping_dict = dict_345
-
-            self.data = xml2dict(content=self.content, mapping_dict=mapping_dict)
-
-
-
-        # will deprecate this when we add html2dict
-        elif self.extension in ['.htm', '.html','.txt']:
-
-
+        
+        if self.extension == '.txt':
+            content = self._preprocess_txt_content()
             if self.type == '10-Q':
                 mapping_dict = dict_10q
             elif self.type == '10-K':
@@ -131,8 +118,24 @@ class Document:
                 mapping_dict = dict_13g
             
             self.data = {}
-            self.data['document'] = dict2dict(txt2dict(content=self.content, mapping_dict=mapping_dict))
-        return self.data
+            self.data['document'] = dict2dict(txt2dict(content=content, mapping_dict=mapping_dict))
+        elif self.extension in ['.htm', '.html']:
+            if self.type == '10-K':
+                mapping_dict = dict_10k_html
+            elif self.type == '10-Q':
+                mapping_dict = dict_10q_html
+            elif self.type == '8-K':
+                mapping_dict = dict_8k_html
+            
+            dct = html2dict(content=self.content, mapping_dict=mapping_dict)
+            self.data = dct
+        elif self.extension == '.xml':
+            if self.type in ['3', '4', '5', '3/A', '4/A', '5/A']:
+                mapping_dict = dict_345
+            
+            self.data = xml2dict(content=self.content, mapping_dict=mapping_dict)
+        else:
+            pass
     
     def write_json(self, output_filename=None):
         if not self.data:
@@ -206,6 +209,31 @@ class Document:
                     })
         
         return items
+    
+    def visualize(self):
+        if not self.data:
+            self.parse()
+
+        if not self.data:
+            if self.extension in ['.jpg', '.png', '.pdf']:
+                webbrowser.open('file://' + str(self.path))
+            else:
+                pass
+        else:
+            visualize_dict(self.data)
+
+    def get_section(self, title, format='dict'):
+        if not self.data:
+            self.parse()
+
+        result = get_title(self.data,title)
+
+        if format == 'text':
+            result = [item[1] for item in result]
+            result = [unnest_dict(item) for item in result]
+
+        return result
+
    
    # this will all have to be changed. default will be to flatten everything
    # candidate for deletion

datamule/helper.py

@@ -5,28 +5,17 @@ import os
 
 def _load_package_csv(name):
     """Load CSV files from package data directory"""
-    # First try to load from the package data directory
-    try:
-        package_dir = os.path.dirname(os.path.dirname(__file__))
-        csv_path = os.path.join(package_dir, "data", f"{name}.csv")
-        
-        # Fallback to the legacy location
-        if not os.path.exists(csv_path):
-            csv_path = Path.home() / ".datamule" / f"{name}.csv"
-        
-        data = []
-        with open(csv_path, 'r') as csvfile:
-            csv_reader = csv.DictReader(csvfile)
-            for row in csv_reader:
-                data.append(row)
-        
-        return data
+    package_dir = os.path.dirname(os.path.dirname(__file__))
+    csv_path = os.path.join(package_dir,"datamule", "data", f"{name}.csv")
+    
+    data = []
+    with open(csv_path, 'r') as csvfile:
+        csv_reader = csv.DictReader(csvfile)
+        for row in csv_reader:
+            data.append(row)
+    
+    return data
     
-    except FileNotFoundError:
-        raise FileNotFoundError(
-            f"Required data file '{name}.csv' not found. "
-            f"This file should be in the datamule package directory or in ~/.datamule/"
-        )
 
 def load_package_dataset(dataset):
     if dataset =='listed_filer_metadata':

datamule/mapping_dicts/html_mapping_dicts.py

@@ -0,0 +1,11 @@
+dict_10k_html = {
+    ('part',r'^part\s*([ivx]+)$') : 0,
+    ('signatures',r'^signatures?\.*$') : 0,
+    ('item',r'^item\s*(\d+)\.?([a-z])?') : 1,
+}
+dict_10q_html = dict_10k_html
+
+dict_8k_html = {
+    ('signatures',r'^signatures?\.*$') : 0,
+    ('item',r'^item\s*(\d+\.\d+)') : 0,
+}

{datamule-1.3.0.dist-info → datamule-1.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.3.0
+Version: 1.4.0
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-1.3.0.dist-info → datamule-1.4.0.dist-info}/RECORD

@@ -1,6 +1,6 @@
 datamule/__init__.py,sha256=glzwBeGJEE6-TG7mRule9GH6L59XaIRR9T7ALcdpMus,1067
 datamule/config.py,sha256=Y--CVv7JcgrjJkMOSLrvm2S8B9ost6RMSkGviP-MKtg,883
-datamule/helper.py,sha256=BB4v73HDd5UWqG212UvSKGTTEW_ugohFPGKAiMboQ3s,4050
+datamule/helper.py,sha256=u6VNbxPuu505sDhXCF0pRyeYl3LSTtaFIVpAzqG39fU,3579
 datamule/index.py,sha256=_7Ox5hyF_7RWdblVFr5rNyv_ARwBP7VY4f703pk9qQ8,2074
 datamule/package_updater.py,sha256=Z9zaa_y0Z5cknpRn8oPea3gg4kquFHfpfhduKKCZ6NU,958
 datamule/portfolio.py,sha256=8fiK-vfZM5-NJSvOEsDR2YDb-2njjzFk6l7BiRyrzOM,7168
@@ -8,7 +8,7 @@ datamule/sheet.py,sha256=TvFqK9eAYuVoJ2uWdAlx5EN6vS9lke-aZf7FqtUiDBc,22304
 datamule/submission.py,sha256=Yh5nG3ioumhl6z30wJdIEmKjDDNSuo0r2xycZSIaeIg,11035
 datamule/data/listed_filer_metadata.csv,sha256=dT9fQ8AC5P1-Udf_UF0ZkdXJ88jNxJb_tuhi5YYL1rc,2426827
 datamule/document/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/document/document.py,sha256=menUFoeWwiY0rJnBkQiqY4NWnO0J17-qs8jFvO_1jiY,9969
+datamule/document/document.py,sha256=fQ7woCklxFwzABvjdFknFR8AFUwPuEEfuXZ_VE5q-7Y,10954
 datamule/document/processing.py,sha256=jDCEzBFDSQtq7nQxRScIsbALnFcvMPOkNkMUCa7mFxg,31921
 datamule/document/table.py,sha256=73yUJKY82ap32jhLmZeTti-jQ_lyhcJGlGwyxLtgYOg,12944
 datamule/document/mappings/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -36,6 +36,7 @@ datamule/document/mappings/thirteenfhr.py,sha256=XpYRIMPZnGLfEE4TqBI0BPXbyuq0xf3
 datamule/document/mappings/twentyfivense.py,sha256=lKyj0ZBhkHX9gQJMTUPrQlxYFg3k-aBnWqtoS5bujZM,905
 datamule/document/mappings/twentyfourf2nt.py,sha256=Q7RPT3JgJHjYdjMuaSyAxclt6QPT_LgCQloxp-ByDuI,4118
 datamule/mapping_dicts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datamule/mapping_dicts/html_mapping_dicts.py,sha256=Tfg7wX6EUUtsg_HD2KHH3Z6S3GR9FZc1pwL8kCGfRRU,282
 datamule/mapping_dicts/txt_mapping_dicts.py,sha256=DQPrGYbAPQxomRUtt4iiMGrwuF7BHc_LeFBQuYBzU9o,6311
 datamule/mapping_dicts/xml_mapping_dicts.py,sha256=Z22yDVwKYonUfM5foQP00dVDE8EHhhMKp0CLqVKV5OI,438
 datamule/sec/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -57,7 +58,7 @@ datamule/seclibrary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSu
 datamule/seclibrary/bq.py,sha256=C8sb_rpXTvchprrFLcbRar4Qi0XWW25tnv1YsHSS5o4,18025
 datamule/seclibrary/downloader.py,sha256=PIgz_7ASUTZOHcUZGcD1SmLaGSbq7xe7EiJT0Z7HU4M,13653
 datamule/seclibrary/query.py,sha256=qGuursTERRbOGfoDcYcpo4oWkW3PCBW6x1Qf1Puiak4,7352
-datamule-1.3.0.dist-info/METADATA,sha256=ANPyXCixCioGwfTqpxX0BsbnxDe7-LS3NtLWg0WledM,469
-datamule-1.3.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-datamule-1.3.0.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-1.3.0.dist-info/RECORD,,
+datamule-1.4.0.dist-info/METADATA,sha256=6H4Zzj37KnA56z-_XNAEv2NNpW3qKiEV_sul921KshM,469
+datamule-1.4.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+datamule-1.4.0.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-1.4.0.dist-info/RECORD,,