datamule 2.1.1__py3-none-any.whl → 2.1.3__py3-none-any.whl

datamule/datamule/downloader.py

@@ -23,7 +23,11 @@ from ..utils.format_accession import format_accession
 # could be cleaned up
 
 # Set up logging
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    handlers=logging.getLogger().handlers,
+)
 logger = logging.getLogger(__name__)
 
 

datamule/document/document.py

@@ -12,6 +12,7 @@ from selectolax.parser import HTMLParser
 from pathlib import Path
 import webbrowser
 from secsgml.utils import bytes_to_str
+import tempfile
 
 from .tables.tables import Tables
 
@@ -36,18 +37,19 @@ class Document:
         # this will be filled by parsed
         self._data = None
         self._tables = None
+        self._text = None
 
 
 
     #_load_text_content
     def _preprocess_txt_content(self):
-            return self.content.translate(str.maketrans({
+            self._text = self.content.decode().translate(str.maketrans({
                 '\xa0': ' ', '\u2003': ' ',
                 '\u2018': "'", '\u2019': "'",
                 '\u201c': '"', '\u201d': '"'
             }))
 
-    # will deprecate this when we add html2dict
+    # needs work
     def _preprocess_html_content(self):
         parser = HTMLParser(self.content,detect_encoding=True,decode_errors='ignore')
     
@@ -95,7 +97,7 @@ class Document:
         while '\n\n\n' in text:
             text = text.replace('\n\n\n', '\n\n')
         
-        return text.translate(str.maketrans({
+        self._text = text.translate(str.maketrans({
             '\xa0': ' ', '\u2003': ' ',
             '\u2018': "'", '\u2019': "'",
             '\u201c': '"', '\u201d': '"'
@@ -116,7 +118,7 @@ class Document:
         mapping_dict = None
         
         if self.extension == '.txt':
-            content = self._preprocess_txt_content()
+            content = self.text
             if self.type == '10-Q':
                 mapping_dict = dict_10q
             elif self.type == '10-K':
@@ -224,6 +226,15 @@ class Document:
             self.parse()
         return self._data
     
+    @property
+    def text(self):
+        if self._text is None:
+            if self.extension in ['.htm','.html']:
+                self._preprocess_html_content()
+            elif self.extension == '.txt':
+                self._preprocess_txt_content()
+            return self._text
+    
     def write_json(self, output_filename=None):
         if not self.data:
             self.parse()
@@ -308,18 +319,28 @@ class Document:
             self.parse()
 
         if not self.data:
-            if self.extension in ['.jpg', '.png', '.pdf']:
-                webbrowser.open('file://' + str(self.path))
-            else:
-                pass
+            pass
         else:
             visualize_dict(self.data)
 
-    def get_section(self, title, format='dict'):
+    # alpha feature
+    def open(self):
+        """Open the document. Experimental. Creates copy in temp, rather than use tar path for now."""
+        if self.extension in ['.htm', '.html','.txt','.jpg','.png', '.pdf']:
+            # Create a temporary file with the content and open it
+
+            with tempfile.NamedTemporaryFile(mode='wb', suffix=self.extension, delete=False) as f:
+                f.write(self.content)
+                temp_path = f.name
+            webbrowser.open('file://' + temp_path)
+        else:
+            print(f"Cannot open files with extension {self.extension}")
+
+    def get_section(self, title=None, title_regex=None,title_class=None, format='dict'):
         if not self.data:
             self.parse()
 
-        result = get_title(self.data,title)
+        result = get_title(self.data,title=title,title_regex=title_regex,title_class=title_class)
 
         if format == 'text':
             result = [item[1] for item in result]

datamule/mapping_dicts/html_mapping_dicts.py

@@ -1,7 +1,7 @@
 dict_10k_html = {
     ('part',r'^part\s*([ivx]+)$') : 0,
     ('signatures',r'^signatures?\.*$') : 0,
-    ('item',r'^item\s*(\d+)\.?([a-z])?') : 1,
+    ('item',r'^item\s*(\d+)\.?([a-z])?(?![a-z])') : 1,
 }
 dict_10q_html = dict_10k_html
 
@@ -48,7 +48,7 @@ dict_10d_html = dict_10k_html
 
 dict_20f_html = {
     ('part',r'^part\s*([ivx]+)') : 0,
-    ('item',r'^item\s*(\d+)\.?([a-z])?') : 1,
+    ('item',r'^item\s*(\d+)\.?([a-z])?(?![a-z])') : 1,
     ('letter',r'\d*\.?([a-z])') : 2,
     ('signatures',r'^signatures?\.*$') : 0,
 }

datamule/submission.py

@@ -163,8 +163,8 @@ class Submission:
                     content = zstd.ZstdDecompressor().decompress(content)
                 
                 # Decode text files
-                if extension in ['.htm', '.html', '.txt', '.xml']:
-                    content = content.decode('utf-8', errors='replace')
+                # if extension in ['.htm', '.html', '.txt', '.xml']:
+                #     content = content.decode('utf-8', errors='replace')
                 
                 document_path = f"{self.batch_tar_path}::{self.accession_prefix}/{filename}"
         
@@ -197,8 +197,8 @@ class Submission:
                         content = zstd.ZstdDecompressor().decompress(content)
                     
                     # Decode text files
-                    if extension in ['.htm', '.html', '.txt', '.xml']:
-                        content = content.decode('utf-8', errors='replace')
+                    # if extension in ['.htm', '.html', '.txt', '.xml']:
+                    #     content = content.decode('utf-8', errors='replace')
                     
                     document_path = f"{self.path}::{actual_filename}"
             
@@ -219,8 +219,8 @@ class Submission:
                     content = zstd.ZstdDecompressor().decompress(content)
                 
                 # Decode text files
-                if extension in ['.htm', '.html', '.txt', '.xml']:
-                    content = content.decode('utf-8', errors='replace')
+                # if extension in ['.htm', '.html', '.txt', '.xml']:
+                #     content = content.decode('utf-8', errors='replace')
 
         return Document(
             type=doc['type'], 

{datamule-2.1.1.dist-info → datamule-2.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 2.1.1
+Version: 2.1.3
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-2.1.1.dist-info → datamule-2.1.3.dist-info}/RECORD

@@ -6,15 +6,15 @@ datamule/package_updater.py,sha256=Z9zaa_y0Z5cknpRn8oPea3gg4kquFHfpfhduKKCZ6NU,9
 datamule/portfolio.py,sha256=YViG1JgJ9SFhg8N3tOOhBI8oc6Pmi2vwnHeHmlkC_5U,12119
 datamule/portfolio_compression_utils.py,sha256=8OPYEN5zAdV1FiTxgVN3S7cTKs99Elv74bwgoIJP4QY,12654
 datamule/sheet.py,sha256=Ws_YRtpvewLVioarngVMe8cgG_sp11MP9_goGbRaiWE,23952
-datamule/submission.py,sha256=7rtN3EEB50iU7E-B_i-e0JHY382EIgmDcrS1KRZ1mUc,15886
+datamule/submission.py,sha256=piMtTyoMZrKmLBpjyi0BBFhlkugi_CEuyox7J-jnusQ,15898
 datamule/data/listed_filer_metadata.csv,sha256=dT9fQ8AC5P1-Udf_UF0ZkdXJ88jNxJb_tuhi5YYL1rc,2426827
 datamule/datamule/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/datamule/datamule_lookup.py,sha256=e8djAg-ctSyHiKk7BjbtgugZ3p8roUjzsym5z3AihUg,9468
 datamule/datamule/datamule_mysql_rds.py,sha256=Q6_h24-SNECWK60RnM6UQjUIp5dhJmfn3SSKzTITB3o,12317
-datamule/datamule/downloader.py,sha256=aTyVUuIwynPtHB0Z9BvCasy9Ao5wfHptNAsjN-7yDTk,18525
+datamule/datamule/downloader.py,sha256=mVg1SApfij_9-dTpcm_YB26Bxc_Yq1FR8xv2k50MHqU,18579
 datamule/datamule/sec_connector.py,sha256=VwOaODpHoAWy8JIky6kLR1-orW_PB61RHw7pIGRpkow,3288
 datamule/document/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/document/document.py,sha256=OtfIvHRxdIXmNq5lOpBiLQw9P4o9OYScdpJxNEX51Yc,14477
+datamule/document/document.py,sha256=lWFmRnX7UfcX2W-2bs5wuFjWDiuFphIlWRXoatAOMZ0,15328
 datamule/document/tables/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/document/tables/tables.py,sha256=8riSAof6o-Gxoo0SkiQAE61fw8NmzDnEhJe6dATzmvA,4487
 datamule/document/tables/tables_13fhr.py,sha256=-6tWcaTyNsb0XuW0WMBrYir9Zn1wLZL0laKxRYfPNyg,4265
@@ -27,7 +27,7 @@ datamule/document/tables/tables_sbsef.py,sha256=X6VKVnAdWxn2TgRmaAd1WWlxPhcLPQ-5
 datamule/document/tables/tables_sdr.py,sha256=BwHRJvtijiYvNJ2lIc_30kct6VEmLimIzX28JjZBBqo,4924
 datamule/document/tables/utils.py,sha256=2-X_1NsiWj_XsD9djxCXwTeIVlg-ip78gG11xACJiDs,738
 datamule/mapping_dicts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/mapping_dicts/html_mapping_dicts.py,sha256=G2PWB__FNg4VH9iFJFkflM0u-qOEtk67IWtGoqesb0k,5388
+datamule/mapping_dicts/html_mapping_dicts.py,sha256=OmelylkccxcPcz6Qv28r6rjbwOhUGnz5Vmzy_BYC0Pg,5406
 datamule/mapping_dicts/txt_mapping_dicts.py,sha256=DQPrGYbAPQxomRUtt4iiMGrwuF7BHc_LeFBQuYBzU9o,6311
 datamule/mapping_dicts/xml_mapping_dicts.py,sha256=Z22yDVwKYonUfM5foQP00dVDE8EHhhMKp0CLqVKV5OI,438
 datamule/sec/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -50,7 +50,7 @@ datamule/seclibrary/bq.py,sha256=C8sb_rpXTvchprrFLcbRar4Qi0XWW25tnv1YsHSS5o4,180
 datamule/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/utils/construct_submissions_data.py,sha256=NB_hvfxlRXPyt4Fgc-5qA8vJRItkLhBedCSTaxwW7Jg,5887
 datamule/utils/format_accession.py,sha256=60RtqoNqoT9zSKVb1DeOv1gncJxzPTFMNW4SNOVmC_g,476
-datamule-2.1.1.dist-info/METADATA,sha256=c7wijhGbi_7q3LNTf1HvZcGJtCgvvO43nxcdqj3tWhs,560
-datamule-2.1.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-datamule-2.1.1.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-2.1.1.dist-info/RECORD,,
+datamule-2.1.3.dist-info/METADATA,sha256=jE2eNeO223VJUDysOc9ENyiOIDbVsv5DEXdCnXrhFRA,560
+datamule-2.1.3.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+datamule-2.1.3.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-2.1.3.dist-info/RECORD,,