datamule 1.4.4__py3-none-any.whl → 1.4.5__py3-none-any.whl

datamule/document/document.py

@@ -12,6 +12,19 @@ from .processing import process_tabular_data
 from pathlib import Path
 import webbrowser
 
+def convert_bytes_keys(obj):
+    if isinstance(obj, dict):
+        return {
+            (k.decode('utf-8').lower() if isinstance(k, bytes) else k): convert_bytes_keys(v) 
+            for k, v in obj.items()
+        }
+    elif isinstance(obj, list):
+        return [convert_bytes_keys(item) for item in obj]
+    elif isinstance(obj, bytes):
+        return obj.decode('utf-8').lower()
+    else:
+        return obj
+
 class Document:
     def __init__(self, type, content, extension,accession,filing_date,path=None):
         
@@ -19,7 +32,11 @@ class Document:
         extension = extension.lower()
         self.accession = accession
         self.filing_date = filing_date
-        self.content = content
+
+        if self.type == 'submission_metadata':
+            self.content = convert_bytes_keys(content)
+        else:
+            self.content = content
 
         if path is not None:
             self.path = path
@@ -157,8 +174,40 @@ class Document:
                 dict_appntc_html
             elif self.type == 'CB':
                 mapping_dict = dict_cb_html
+            elif self.type == 'DSTRBRPT':
+                mapping_dict = dict_dstrbrpt_html
+            elif self.type == 'N-18F1':
+                mapping_dict = dict_n18f1_html
+            elif self.type == 'N-CSRS':
+                mapping_dict = dict_ncsrs_html
+            elif self.type == 'NT-10K':
+                mapping_dict = dict_nt10k_html
+            elif self.type == 'NT-10Q':
+                mapping_dict = dict_nt10q_html
+            elif self.type == 'NT 20-F':
+                mapping_dict = dict_nt20f_html
+            elif self.type == 'NT-NCEN':
+                mapping_dict = dict_ntncen_html
+            elif self.type == 'NT-NCSR':
+                mapping_dict = dict_ntncsr_html
+            elif self.type == 'NTFNCEN':
+                mapping_dict = dict_ntfcen_html
+            elif self.type == 'NTFNCSR':
+                mapping_dict = dict_ntfncsr_html
+            elif self.type == 'EX-99.CERT':
+                mapping_dict = dict_ex99cert_html
+            elif self.type == 'SC 13E3':
+                mapping_dict = dict_sc13e3_html
+            elif self.type == 'SC 14D9':
+                mapping_dict = dict_sc14d9_html
+            elif self.type == 'SP 15D2':
+                mapping_dict = dict_sp15d2_html
+
             elif self.type == 'SD':
                 mapping_dict = dict_sd_html
+
+            elif self.type == 'T-3':
+                mapping_dict = dict_t3_html
             elif self.type in ['NT 10-K', 'NT 10-Q','NT 20-F']:
                 mapping_dict = dict_nt10k_html
             

datamule/mapping_dicts/html_mapping_dicts.py

@@ -26,6 +26,12 @@ dict_abs15g_html = {
 dict_nt10k_html = {
     ('part',r'^part\s*([ivx]+)') : 0,
 }
+dict_nt10q_html = dict_nt10k_html
+dict_nt20f_html = dict_nt10k_html
+dict_ntncen_html = dict_nt10k_html
+dict_ntncsr_html = dict_nt10k_html
+dict_ntfcen_html = dict_nt10k_html
+dict_ntfncsr_html = dict_nt10k_html
 
 dict_1kpartii_html = {
     ('item',r'^item\s*(\d+)') : 0,
@@ -72,4 +78,44 @@ dict_appntc_html = {('agency',r'^agency') : 0,
 dict_cb_html = {
     ('part', r'^part\s*([ivx]+)') : 0,
     ('item', r'^item\s*(\d+)') : 1,
-}
+}
+
+dict_dstrbrpt_html = dict_1kpartii_html
+
+dict_n18f1_html = {
+    ('notification of election', r'^notification of election') : 0,
+    ('signatures', r'^signatures?\.*$') : 0,
+}
+
+dict_ex99cert_html = {
+    ('item',r'^(\d+)') : 0,
+    ('letter',r'^\(?([a-z])') : 1,
+}
+
+dict_ncsrs_html = {
+    ('item',r'^(\d+)') : 0,
+    ('signatures',r'^signatures?\.*$') : 0,
+}
+
+dict_sc13e3_html = {
+    ('item', r'^item\s*(\d+)') : 0,
+    ('signatures', r'^signatures?\.*$') : 0,
+    ('letter', r'^\(?([a-z])') : 1,
+}
+
+dict_sc14d9_html = {
+    ('item', r'^item\s*(\d+)') : 0,
+    ('signatures', r'^signatures?\.*$') : 0,
+    ('annex', r'^annex') : 0,
+}
+
+dict_sp15d2_html = dict_10k_html
+
+dict_t3_html = {('general',r'^general'):0,
+                ('affiliations',r'^affiliations'):0,
+                ('management and control',r'^management and control'):0,
+                ('underwriters',r'^underwriters'):0,
+                ('capital securities',r'^capital securities'):0,
+                ('indenture securities',r'^indenture securities'):0,
+                ('signatures',r'^signatures?\.*$') : 0,
+                ('number',r'^(\d+)') : 1,}

datamule/sec/submissions/downloader.py

@@ -9,7 +9,7 @@ async def download_callback(hit, content, cik, accno, url, output_dir="filings",
     try:
         # Create a Submission object directly from the content
         # Note: the content needs to be decoded from bytes to string for the parser
-        submission = Submission(sgml_content=content.decode('utf-8', errors='replace'), 
+        submission = Submission(sgml_content=content, 
                                keep_document_types=keep_document_types)
         
         # Use the async save method to write the submission to disk

datamule/seclibrary/downloader.py

@@ -99,6 +99,7 @@ class Downloader:
                 self.downloader._run_coroutine(submission.save_async(output_dir=self.output_dir))
                 self.pbar.update(1)
             except Exception as e:
+                print(f"Exception {e} in {filename}")
                 accession_dir = os.path.join(self.output_dir, filename.split('.')[0])  
                 if os.path.exists(accession_dir):
                     shutil.rmtree(accession_dir)
@@ -143,7 +144,7 @@ class Downloader:
             with dctx.stream_reader(input_buffer) as reader:
                 shutil.copyfileobj(reader, decompressed_content)
                 
-            content = decompressed_content.getvalue().decode('utf-8')
+            content = decompressed_content.getvalue()
             processor.processing_queue.put((filename, content))
             return True
                 
@@ -159,7 +160,7 @@ class Downloader:
 
     def save_regular_file(self, chunks, filename, output_dir, processor):
         try:
-            content = b''.join(chunks).decode('utf-8')
+            content = b''.join(chunks)
             processor.processing_queue.put((filename, content))
             return True
                 

datamule/submission.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 import json
 from .document.document import Document
-from secsgml import parse_sgml_submission_into_memory
+from secsgml import parse_sgml_content_into_memory
 import os
 import aiofiles
 import tempfile
@@ -79,9 +79,8 @@ class Submission:
         
         if sgml_content is not None:
             self.path = None
-            metadata, raw_documents = parse_sgml_submission_into_memory(sgml_content)
+            metadata, raw_documents = parse_sgml_content_into_memory(sgml_content)
             self.metadata = Document(type='submission_metadata', content=metadata, extension='.json',filing_date=None,accession=None,path=None)
-
             # code dupe
             self.accession = self.metadata.content['accession-number']
             self.filing_date= f"{self.metadata.content['filing-date'][:4]}-{self.metadata.content['filing-date'][4:6]}-{self.metadata.content['filing-date'][6:8]}"
@@ -95,7 +94,9 @@ class Submission:
                 # Keep only specified types
                 if keep_document_types is not None and type not in keep_document_types:
                     continue
-                filename = doc.get('filename')
+
+                # write as txt if not declared
+                filename = doc.get('filename','.txt')
                 extension = Path(filename).suffix
                 self.documents.append(Document(type=type, content=raw_documents[idx], extension=extension,filing_date=self.filing_date,accession=self.accession))
 
@@ -190,12 +191,9 @@ class Submission:
             json.dump(self.metadata.content, f, indent=4)
         
         for idx, doc in enumerate(self.metadata.content['documents']):
-            try:
-                filename = doc.get('filename')
-                if filename is None:
-                    filename = f"{doc.get('sequence', idx)}.txt"
-            except (KeyError, IndexError):
-                filename = f"{idx}.txt"
+            filename = doc.get('filename')
+            if filename is None:
+                filename = f"{doc.get('sequence')}.txt"
             
             doc_path = file_dir / filename
             
@@ -231,12 +229,11 @@ class Submission:
             await f.write(json.dumps(self.metadata.content, indent=4))
         
         for idx, doc in enumerate(self.metadata.content['documents']):
-            try:
-                filename = doc.get('filename')
-                if filename is None:
-                    filename = f"{doc.get('sequence', idx)}.txt"
-            except (KeyError, IndexError):
-                filename = f"{idx}.txt"
+            filename = doc.get('filename')
+            # oh we need handling here for sequences case
+            if filename is None:
+                filename = doc['sequence'] + '.txt'
+
             
             doc_path = file_dir / filename
             

{datamule-1.4.4.dist-info → datamule-1.4.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.4.4
+Version: 1.4.5
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-1.4.4.dist-info → datamule-1.4.5.dist-info}/RECORD

@@ -5,10 +5,10 @@ datamule/index.py,sha256=_7Ox5hyF_7RWdblVFr5rNyv_ARwBP7VY4f703pk9qQ8,2074
 datamule/package_updater.py,sha256=Z9zaa_y0Z5cknpRn8oPea3gg4kquFHfpfhduKKCZ6NU,958
 datamule/portfolio.py,sha256=8fiK-vfZM5-NJSvOEsDR2YDb-2njjzFk6l7BiRyrzOM,7168
 datamule/sheet.py,sha256=TvFqK9eAYuVoJ2uWdAlx5EN6vS9lke-aZf7FqtUiDBc,22304
-datamule/submission.py,sha256=Yh5nG3ioumhl6z30wJdIEmKjDDNSuo0r2xycZSIaeIg,11035
+datamule/submission.py,sha256=EtWdEnAyWLZdu69Dyzbs4qb5YL41HlExFGMjwEoMhsg,10904
 datamule/data/listed_filer_metadata.csv,sha256=dT9fQ8AC5P1-Udf_UF0ZkdXJ88jNxJb_tuhi5YYL1rc,2426827
 datamule/document/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/document/document.py,sha256=RQkeBl9VxyjP2_3BT2GyCiwuFrQM-YlJlqMq3SdeIdU,12438
+datamule/document/document.py,sha256=3vX850H7rZH4H8BysitZDaLhT6WPJuIreoV1PSjACno,14301
 datamule/document/processing.py,sha256=jDCEzBFDSQtq7nQxRScIsbALnFcvMPOkNkMUCa7mFxg,31921
 datamule/document/table.py,sha256=73yUJKY82ap32jhLmZeTti-jQ_lyhcJGlGwyxLtgYOg,12944
 datamule/document/mappings/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -36,7 +36,7 @@ datamule/document/mappings/thirteenfhr.py,sha256=XpYRIMPZnGLfEE4TqBI0BPXbyuq0xf3
 datamule/document/mappings/twentyfivense.py,sha256=lKyj0ZBhkHX9gQJMTUPrQlxYFg3k-aBnWqtoS5bujZM,905
 datamule/document/mappings/twentyfourf2nt.py,sha256=Q7RPT3JgJHjYdjMuaSyAxclt6QPT_LgCQloxp-ByDuI,4118
 datamule/mapping_dicts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/mapping_dicts/html_mapping_dicts.py,sha256=bh3aM0IXKA6v_LhFdO_6akUN7sfp8mhXydeDyVe1ht4,1985
+datamule/mapping_dicts/html_mapping_dicts.py,sha256=rf2cNLhNRuDpYVX_gNFAfR7fbjLb_Wbo114EScj53RU,3321
 datamule/mapping_dicts/txt_mapping_dicts.py,sha256=DQPrGYbAPQxomRUtt4iiMGrwuF7BHc_LeFBQuYBzU9o,6311
 datamule/mapping_dicts/xml_mapping_dicts.py,sha256=Z22yDVwKYonUfM5foQP00dVDE8EHhhMKp0CLqVKV5OI,438
 datamule/sec/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -44,7 +44,7 @@ datamule/sec/utils.py,sha256=JUxwijJiqRMnRJNQzVUamyF5h9ZGc7RnO_zsLOIM73g,2079
 datamule/sec/infrastructure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/sec/infrastructure/submissions_metadata.py,sha256=f1KarzFSryKm0EV8DCDNsBw5Jv0Tx5aljiGUJkk7DRk,18745
 datamule/sec/submissions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/sec/submissions/downloader.py,sha256=60wX2Yml1UCuxOtU0xMxqqeyHhrypCmlDQ0jZF-StJo,2665
+datamule/sec/submissions/downloader.py,sha256=izaz559PtBCAWPWGzqUReloawJtXwnraclgXdzEOteI,2631
 datamule/sec/submissions/eftsquery.py,sha256=mSZon8rlW8dxma7M49ZW5V02Fn-ENOdt9TNO6elBrhE,27983
 datamule/sec/submissions/monitor.py,sha256=dZYuVCi_X82eYA8l_9cbnkRjiawz3K4U-FnCAyJcgk4,7892
 datamule/sec/submissions/streamer.py,sha256=EXyWNCD9N6mZmvm9lFSCFodF19zSQ8jfIbWPZNp0K5Y,11253
@@ -56,9 +56,9 @@ datamule/sec/xbrl/streamcompanyfacts.py,sha256=Qq88PqW5_j1k3Aqrl0KRmKeF54D6Wbb6H
 datamule/sec/xbrl/xbrlmonitor.py,sha256=TKFVfSyyUUfUgFQw4WxEVs4g8Nh-2C0tygNIRmTqW3Y,5848
 datamule/seclibrary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/seclibrary/bq.py,sha256=C8sb_rpXTvchprrFLcbRar4Qi0XWW25tnv1YsHSS5o4,18025
-datamule/seclibrary/downloader.py,sha256=PIgz_7ASUTZOHcUZGcD1SmLaGSbq7xe7EiJT0Z7HU4M,13653
+datamule/seclibrary/downloader.py,sha256=VIdaQq5wDcYWnqrv9t8J7z0KtdNRGK8ahfBsgvTfdQQ,13675
 datamule/seclibrary/query.py,sha256=qGuursTERRbOGfoDcYcpo4oWkW3PCBW6x1Qf1Puiak4,7352
-datamule-1.4.4.dist-info/METADATA,sha256=-_m15CLGbd7csE_aiXo9Q54wMCqFjW0VfN1QJKplQtY,469
-datamule-1.4.4.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-datamule-1.4.4.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-1.4.4.dist-info/RECORD,,
+datamule-1.4.5.dist-info/METADATA,sha256=oKn8LWnuOVozqGyVe7ggRcCmoyjy7vwZbgZG7AKYWGY,469
+datamule-1.4.5.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+datamule-1.4.5.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-1.4.5.dist-info/RECORD,,