datamule 0.416__cp310-cp310-macosx_10_9_universal2.whl → 0.418__cp310-cp310-macosx_10_9_universal2.whl

datamule/parser/document_parsing/sgml_parser.py

@@ -0,0 +1,94 @@
+import os
+import json
+import uu
+from io import BytesIO
+
+class SimpleSGMLParser:
+    def _extract_tag_content(self, line: str) -> tuple[str, str] | None:
+        if not (line.startswith('<') and '>' in line):
+            return None
+            
+        tag_end = line.index('>')
+        tag = line[1:tag_end]
+        
+        if tag.startswith('/'):
+            return None
+        
+        content = line[tag_end + 1:].strip()
+        return (tag, content)
+
+    def _write_document(self, content: str, document_info: dict, output_dir: str) -> None:
+        if not content:
+            return
+
+        output_path = os.path.join(output_dir, document_info.get('FILENAME', f"{document_info.get('SEQUENCE', 'unknown')}.txt"))
+        
+        first_line = content.partition('\n')[0].strip()
+        if first_line.startswith('begin '):
+            with BytesIO(content.encode()) as input_file:
+                uu.decode(input_file, output_path,quiet=True)
+        else:
+            with open(output_path, 'w', encoding='utf-8') as f:
+                f.write(content)
+
+    def parse_file(self, filepath: str, output_dir: str) -> None:
+        os.makedirs(output_dir, exist_ok=True)
+        
+        submission_data = {}
+        documents = []
+        current_document = {}
+        text_buffer = []
+        
+        in_document = False
+        in_text = False
+        in_submission = True
+        
+        with open(filepath, 'r', encoding='utf-8') as file:
+            for line in file:
+                stripped = line.strip()
+                
+                if stripped == '<DOCUMENT>':
+                    in_document = True
+                    in_submission = False
+                    current_document = {}
+                    
+                elif stripped == '</DOCUMENT>':
+                    documents.append(current_document)
+                    self._write_document(''.join(text_buffer), current_document, output_dir)
+                    text_buffer = []
+                    in_document = False
+                    
+                elif stripped == '<TEXT>':
+                    in_text = True
+                    text_buffer = []
+                    
+                elif stripped == '</TEXT>':
+                    in_text = False
+                    
+                elif in_text:
+                    if stripped not in ['<PDF>', '</PDF>']:
+                        text_buffer.append(line)
+                        
+                else:
+                    tag_content = self._extract_tag_content(stripped)
+                    if tag_content:
+                        key, value = tag_content
+                        if in_submission:
+                            submission_data[key] = value
+                        elif in_document:
+                            current_document[key] = value
+        
+        metadata = {
+            'submission': submission_data,
+            'documents': documents
+        }
+        
+        with open(os.path.join(output_dir, 'metadata.json'), 'w', encoding='utf-8') as f:
+            json.dump(metadata, f, indent=4)
+
+def parse_sgml_submission(filepath: str, output_dir: str | None = None) -> None:
+    if output_dir is None:
+        output_dir = os.path.splitext(filepath)[0] + '_output'
+        
+    parser = SimpleSGMLParser()
+    parser.parse_file(filepath, output_dir)

{datamule-0.416.dist-info → datamule-0.418.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 0.416
+Version: 0.418
 Summary: Making it easier to use SEC filings.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman
@@ -16,11 +16,11 @@ Requires-Dist: selectolax
 Requires-Dist: pytz
 Requires-Dist: zstandard
 Provides-Extra: all
-Requires-Dist: pandas; extra == "all"
 Requires-Dist: openai; extra == "all"
-Requires-Dist: google-generativeai; extra == "all"
-Requires-Dist: psutil; extra == "all"
 Requires-Dist: flask; extra == "all"
+Requires-Dist: psutil; extra == "all"
+Requires-Dist: google-generativeai; extra == "all"
+Requires-Dist: pandas; extra == "all"
 Provides-Extra: dataset_builder
 Requires-Dist: pandas; extra == "dataset-builder"
 Requires-Dist: google-generativeai; extra == "dataset-builder"

{datamule-0.416.dist-info → datamule-0.418.dist-info}/RECORD

@@ -1,4 +1,4 @@
-datamule/__init__.py,sha256=8zx1hqAJeTxqmJ43zjJpJl8SRIRBx1zj8RNrsvYrQhE,1972
+datamule/__init__.py,sha256=i3HnWFrqEdsK2OmvE7NUjcO05w1BuIYBV6J4cSFaN3s,2268
 datamule/document.py,sha256=Yn8UqUjKwYPE29MrMjreHK_HY9eTqOSjPyM5B1VBrHQ,5144
 datamule/helper.py,sha256=tr3AQWus9dHNZFKpLSglWjcb8zmm5qDXjOWACMhvMxQ,4594
 datamule/monitor.py,sha256=mRaM8v5NgcMF9DJ1s_YBzucjrbr-3yFwW422MVml-_Q,9114
@@ -11,6 +11,8 @@ datamule/data/company_tickers.csv,sha256=GW6lOP54RiGJCx-d9N5jEBy7tGVgU3zI-5xHJXr
 datamule/data/sec-glossary.csv,sha256=-cN7GjiadLw5C1sv4zSeCnfeZZDYeSgJl-0ydarMAo0,251209
 datamule/data/xbrl_descriptions.csv,sha256=SQ9wUURNqG424rnTiZtopsxV2q-PvU4NMj52LqgDsvg,2621524
 datamule/dataset_builder/dataset_builder.py,sha256=NCvNbDwlEkA_eAbqbsG--YlqPBDREFTVSM1GJquR0RE,9747
+datamule/downloader/downloader.py,sha256=XJF0FfoCB43_22lhPakq9dI-oJ_SsXIvNx4PUVmcEOc,14509
+datamule/downloader/premiumdownloader.py,sha256=JH4aZ-ZwARCIACKwgzSgHAuOkKPc_GnhiUHSSu22XO4,14206
 datamule/mulebot/__init__.py,sha256=YvZXV6xQ0iP-oGD8rloufjdwJL6D46P3NNr0CY9PQCA,29
 datamule/mulebot/helper.py,sha256=olztOwltfELZ-IERM2bRNLBavD04kfB6ueWTisJAleA,1080
 datamule/mulebot/mulebot.py,sha256=XbtgvXBSFu9OaaLW_k1KDgHVTNQGV8_0ZwNMFad-pPU,5837
@@ -29,9 +31,25 @@ datamule/mulebot/mulebot_server/static/scripts/suggestions.js,sha256=TCyz8OYuXeI
 datamule/mulebot/mulebot_server/static/scripts/tableArtifacts.js,sha256=UtkUpLvELNI4Ibpb7VstgVA9Tk-8jbkxXhmXsgufFa4,4437
 datamule/mulebot/mulebot_server/static/scripts/utils.js,sha256=oGPMtyT9dvuqHqrfZj33t4vLZiF8UJrMXB1hpPXRNu4,1255
 datamule/mulebot/mulebot_server/templates/chat-minimalist.html,sha256=MsTbgpnLD0JCQiKKP3XeeNJRNsRqKsRa1j_XXW7nBKw,6975
+datamule/parser/document_parsing/basic_10k_parser.py,sha256=-_q0X9K4KyLccF6j_zNp7FknGPBW1r4U3AT9bPjQUgA,3056
+datamule/parser/document_parsing/basic_10q_parser.py,sha256=ccQc3pwBqevDb6-vBwEE5RTZwRcnrSxjRxZEk_zPO-s,2623
+datamule/parser/document_parsing/basic_13d_parser.py,sha256=loJC97H_ccu_hWMhgNt5tvGZnN3--7tsqZxzBnWB_FY,1528
+datamule/parser/document_parsing/basic_13g_parser.py,sha256=sWg83-QTAzUDNs45iWtpxnMxQgtC3zJlFj0R9ybZpNI,1631
+datamule/parser/document_parsing/basic_8k_parser.py,sha256=inCSmlH_BkLK0Lkvt0kZ6EUJ0nijul_RkdXzccyOmRI,2466
+datamule/parser/document_parsing/company_concepts_parser.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datamule/parser/document_parsing/form_d_parser.py,sha256=dWlGeVZRzh0kfT3gVMC8eyqeQORdVV3r8KXUwEqAW3s,2036
+datamule/parser/document_parsing/generalized_item_parser.py,sha256=67_DFb1BQbMmdHefEgoCPlEoiUT0zyxh3eBNJpjGXUk,2616
+datamule/parser/document_parsing/generalized_xml_parser.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datamule/parser/document_parsing/helper.py,sha256=QPhVxLxMSx6Qdi7sR4D4iPObGoTnVD3tXTCNWzNxStg,2533
+datamule/parser/document_parsing/information_table_parser_13fhr.py,sha256=R4Up1oDx3xAlzHwXzVzUkdOSsk8YPuJBPS_3I_bNQSE,1767
+datamule/parser/document_parsing/insider_trading_parser.py,sha256=OVQDeLcfaZtgmOWvWPDotftO6jxx-doFAqBYVqNgypo,7106
+datamule/parser/document_parsing/mappings.py,sha256=VKdnT3C5yPTbB4ZBa4El4jnB-6_osomm2rbJx6Ac6HE,5286
+datamule/parser/document_parsing/n_port_p_parser.py,sha256=GmmQFkCZt57WikUZ5DahtTYMhhk0VcfkhOJusM4Tkow,2224
+datamule/parser/document_parsing/sec_parser.py,sha256=AS8H4h1sfUAdWP2gotULcjbylsYN_nHgTfkeVRyENPo,2716
+datamule/parser/document_parsing/sgml_parser.py,sha256=tC1cL3cdVQPWbc9QtoRUYSo2wRuYNaglFaCmP57oEfA,3317
 datamule/parser/sgml_parsing/sgml_parser_cy.c,sha256=UwXSuLGSBLKfO5bM7xTzjFLnGDV3-NNnCjgUCPAY1gk,796689
-datamule/parser/sgml_parsing/sgml_parser_cy.cpython-310-darwin.so,sha256=Zs-4Wm-pN-4ddHLBniegMkO2WbCsiJF09hIMcLZbSQU,362232
-datamule-0.416.dist-info/METADATA,sha256=BLgydCvASmb5OR6jlTOoTH06YFx3LH7ygfaZCK7UH2Y,1007
-datamule-0.416.dist-info/WHEEL,sha256=Sooupui5EnBW-HNXjQU0OJoerPN0TT-a4xxlvmpUy2g,115
-datamule-0.416.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-0.416.dist-info/RECORD,,
+datamule/parser/sgml_parsing/sgml_parser_cy.cpython-310-darwin.so,sha256=r7PGnO6cbHnXO1BwDGGuOX6m9qyNtJcPeYpFm1M8t3o,362232
+datamule-0.418.dist-info/METADATA,sha256=3s-xzrQ7TOuColC022WoYWoC2IOntbvhhhqlztGWoB8,1007
+datamule-0.418.dist-info/WHEEL,sha256=Sooupui5EnBW-HNXjQU0OJoerPN0TT-a4xxlvmpUy2g,115
+datamule-0.418.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-0.418.dist-info/RECORD,,