datamule 0.416__cp39-cp39-win_amd64.whl → 0.418__cp39-cp39-win_amd64.whl

datamule/parser/document_parsing/sgml_parser.py

@@ -0,0 +1,94 @@
+import os
+import json
+import uu
+from io import BytesIO
+
+class SimpleSGMLParser:
+    def _extract_tag_content(self, line: str) -> tuple[str, str] | None:
+        if not (line.startswith('<') and '>' in line):
+            return None
+            
+        tag_end = line.index('>')
+        tag = line[1:tag_end]
+        
+        if tag.startswith('/'):
+            return None
+        
+        content = line[tag_end + 1:].strip()
+        return (tag, content)
+
+    def _write_document(self, content: str, document_info: dict, output_dir: str) -> None:
+        if not content:
+            return
+
+        output_path = os.path.join(output_dir, document_info.get('FILENAME', f"{document_info.get('SEQUENCE', 'unknown')}.txt"))
+        
+        first_line = content.partition('\n')[0].strip()
+        if first_line.startswith('begin '):
+            with BytesIO(content.encode()) as input_file:
+                uu.decode(input_file, output_path,quiet=True)
+        else:
+            with open(output_path, 'w', encoding='utf-8') as f:
+                f.write(content)
+
+    def parse_file(self, filepath: str, output_dir: str) -> None:
+        os.makedirs(output_dir, exist_ok=True)
+        
+        submission_data = {}
+        documents = []
+        current_document = {}
+        text_buffer = []
+        
+        in_document = False
+        in_text = False
+        in_submission = True
+        
+        with open(filepath, 'r', encoding='utf-8') as file:
+            for line in file:
+                stripped = line.strip()
+                
+                if stripped == '<DOCUMENT>':
+                    in_document = True
+                    in_submission = False
+                    current_document = {}
+                    
+                elif stripped == '</DOCUMENT>':
+                    documents.append(current_document)
+                    self._write_document(''.join(text_buffer), current_document, output_dir)
+                    text_buffer = []
+                    in_document = False
+                    
+                elif stripped == '<TEXT>':
+                    in_text = True
+                    text_buffer = []
+                    
+                elif stripped == '</TEXT>':
+                    in_text = False
+                    
+                elif in_text:
+                    if stripped not in ['<PDF>', '</PDF>']:
+                        text_buffer.append(line)
+                        
+                else:
+                    tag_content = self._extract_tag_content(stripped)
+                    if tag_content:
+                        key, value = tag_content
+                        if in_submission:
+                            submission_data[key] = value
+                        elif in_document:
+                            current_document[key] = value
+        
+        metadata = {
+            'submission': submission_data,
+            'documents': documents
+        }
+        
+        with open(os.path.join(output_dir, 'metadata.json'), 'w', encoding='utf-8') as f:
+            json.dump(metadata, f, indent=4)
+
+def parse_sgml_submission(filepath: str, output_dir: str | None = None) -> None:
+    if output_dir is None:
+        output_dir = os.path.splitext(filepath)[0] + '_output'
+        
+    parser = SimpleSGMLParser()
+    parser.parse_file(filepath, output_dir)

{datamule-0.416.dist-info → datamule-0.418.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 0.416
+Version: 0.418
 Summary: Making it easier to use SEC filings.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman
@@ -18,9 +18,9 @@ Requires-Dist: selectolax
 Requires-Dist: pytz
 Requires-Dist: zstandard
 Provides-Extra: all
+Requires-Dist: openai; extra == "all"
 Requires-Dist: flask; extra == "all"
 Requires-Dist: psutil; extra == "all"
-Requires-Dist: openai; extra == "all"
 Requires-Dist: google-generativeai; extra == "all"
 Requires-Dist: pandas; extra == "all"
 Provides-Extra: dataset_builder

{datamule-0.416.dist-info → datamule-0.418.dist-info}/RECORD

@@ -1,4 +1,4 @@
-datamule/__init__.py,sha256=c5Ozl19hzvyMqG75Zl3XFHIBQihxfB47uoTm7EfOOTQ,2033
+datamule/__init__.py,sha256=Li3iau_u87wQQhoPliSTTpGaf3OMf5jIvqtHFJmCvnw,2338
 datamule/document.py,sha256=6xEaI-32AQiBxX3gZcX4Qr49bgvcvLviFAwUGpTwtr0,5273
 datamule/helper.py,sha256=8HOjB3Y7svw_zjEY-AY5JKOJ-LrBiuQMPyok3MH6CCg,4716
 datamule/monitor.py,sha256=WVds1HGV_ojYgWmo0b4Dsiv9mzZ85HHnCucH-7XoUw8,9350
@@ -11,6 +11,8 @@ datamule/data/company_tickers.csv,sha256=ihU6aNFriN0lADloCO85Op04deFk3qVcLZ0EJhi
 datamule/data/sec-glossary.csv,sha256=TPjTBVM3kyFd8xHsmihykepvKbuLAAthOfEDjh_H-Kk,251937
 datamule/data/xbrl_descriptions.csv,sha256=Hg9BOo9zSjR7Khvx0pikILcbmDK_A404dmQtWuESK4s,2631548
 datamule/dataset_builder/dataset_builder.py,sha256=h1JDzLcMKxxMcXcD24EyqjUPp78iLWCovfNLQtpwZi4,10005
+datamule/downloader/downloader.py,sha256=D95tUD-y7xmk8JatgkzZnyIouH6AMwUF78nlgOY-FM8,14872
+datamule/downloader/premiumdownloader.py,sha256=jLr9jWDUrOQQXagbpbqc9rc2H_od6tDhyKnmNKABids,14537
 datamule/mulebot/__init__.py,sha256=GM5cTnijSSLO9GXFdsCuz5B1iwGUcxDbpoBQ6zw1Odo,30
 datamule/mulebot/helper.py,sha256=Hzzr2HReHpFe2GfpVU79EXvQFx3oL9UiwkJp--Sd1N4,1114
 datamule/mulebot/mulebot.py,sha256=wN0Tv5fvarXgk1LRCcgPhj8Xgd8uYFn-cMucxTxRsEo,5966
@@ -29,9 +31,25 @@ datamule/mulebot/mulebot_server/static/scripts/suggestions.js,sha256=vqFoHG0z0YI
 datamule/mulebot/mulebot_server/static/scripts/tableArtifacts.js,sha256=jelibkbM9YlrlU7_-kaVisMh8OjIKWZpJQ9_FiHV4OQ,4565
 datamule/mulebot/mulebot_server/static/scripts/utils.js,sha256=pOzsJ6bGxakN6G5qUvmkZd34u0GRHFxvH3pKpUmnhMc,1282
 datamule/mulebot/mulebot_server/templates/chat-minimalist.html,sha256=mm-LomSoNcJImsFwLpFWt3kIFnxUzIPmUZ1Dt5BFN0s,7065
+datamule/parser/document_parsing/basic_10k_parser.py,sha256=z1ZP8L8ZOfcpo6S3jmS3aQvefOurqS4u6oy9bh1UBHA,3137
+datamule/parser/document_parsing/basic_10q_parser.py,sha256=kZWGljvC0TYLh4eWl4JQyl4XYto8X6NN6YIW-_iffqA,2695
+datamule/parser/document_parsing/basic_13d_parser.py,sha256=1nxBi9KFJuzw7CeEXpJFDzYVUd6rj4eCYiKHsGjK2b4,1585
+datamule/parser/document_parsing/basic_13g_parser.py,sha256=H9_MuOgkYVTIGwhj9w_WIpfMDdgakE_qs0Y3fvaUj_E,1691
+datamule/parser/document_parsing/basic_8k_parser.py,sha256=fzf8q9LOpBMHGWw-sfqUq3pyFZBlw47nLJBQWPhtGGg,2549
+datamule/parser/document_parsing/company_concepts_parser.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datamule/parser/document_parsing/form_d_parser.py,sha256=NTAfC8W3i2y7aIofXoLlAbY-4F6QVELYfIPIrVErjVY,2105
+datamule/parser/document_parsing/generalized_item_parser.py,sha256=M2bmYivSXe0POyBtDlPMykyyCgG8n1egRpJuZtZTR_g,2694
+datamule/parser/document_parsing/generalized_xml_parser.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datamule/parser/document_parsing/helper.py,sha256=wEhqx70CQ0SyEfmOfSCcNsf2TNTtomZvTbmoOhwG3kk,2607
+datamule/parser/document_parsing/information_table_parser_13fhr.py,sha256=vCIEqxOoBf1lnOKTJAifnPEXihtSPZcwRFW8yWhEZLc,1808
+datamule/parser/document_parsing/insider_trading_parser.py,sha256=IrH5a3qikPjC1GFoYzapfYW_CArbK0FkbJLbWSfsYNQ,7264
+datamule/parser/document_parsing/mappings.py,sha256=dq6EjaxxDHjH-sg62adRwJOf1v736QiLwXavOHs2vy8,5380
+datamule/parser/document_parsing/n_port_p_parser.py,sha256=T6GliMm-TETPsFM-hDKt1BkMnSenvDeThj50RsIip50,2293
+datamule/parser/document_parsing/sec_parser.py,sha256=YewOdOsi0P25teQuxS5DNEND9ZCyxE2ewK1DoP9mPto,2788
+datamule/parser/document_parsing/sgml_parser.py,sha256=ASpe1SzgPj4qk0VOmmuMiEQeatjcwZzsuO3MvsYCHhc,3410
 datamule/parser/sgml_parsing/sgml_parser_cy.c,sha256=vxLnjpUgZ5LLvBvzYI_CZVxjpgRpulnzj3EFQG5eB8g,797203
-datamule/parser/sgml_parsing/sgml_parser_cy.cp39-win_amd64.pyd,sha256=O1R8LEUOW-gUr4aYjvhOD7EoSkJeGlKabUZ3cAmVPLM,142848
-datamule-0.416.dist-info/METADATA,sha256=V6pjBZ8Psta5fMACOBkq7psulhTOuWoWY7XlFaS83DM,1087
-datamule-0.416.dist-info/WHEEL,sha256=yA7mxgqX2UV73NtJdMh2AAmdb628loM81912H3s5r00,100
-datamule-0.416.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-0.416.dist-info/RECORD,,
+datamule/parser/sgml_parsing/sgml_parser_cy.cp39-win_amd64.pyd,sha256=Ts6VLobOSehNoSITBlRx0iisf5uGWuzxyJ8ljw54Fvk,142848
+datamule-0.418.dist-info/METADATA,sha256=H3rV4RWQImJgtSxpaW0-dGnnR7-mbl-_5iUKNQIv49k,1087
+datamule-0.418.dist-info/WHEEL,sha256=yA7mxgqX2UV73NtJdMh2AAmdb628loM81912H3s5r00,100
+datamule-0.418.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-0.418.dist-info/RECORD,,