datamule 1.2.5__py3-none-any.whl → 1.2.9__py3-none-any.whl

datamule/submission.py

@@ -4,6 +4,70 @@ from .document.document import Document
 from secsgml import parse_sgml_submission_into_memory
 import os
 import aiofiles
+import tempfile
+
+
+# # NEW CODE YAY. probably will remove
+
+# def save_metadata_atomically(metadata_file_path, metadata_content):
+#     """Save metadata to a JSONL file atomically, works on any filesystem"""
+    
+#     # Create directory if it doesn't exist
+#     os.makedirs(os.path.dirname(metadata_file_path), exist_ok=True)
+    
+#     # Format the JSON with newline
+#     json_str = json.dumps(metadata_content, indent=4) + "\n"
+    
+#     # Write complete content to a temporary file first
+#     fd, temp_path = tempfile.mkstemp(dir=os.path.dirname(metadata_file_path))
+#     try:
+#         with os.fdopen(fd, 'w') as temp_file:
+#             temp_file.write(json_str)
+#             temp_file.flush()
+#             os.fsync(temp_file.fileno())  # Force write to disk
+        
+#         # Append the temporary file to the main file
+#         with open(metadata_file_path, 'a') as target_file:
+#             with open(temp_path, 'r') as temp_read:
+#                 content = temp_read.read()
+#                 target_file.write(content)
+#                 target_file.flush()
+#                 os.fsync(target_file.fileno())  # Force write to disk
+#     finally:
+#         # Clean up the temporary file
+#         if os.path.exists(temp_path):
+#             os.unlink(temp_path)
+
+# async def save_metadata_atomically_async(metadata_file_path, metadata_content):
+#     """Save metadata to a JSONL file atomically in async mode"""
+    
+#     # Create directory if it doesn't exist
+#     os.makedirs(os.path.dirname(metadata_file_path), exist_ok=True)
+    
+#     # Format the JSON with newline
+#     json_str = json.dumps(metadata_content, indent=4) + "\n"
+    
+#     # Write to a temporary file first
+#     fd, temp_path = tempfile.mkstemp(dir=os.path.dirname(metadata_file_path))
+#     os.close(fd)  # Close the file descriptor
+    
+#     try:
+#         async with aiofiles.open(temp_path, 'w') as temp_file:
+#             await temp_file.write(json_str)
+#             await temp_file.flush()
+        
+#         # Append the temporary file to the main file
+#         async with aiofiles.open(metadata_file_path, 'a') as target_file:
+#             async with aiofiles.open(temp_path, 'r') as temp_read:
+#                 content = await temp_read.read()
+#                 await target_file.write(content)
+#                 await target_file.flush()
+#     finally:
+#         # Clean up the temporary file
+#         if os.path.exists(temp_path):
+#             os.unlink(temp_path)
+
+# # END OF NEW CODE
 
 
 class Submission:
@@ -15,16 +79,17 @@ class Submission:
         
         if sgml_content is not None:
             self.path = None
-            self.metadata, raw_documents = parse_sgml_submission_into_memory(sgml_content)
+            metadata, raw_documents = parse_sgml_submission_into_memory(sgml_content)
+            self.metadata = Document(type='submission_metadata', content=metadata, extension='.json',filing_date=None,accession=None,path=None)
 
             # code dupe
-            self.accession = self.metadata['accession-number']
-            self.filing_date= f"{self.metadata['filing-date'][:4]}-{self.metadata['filing-date'][4:6]}-{self.metadata['filing-date'][6:8]}"
+            self.accession = self.metadata.content['accession-number']
+            self.filing_date= f"{self.metadata.content['filing-date'][:4]}-{self.metadata.content['filing-date'][4:6]}-{self.metadata.content['filing-date'][6:8]}"
     
             self.documents = []
             filtered_metadata_documents = []
 
-            for idx,doc in enumerate(self.metadata['documents']):
+            for idx,doc in enumerate(self.metadata.content['documents']):
                 type = doc.get('type')
                 
                 # Keep only specified types
@@ -36,17 +101,18 @@ class Submission:
 
                 filtered_metadata_documents.append(doc)
             
-            self.metadata['documents'] = filtered_metadata_documents
+            self.metadata.content['documents'] = filtered_metadata_documents
 
         if path is not None:
             self.path = Path(path)  
             metadata_path = self.path / 'metadata.json'
             with metadata_path.open('r') as f:
-                self.metadata = json.load(f)
+                metadata = json.load(f) 
+            self.metadata = Document(type='submission_metadata', content=metadata, extension='.json',filing_date=None,accession=None,path=metadata_path)
 
             # Code dupe
-            self.accession = self.metadata['accession-number']
-            self.filing_date= f"{self.metadata['filing-date'][:4]}-{self.metadata['filing-date'][4:6]}-{self.metadata['filing-date'][6:8]}"
+            self.accession = self.metadata.content['accession-number']
+            self.filing_date= f"{self.metadata.content['filing-date'][:4]}-{self.metadata.content['filing-date'][4:6]}-{self.metadata.content['filing-date'][6:8]}"
     
 
 
@@ -58,7 +124,7 @@ class Submission:
         else:
             document_types = document_type
 
-        for idx,doc in enumerate(self.metadata['documents']):
+        for idx,doc in enumerate(self.metadata.content['documents']):
             if doc['type'] in document_types:
                 
                 # if loaded from path
@@ -84,7 +150,7 @@ class Submission:
 
     
     def __iter__(self):
-        for idx,doc in enumerate(self.metadata['documents']):
+        for idx,doc in enumerate(self.metadata.content['documents']):
             # if loaded from path
             if self.path is not None:
                 filename = doc.get('filename')
@@ -121,9 +187,9 @@ class Submission:
         
         metadata_path = file_dir / "metadata.json"
         with open(metadata_path, 'w') as f:
-            json.dump(self.metadata, f, indent=4)
+            json.dump(self.metadata.content, f, indent=4)
         
-        for idx, doc in enumerate(self.metadata['documents']):
+        for idx, doc in enumerate(self.metadata.content['documents']):
             try:
                 filename = doc.get('filename')
                 if filename is None:
@@ -162,9 +228,9 @@ class Submission:
         
         metadata_path = file_dir / "metadata.json"
         async with aiofiles.open(metadata_path, 'w') as f:
-            await f.write(json.dumps(self.metadata, indent=4))
+            await f.write(json.dumps(self.metadata.content, indent=4))
         
-        for idx, doc in enumerate(self.metadata['documents']):
+        for idx, doc in enumerate(self.metadata.content['documents']):
             try:
                 filename = doc.get('filename')
                 if filename is None:

{datamule-1.2.5.dist-info → datamule-1.2.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.2.5
+Version: 1.2.9
 Summary: Making it easier to use SEC filings.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman
@@ -10,7 +10,6 @@ Requires-Dist: tqdm
 Requires-Dist: requests
 Requires-Dist: nest-asyncio
 Requires-Dist: aiofiles
-Requires-Dist: polars
 Requires-Dist: setuptools
 Requires-Dist: selectolax
 Requires-Dist: pytz

{datamule-1.2.5.dist-info → datamule-1.2.9.dist-info}/RECORD

@@ -1,35 +1,38 @@
-datamule/__init__.py,sha256=8KioESb9y0Xwy72WuTfsYZnnMFdCrRhSv8DW-kZ4-To,1066
+datamule/__init__.py,sha256=glzwBeGJEE6-TG7mRule9GH6L59XaIRR9T7ALcdpMus,1067
 datamule/config.py,sha256=Y--CVv7JcgrjJkMOSLrvm2S8B9ost6RMSkGviP-MKtg,883
-datamule/helper.py,sha256=xgOVnea-lUlQ5I-U0vYUp0VeKPNZehNhqjJvegA3lYE,3342
-datamule/index.py,sha256=0txvbzPcvY1GsdxA-wGdLzAByxSeE_1VyyBp9mZEQRM,2292
+datamule/helper.py,sha256=g9Kb1DWbViCoON06PjOkSX5Ucu0uG7zPwhsO2LQ6C1g,3579
+datamule/index.py,sha256=_7Ox5hyF_7RWdblVFr5rNyv_ARwBP7VY4f703pk9qQ8,2074
 datamule/package_updater.py,sha256=Z9zaa_y0Z5cknpRn8oPea3gg4kquFHfpfhduKKCZ6NU,958
-datamule/portfolio.py,sha256=zn7Ib29UU_HD4JxPiIRT3v5QgWxmPh4Q1l0VH0aJzUU,7104
+datamule/portfolio.py,sha256=8fiK-vfZM5-NJSvOEsDR2YDb-2njjzFk6l7BiRyrzOM,7168
 datamule/sheet.py,sha256=TvFqK9eAYuVoJ2uWdAlx5EN6vS9lke-aZf7FqtUiDBc,22304
-datamule/submission.py,sha256=HXuFL6snLevGk7DGlvPbjcBOJuccAIxEPXnkA1TXX8Y,8121
+datamule/submission.py,sha256=Yh5nG3ioumhl6z30wJdIEmKjDDNSuo0r2xycZSIaeIg,11035
 datamule/document/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/document/document.py,sha256=eckqSvORnYp9KbZLTQAAjvsftATcIGtYIsFxQHI7dtQ,9859
-datamule/document/processing.py,sha256=fw-1OWfbmZhG1R8XpJx_vcGwz3_djmk0FrblHAMPmwc,27476
-datamule/document/table.py,sha256=Sv9jTGiVhnWIY9nHaynUUixwbCrvbLsf0fdOnFR-NCY,10791
+datamule/document/document.py,sha256=menUFoeWwiY0rJnBkQiqY4NWnO0J17-qs8jFvO_1jiY,9969
+datamule/document/processing.py,sha256=jDCEzBFDSQtq7nQxRScIsbALnFcvMPOkNkMUCa7mFxg,31921
+datamule/document/table.py,sha256=73yUJKY82ap32jhLmZeTti-jQ_lyhcJGlGwyxLtgYOg,12944
 datamule/document/mappings/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/document/mappings/atsn.py,sha256=qkZGNIhyPC3VTTOjQ8-FSCQIhUy4XeSycUGLShxNVCo,17743
 datamule/document/mappings/cfportal.py,sha256=bR9d6DDY0kJ_HGx_hND2y1PNNkZjemYZ2KdyFAcv760,25257
+datamule/document/mappings/d.py,sha256=ayRK-bTzelNH6fspp-n3gz6RXOrHVx6IjX-TmisrFe4,7714
+datamule/document/mappings/ex102_abs.py,sha256=FdGKvteRh_HsYgILF-8o4R6aSsjYwcaLpJxzdru4FTE,3976
 datamule/document/mappings/ex99a_sdr.py,sha256=PNdj9I0ZhNicPObLelNmjp33EgTwzvukqkBDnwxarE0,19
 datamule/document/mappings/ex99c_sdr.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/document/mappings/ex99g_sdr.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/document/mappings/ex99i_sdr.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/document/mappings/information_table.py,sha256=t8xshuUPvPFbT3pGUiXcSIw2gnfw7rCvbh_aBV9NSQM,1560
+datamule/document/mappings/information_table.py,sha256=6l2Via728I59RS0y9Pit37NoOSAbaT-vclArYxU1vtY,1585
 datamule/document/mappings/nmfp.py,sha256=WuTyM1SkBiiLVAHqFF4DTZ_8AvsIuonT2w7pwYDPTDw,17767
 datamule/document/mappings/npx.py,sha256=xwruBueC09kfWhXV3fNUnQWYwCWrdrhQoVO3cKfPTO4,6556
 datamule/document/mappings/onefourtyfour.py,sha256=_-w9h6wGINGH5pQqQvPrd0cgB5QfCtPG5M40ewf_w8Q,2604
-datamule/document/mappings/ownership.py,sha256=DcOxZW2cPiVD3ra2ojoZX47CYcR-_86O-6Ts8T4oaF4,10214
-datamule/document/mappings/proxy_voting_record.py,sha256=gTjx72HM3ge9dKwOFyADjQjlD55pLdqnGjik6dHehA4,29
+datamule/document/mappings/ownership.py,sha256=piD9vs4WFrB4yvp6c0pT5bibLKXgsM7hpnBUzaY0Xxs,10155
+datamule/document/mappings/proxy_voting_record.py,sha256=tSqLH065EOUq7U80P5GP1JBqipmAiqniPpP3E4adA1A,721
 datamule/document/mappings/sbs.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/document/mappings/sbsef.py,sha256=Zw58rbYcnitynk1mh9g1jDrCfqmFlY60OEjPM6p9iF0,534
 datamule/document/mappings/schedule13.py,sha256=lh9sukpEte514Gid77Nz9zh3uBEFZEemrZ2Uau0qsgk,6295
 datamule/document/mappings/sdr.py,sha256=UekqZId5PFMMWRAJSaPvCpN4c1Hx-SLAQPEN8GW_Gbg,4829
+datamule/document/mappings/submission_metadata.py,sha256=pi1eW-tnoAQ6y3laRI29Op80E9BPqqmcfe45owKYStw,271
 datamule/document/mappings/ta.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/document/mappings/thirteenfhr.py,sha256=QD4WztCebi6CwVPsgH9vQti0KpP257mYdFhNBQTbfwg,86
-datamule/document/mappings/twentyfivense.py,sha256=1IkOk3qODAXqUqO84ZrD-hSqgOJyhSKcncX9aARzl9Y,880
+datamule/document/mappings/thirteenfhr.py,sha256=XpYRIMPZnGLfEE4TqBI0BPXbyuq0xf3hut1fePOF6kU,4250
+datamule/document/mappings/twentyfivense.py,sha256=lKyj0ZBhkHX9gQJMTUPrQlxYFg3k-aBnWqtoS5bujZM,905
 datamule/document/mappings/twentyfourf2nt.py,sha256=Q7RPT3JgJHjYdjMuaSyAxclt6QPT_LgCQloxp-ByDuI,4118
 datamule/mapping_dicts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/mapping_dicts/txt_mapping_dicts.py,sha256=DQPrGYbAPQxomRUtt4iiMGrwuF7BHc_LeFBQuYBzU9o,6311
@@ -38,24 +41,22 @@ datamule/sec/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/sec/utils.py,sha256=JUxwijJiqRMnRJNQzVUamyF5h9ZGc7RnO_zsLOIM73g,2079
 datamule/sec/infrastructure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/sec/infrastructure/submissions_metadata.py,sha256=f1KarzFSryKm0EV8DCDNsBw5Jv0Tx5aljiGUJkk7DRk,18745
-datamule/sec/rss/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/sec/rss/monitor.py,sha256=6r4EYaSlGu6VYErlj9zXJsIMLVie1cfacSZU-ESfuBI,18231
 datamule/sec/submissions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/sec/submissions/downloader.py,sha256=60wX2Yml1UCuxOtU0xMxqqeyHhrypCmlDQ0jZF-StJo,2665
 datamule/sec/submissions/eftsquery.py,sha256=mSZon8rlW8dxma7M49ZW5V02Fn-ENOdt9TNO6elBrhE,27983
-datamule/sec/submissions/monitor.py,sha256=Im2kgnUehhTgyY2Vq3uk07n4Vkj4PjII_SsRDi8ehAE,5384
+datamule/sec/submissions/monitor.py,sha256=s6uknn1dF1EemiI3Hl4nEq3txwK7nYl6wmayuUPYpRs,7844
 datamule/sec/submissions/streamer.py,sha256=EXyWNCD9N6mZmvm9lFSCFodF19zSQ8jfIbWPZNp0K5Y,11253
-datamule/sec/submissions/textsearch.py,sha256=-a5yIrrxxtaK10IJeywFmXuJmSndYL9VKm4SC4I9JAs,5808
+datamule/sec/submissions/textsearch.py,sha256=zEr3NXdhVFL8eMh2jruVXIt7taUZTMdNy2hOAyRM2pA,5706
 datamule/sec/xbrl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/sec/xbrl/downloadcompanyfacts.py,sha256=rMWRiCF9ci_gNZMJ9MC2c_PGEd-yEthawQ0CtVwWTjM,3323
 datamule/sec/xbrl/filter_xbrl.py,sha256=g9OT4zrNS0tiUJeBIwbCs_zMisOBkpFnMR3tV4Tr39Q,1316
-datamule/sec/xbrl/streamcompanyfacts.py,sha256=WyJIwuy5mNMXWpx_IkhFzDMe9MOfQ-vNkWl_JzBzFmc,3323
+datamule/sec/xbrl/streamcompanyfacts.py,sha256=Qq88PqW5_j1k3Aqrl0KRmKeF54D6Wbb6H5N2tbvKUzM,3307
 datamule/sec/xbrl/xbrlmonitor.py,sha256=TKFVfSyyUUfUgFQw4WxEVs4g8Nh-2C0tygNIRmTqW3Y,5848
 datamule/seclibrary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/seclibrary/bq.py,sha256=C8sb_rpXTvchprrFLcbRar4Qi0XWW25tnv1YsHSS5o4,18025
-datamule/seclibrary/downloader.py,sha256=fJztJ_sEfv2oHHbDff07DRlXLmztXnzt3Yvv5YaZgGk,13718
+datamule/seclibrary/downloader.py,sha256=PIgz_7ASUTZOHcUZGcD1SmLaGSbq7xe7EiJT0Z7HU4M,13653
 datamule/seclibrary/query.py,sha256=qGuursTERRbOGfoDcYcpo4oWkW3PCBW6x1Qf1Puiak4,7352
-datamule-1.2.5.dist-info/METADATA,sha256=7sZeCfLgGsMTOqhKrRzmCVrjn2oKnWXHcirDjoXZ-zI,512
-datamule-1.2.5.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-datamule-1.2.5.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-1.2.5.dist-info/RECORD,,
+datamule-1.2.9.dist-info/METADATA,sha256=5bMwIRcARNqP6S1cdPzoIBuu1miiUJUUdWnTXvwtPNk,490
+datamule-1.2.9.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+datamule-1.2.9.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-1.2.9.dist-info/RECORD,,