datamule 2.0.6__py3-none-any.whl → 2.0.8__py3-none-any.whl

datamule/document/document.py

@@ -231,13 +231,13 @@ class Document:
         with open(output_filename, 'w',encoding='utf-8') as f:
             json.dump(self.data, f, indent=2)
 
-    def parse_tables(self):
+    def parse_tables(self,must_exist_in_mapping=True):
         if self.extension != '.xml':
             self._tables = []
         else:
             # Use the property to trigger parsing if needed
             data = self.data
-            tables = Tables(document_type = self.type, accession=self.accession, data=data)
+            tables = Tables(document_type = self.type, accession=self.accession, data=data,must_exist_in_mapping=must_exist_in_mapping)
             self._tables = tables.tables
 
     @property

datamule/document/tables/tables.py

@@ -64,14 +64,14 @@ def seperate_data(tables_dict, data):
     
     return data_list
 
-def apply_mapping(flattened_data, mapping_dict, accession):
+def apply_mapping(flattened_data, mapping_dict, accession, must_exist_in_mapping=False):
     """Apply mapping to flattened data and add accession"""
     
     # Handle case where flattened_data is a list of dictionaries
     if isinstance(flattened_data, list):
         results = []
         for data_dict in flattened_data:
-            results.append(apply_mapping(data_dict, mapping_dict, accession))
+            results.extend(apply_mapping(data_dict, mapping_dict, accession,must_exist_in_mapping))
         return results
     
     # Original logic for single dictionary
@@ -85,10 +85,11 @@ def apply_mapping(flattened_data, mapping_dict, accession):
             ordered_row[new_key] = None
     
     # Add any remaining keys that weren't in the mapping
-    for key, value in flattened_data.items():
-        ordered_row[key] = value
+    if not must_exist_in_mapping:
+        for key, value in flattened_data.items():
+            ordered_row[key] = value
     
-    return ordered_row
+    return [ordered_row]
 
 # should have table type, accession, data
 class Table:
@@ -99,7 +100,7 @@ class Table:
 
 
 class Tables():
-    def __init__(self,document_type,accession,data):
+    def __init__(self,document_type,accession,data,must_exist_in_mapping=True):
         self.document_type = document_type
         self.accession = accession
         self.data = data
@@ -107,9 +108,9 @@ class Tables():
         # to fill in
         self.tables = []
 
-        self.parse_tables()
+        self.parse_tables(must_exist_in_mapping=must_exist_in_mapping)
 
-    def parse_tables(self):
+    def parse_tables(self,must_exist_in_mapping=True):
         # first select dict
 
         try:
@@ -125,5 +126,5 @@ class Tables():
         
         for table_name, flattened_data in data_dicts:
             mapping_dict = tables_dict[table_name]['mapping']
-            mapped_data = apply_mapping(flattened_data, mapping_dict, self.accession)
+            mapped_data = apply_mapping(flattened_data, mapping_dict, self.accession,must_exist_in_mapping)
             self.tables.append(Table(mapped_data, table_name, self.accession))

datamule/submission.py

@@ -250,7 +250,7 @@ class Submission:
     def parse_xbrl(self):
         if self._xbrl:
             return
-        
+
         for idx, doc in enumerate(self.metadata.content['documents']):
             if doc['type'] in ['EX-100.INS','EX-101.INS']:
                 document = self._load_document_by_index(idx)
@@ -310,8 +310,8 @@ class Submission:
                 
                 if xbrl_record.get('_context'):
                     context = xbrl_record['_context']
-                    period_start_date = context.get('context_period_instant') or context.get('context_period_startdate')
-                    period_end_date = context.get('context_period_enddate')
+                    period_start_date = context.get('period_instant') or context.get('period_startdate')
+                    period_end_date = context.get('period_enddate')
                 
                 # Create record in the format expected by construct_fundamentals
                 record = {

{datamule-2.0.6.dist-info → datamule-2.0.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 2.0.6
+Version: 2.0.8
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-2.0.6.dist-info → datamule-2.0.8.dist-info}/RECORD

@@ -6,7 +6,7 @@ datamule/package_updater.py,sha256=Z9zaa_y0Z5cknpRn8oPea3gg4kquFHfpfhduKKCZ6NU,9
 datamule/portfolio.py,sha256=YViG1JgJ9SFhg8N3tOOhBI8oc6Pmi2vwnHeHmlkC_5U,12119
 datamule/portfolio_compression_utils.py,sha256=8OPYEN5zAdV1FiTxgVN3S7cTKs99Elv74bwgoIJP4QY,12654
 datamule/sheet.py,sha256=Ws_YRtpvewLVioarngVMe8cgG_sp11MP9_goGbRaiWE,23952
-datamule/submission.py,sha256=tX1r-KeytnvZRcyeVeBVOS9KbT0baOs7eOgE9qGPU0g,15918
+datamule/submission.py,sha256=7rtN3EEB50iU7E-B_i-e0JHY382EIgmDcrS1KRZ1mUc,15886
 datamule/data/listed_filer_metadata.csv,sha256=dT9fQ8AC5P1-Udf_UF0ZkdXJ88jNxJb_tuhi5YYL1rc,2426827
 datamule/datamule/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/datamule/datamule_lookup.py,sha256=e8djAg-ctSyHiKk7BjbtgugZ3p8roUjzsym5z3AihUg,9468
@@ -14,9 +14,9 @@ datamule/datamule/datamule_mysql_rds.py,sha256=P5vL3RJnOwLz25hPKuoYmxSX7XeDe83YE
 datamule/datamule/downloader.py,sha256=aTyVUuIwynPtHB0Z9BvCasy9Ao5wfHptNAsjN-7yDTk,18525
 datamule/datamule/sec_connector.py,sha256=VwOaODpHoAWy8JIky6kLR1-orW_PB61RHw7pIGRpkow,3288
 datamule/document/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/document/document.py,sha256=SMsofx2xmO92pGEM0lQt9cm7lxfuGT8WqcFvL4rxOVk,14406
+datamule/document/document.py,sha256=OtfIvHRxdIXmNq5lOpBiLQw9P4o9OYScdpJxNEX51Yc,14477
 datamule/document/tables/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/document/tables/tables.py,sha256=qGGMG2p85AKfS5uBY96m77qREyTtsA7BqfqjELT-x0U,4273
+datamule/document/tables/tables.py,sha256=8riSAof6o-Gxoo0SkiQAE61fw8NmzDnEhJe6dATzmvA,4487
 datamule/document/tables/tables_13fhr.py,sha256=-6tWcaTyNsb0XuW0WMBrYir9Zn1wLZL0laKxRYfPNyg,4265
 datamule/document/tables/tables_25nse.py,sha256=kpoOcIpra6i3Wx_6pUCj1fkx0wUbMhx7pc8yUkrBJb4,980
 datamule/document/tables/tables_informationtable.py,sha256=L7qSNTiyr45iIa8z4gm7pKw_meHjIqMFc1SF5Y6XAGg,1658
@@ -50,7 +50,7 @@ datamule/seclibrary/bq.py,sha256=C8sb_rpXTvchprrFLcbRar4Qi0XWW25tnv1YsHSS5o4,180
 datamule/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/utils/construct_submissions_data.py,sha256=NB_hvfxlRXPyt4Fgc-5qA8vJRItkLhBedCSTaxwW7Jg,5887
 datamule/utils/format_accession.py,sha256=60RtqoNqoT9zSKVb1DeOv1gncJxzPTFMNW4SNOVmC_g,476
-datamule-2.0.6.dist-info/METADATA,sha256=j3-QjEqEptbptQCKn52ImVShwyjvurA0v-Bqt4aXel8,560
-datamule-2.0.6.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-datamule-2.0.6.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-2.0.6.dist-info/RECORD,,
+datamule-2.0.8.dist-info/METADATA,sha256=gaqli_hX9EdM_jVeAO8uGdh4xJQO5iOXI8wi0BL-oUc,560
+datamule-2.0.8.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+datamule-2.0.8.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-2.0.8.dist-info/RECORD,,