datamule 2.0.7__py3-none-any.whl → 2.0.9__py3-none-any.whl

datamule/datamule/datamule_mysql_rds.py

@@ -67,6 +67,8 @@ class DatamuleMySQL:
             database = 'lookup_db'
         elif table == 'submission_details':
             database = 'lookup_db'
+        elif table == 'proxy_voting_record':
+            database = 'npx_db'
         else:
             raise ValueError(f"Unsupported table: {table}")
         

datamule/document/document.py

@@ -231,13 +231,13 @@ class Document:
         with open(output_filename, 'w',encoding='utf-8') as f:
             json.dump(self.data, f, indent=2)
 
-    def parse_tables(self):
+    def parse_tables(self,must_exist_in_mapping=True):
         if self.extension != '.xml':
             self._tables = []
         else:
             # Use the property to trigger parsing if needed
             data = self.data
-            tables = Tables(document_type = self.type, accession=self.accession, data=data)
+            tables = Tables(document_type = self.type, accession=self.accession, data=data,must_exist_in_mapping=must_exist_in_mapping)
             self._tables = tables.tables
 
     @property

datamule/document/tables/tables.py

@@ -64,14 +64,14 @@ def seperate_data(tables_dict, data):
     
     return data_list
 
-def apply_mapping(flattened_data, mapping_dict, accession):
+def apply_mapping(flattened_data, mapping_dict, accession, must_exist_in_mapping=False):
     """Apply mapping to flattened data and add accession"""
     
     # Handle case where flattened_data is a list of dictionaries
     if isinstance(flattened_data, list):
         results = []
         for data_dict in flattened_data:
-            results.append(apply_mapping(data_dict, mapping_dict, accession))
+            results.extend(apply_mapping(data_dict, mapping_dict, accession,must_exist_in_mapping))
         return results
     
     # Original logic for single dictionary
@@ -85,10 +85,11 @@ def apply_mapping(flattened_data, mapping_dict, accession):
             ordered_row[new_key] = None
     
     # Add any remaining keys that weren't in the mapping
-    for key, value in flattened_data.items():
-        ordered_row[key] = value
+    if not must_exist_in_mapping:
+        for key, value in flattened_data.items():
+            ordered_row[key] = value
     
-    return ordered_row
+    return [ordered_row]
 
 # should have table type, accession, data
 class Table:
@@ -99,7 +100,7 @@ class Table:
 
 
 class Tables():
-    def __init__(self,document_type,accession,data):
+    def __init__(self,document_type,accession,data,must_exist_in_mapping=True):
         self.document_type = document_type
         self.accession = accession
         self.data = data
@@ -107,9 +108,9 @@ class Tables():
         # to fill in
         self.tables = []
 
-        self.parse_tables()
+        self.parse_tables(must_exist_in_mapping=must_exist_in_mapping)
 
-    def parse_tables(self):
+    def parse_tables(self,must_exist_in_mapping=True):
         # first select dict
 
         try:
@@ -125,5 +126,5 @@ class Tables():
         
         for table_name, flattened_data in data_dicts:
             mapping_dict = tables_dict[table_name]['mapping']
-            mapped_data = apply_mapping(flattened_data, mapping_dict, self.accession)
+            mapped_data = apply_mapping(flattened_data, mapping_dict, self.accession,must_exist_in_mapping)
             self.tables.append(Table(mapped_data, table_name, self.accession))

{datamule-2.0.7.dist-info → datamule-2.0.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 2.0.7
+Version: 2.0.9
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-2.0.7.dist-info → datamule-2.0.9.dist-info}/RECORD

@@ -10,13 +10,13 @@ datamule/submission.py,sha256=7rtN3EEB50iU7E-B_i-e0JHY382EIgmDcrS1KRZ1mUc,15886
 datamule/data/listed_filer_metadata.csv,sha256=dT9fQ8AC5P1-Udf_UF0ZkdXJ88jNxJb_tuhi5YYL1rc,2426827
 datamule/datamule/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/datamule/datamule_lookup.py,sha256=e8djAg-ctSyHiKk7BjbtgugZ3p8roUjzsym5z3AihUg,9468
-datamule/datamule/datamule_mysql_rds.py,sha256=P5vL3RJnOwLz25hPKuoYmxSX7XeDe83YETnwT6miRMo,11858
+datamule/datamule/datamule_mysql_rds.py,sha256=Mw3o21p7ZBPvL4vhRot4cAlM8Hf2FDAjuBjTp5WzpHU,11935
 datamule/datamule/downloader.py,sha256=aTyVUuIwynPtHB0Z9BvCasy9Ao5wfHptNAsjN-7yDTk,18525
 datamule/datamule/sec_connector.py,sha256=VwOaODpHoAWy8JIky6kLR1-orW_PB61RHw7pIGRpkow,3288
 datamule/document/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/document/document.py,sha256=SMsofx2xmO92pGEM0lQt9cm7lxfuGT8WqcFvL4rxOVk,14406
+datamule/document/document.py,sha256=OtfIvHRxdIXmNq5lOpBiLQw9P4o9OYScdpJxNEX51Yc,14477
 datamule/document/tables/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/document/tables/tables.py,sha256=qGGMG2p85AKfS5uBY96m77qREyTtsA7BqfqjELT-x0U,4273
+datamule/document/tables/tables.py,sha256=8riSAof6o-Gxoo0SkiQAE61fw8NmzDnEhJe6dATzmvA,4487
 datamule/document/tables/tables_13fhr.py,sha256=-6tWcaTyNsb0XuW0WMBrYir9Zn1wLZL0laKxRYfPNyg,4265
 datamule/document/tables/tables_25nse.py,sha256=kpoOcIpra6i3Wx_6pUCj1fkx0wUbMhx7pc8yUkrBJb4,980
 datamule/document/tables/tables_informationtable.py,sha256=L7qSNTiyr45iIa8z4gm7pKw_meHjIqMFc1SF5Y6XAGg,1658
@@ -50,7 +50,7 @@ datamule/seclibrary/bq.py,sha256=C8sb_rpXTvchprrFLcbRar4Qi0XWW25tnv1YsHSS5o4,180
 datamule/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/utils/construct_submissions_data.py,sha256=NB_hvfxlRXPyt4Fgc-5qA8vJRItkLhBedCSTaxwW7Jg,5887
 datamule/utils/format_accession.py,sha256=60RtqoNqoT9zSKVb1DeOv1gncJxzPTFMNW4SNOVmC_g,476
-datamule-2.0.7.dist-info/METADATA,sha256=8zSjabg29zw4l4IMkE_akkY7Q6Rnmah37xeUzGddMQU,560
-datamule-2.0.7.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-datamule-2.0.7.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-2.0.7.dist-info/RECORD,,
+datamule-2.0.9.dist-info/METADATA,sha256=HPPYbuYA54lDhy82ZrZ8io3OsQNh-ZJRGfMUKO8V0cQ,560
+datamule-2.0.9.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+datamule-2.0.9.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-2.0.9.dist-info/RECORD,,