datamule 1.2.6__py3-none-any.whl → 1.2.7__py3-none-any.whl

datamule/document/document.py

@@ -141,7 +141,7 @@ class Document:
         with open(output_filename, 'w',encoding='utf-8') as f:
             json.dump(self.data, f, indent=2)
 
-    def to_tabular(self):
+    def tables(self):
         if self.type == 'submission_metadata':
             return process_tabular_data(self)
         elif self.extension != '.xml':
@@ -155,7 +155,7 @@ class Document:
         output_folder = Path(output_folder)
         output_folder.mkdir(exist_ok=True)
             
-        tables = self.to_tabular()
+        tables = self.tables()
 
         if not tables:
             return

datamule/document/mappings/ex102_abs.py

@@ -0,0 +1,63 @@
+# Assets dictionary mapping
+assets_dict_ex102_abs = {
+    'assetNumber': 'assetNumber',
+    'DefeasedStatusCode': 'DefeasedStatusCode',
+    'defeasanceOptionStartDate': 'defeasanceOptionStartDate',
+    'mostRecentDebtServiceCoverageNetOperatingIncomePercentage': 'mostRecentDebtServiceCoverageNetOperatingIncomePercentage',
+    'mostRecentDebtServiceAmount': 'mostRecentDebtServiceAmount',
+    'debtServiceCoverageSecuritizationCode': 'debtServiceCoverageSecuritizationCode',
+    'debtServiceCoverageNetOperatingIncomeSecuritizationPercentage': 'debtServiceCoverageNetOperatingIncomeSecuritizationPercentage',
+    'valuationSecuritizationDate': 'valuationSecuritizationDate',
+    'physicalOccupancySecuritizationPercentage': 'physicalOccupancySecuritizationPercentage',
+    'revenueSecuritizationAmount': 'revenueSecuritizationAmount',
+    'valuationSourceSecuritizationCode': 'valuationSourceSecuritizationCode',
+    'financialsSecuritizationDate': 'financialsSecuritizationDate',
+    'mostRecentNetCashFlowAmount': 'mostRecentNetCashFlowAmount',
+    'operatingExpensesAmount': 'operatingExpensesAmount',
+    'operatingExpensesSecuritizationAmount': 'operatingExpensesSecuritizationAmount',
+    'netOperatingIncomeNetCashFlowSecuritizationCode': 'netOperatingIncomeNetCashFlowSecuritizationCode',
+    'mostRecentValuationSourceCode': 'mostRecentValuationSourceCode',
+    'mostRecentDebtServiceCoverageNetCashFlowpercentage': 'mostRecentDebtServiceCoverageNetCashFlowpercentage',
+    'debtServiceCoverageNetCashFlowSecuritizationPercentage': 'debtServiceCoverageNetCashFlowSecuritizationPercentage',
+    'mostRecentAnnualLeaseRolloverReviewDate': 'mostRecentAnnualLeaseRolloverReviewDate',
+    'mostRecentRevenueAmount': 'mostRecentRevenueAmount',
+    'mostRecentPhysicalOccupancyPercentage': 'mostRecentPhysicalOccupancyPercentage',
+    'mostRecentNetOperatingIncomeAmount': 'mostRecentNetOperatingIncomeAmount',
+    'netOperatingIncomeSecuritizationAmount': 'netOperatingIncomeSecuritizationAmount',
+    'netOperatingIncomeNetCashFlowCode': 'netOperatingIncomeNetCashFlowCode',
+    'mostRecentFinancialsStartDate': 'mostRecentFinancialsStartDate',
+    'mostRecentFinancialsEndDate': 'mostRecentFinancialsEndDate',
+    'accession': 'accession',
+    'valuationSecuritizationAmount': 'valuationSecuritizationAmount',
+    'mostRecentValuationDate': 'mostRecentValuationDate',
+    'mostRecentValuationAmount': 'mostRecentValuationAmount',
+    'mostRecentDebtServiceCoverageCode': 'mostRecentDebtServiceCoverageCode',
+    'netCashFlowFlowSecuritizationAmount': 'netCashFlowFlowSecuritizationAmount'
+}
+
+# Properties dictionary mapping
+properties_dict_ex102_abs = {
+    'unitsBedsRoomsNumber': 'unitsBedsRoomsNumber',
+    'propertyCounty': 'propertyCounty',
+    'squareFeetLargestTenantNumber': 'squareFeetLargestTenantNumber',
+    'netRentableSquareFeetNumber': 'netRentableSquareFeetNumber',
+    'leaseExpirationThirdLargestTenantDate': 'leaseExpirationThirdLargestTenantDate',
+    'leaseExpirationLargestTenantDate': 'leaseExpirationLargestTenantDate',
+    'propertyZip': 'propertyZip',
+    'squareFeetThirdLargestTenantNumber': 'squareFeetThirdLargestTenantNumber',
+    'propertyStatusCode': 'propertyStatusCode',
+    'propertyState': 'propertyState',
+    'yearBuiltNumber': 'yearBuiltNumber',
+    'propertyCity': 'propertyCity',
+    'propertyName': 'propertyName',
+    'propertyAddress': 'propertyAddress',
+    'yearLastRenovated': 'yearLastRenovated',
+    'leaseExpirationSecondLargestTenantDate': 'leaseExpirationSecondLargestTenantDate',
+    'thirdLargestTenant': 'thirdLargestTenant',
+    'unitsBedsRoomsSecuritizationNumber': 'unitsBedsRoomsSecuritizationNumber',
+    'propertyTypeCode': 'propertyTypeCode',
+    'largestTenant': 'largestTenant',
+    'squareFeetSecondLargestTenantNumber': 'squareFeetSecondLargestTenantNumber',
+    'netRentableSquareFeetSecuritizationNumber': 'netRentableSquareFeetSecuritizationNumber',
+    'secondLargestTenant': 'secondLargestTenant'
+}

datamule/document/processing.py

@@ -22,6 +22,8 @@ def process_tabular_data(self):
     # complete mark:
     elif self.type in ["N-PX","N-PX/A"]:
         tables = process_npx(self.data, self.accession)
+    elif self.type in ["EX-102"]:
+        tables = process_ex102_abs(self.data, self.accession)
 
     elif self.type in ["SBSEF","SBSEF/A","SBSEF-V","SBSEF-W"]:
         tables = process_sbsef(self.data, self.accession)
@@ -70,8 +72,7 @@ def process_tabular_data(self):
         tables = process_reg_a(self.data, self.accession)
     # elif self.type in ["SBSE","SBSE/A","SBSE-A","SBSE-A/A","SBSE-BD","SBSE-BD/A","SBSE-C","SBSE-W","SBSE-CCO-RPT","SBSE-CCO-RPT/A"]:
     #     tables = process_sbs(self.data, self.accession)
-    # elif self.type in ["EX-102"]:
-    #     tables = process_ex102_abs(self.data, self.accession)
+
     elif self.type == "PROXY VOTING RECORD":
         tables = process_proxy_voting_record(self.data, self.accession)
     elif self.type == 'submission_metadata':
@@ -589,13 +590,39 @@ def process_reg_a(data, accession):
     
 #     return tables
 
-# def process_ex102_abs(data, accession):
-#     tables = []
-#     asset_data = safe_get(data, ['assetData'])
-#     if asset_data:
-#         tables.append(Table(_flatten_dict(asset_data), 'abs', accession))
-#     raise NotImplementedError("Need to implement the rest of the ABS processing")
-#     return tables
+def process_ex102_abs(data, accession):
+    tables = []
+    data = safe_get(data, ['assetData', 'assets'])
+    
+    # Create assets list: all items without their 'property' field
+    assets = [{k: v for k, v in item.items() if k != 'property'} for item in data]
+
+    # Create properties list in a more vectorized way
+    properties = []
+    
+    # Handle dictionary properties
+    properties.extend([
+        item['property'] | {'assetNumber': item['assetNumber']}
+        for item in data
+        if 'property' in item and isinstance(item['property'], dict)
+    ])
+    
+    # Handle list properties - flatten in one operation
+    properties.extend([
+        prop | {'assetNumber': item['assetNumber']}
+        for item in data
+        if 'property' in item and isinstance(item['property'], list)
+        for prop in item['property']
+        if isinstance(prop, dict)
+    ])
+    
+    if assets:
+        tables.append(Table(_flatten_dict(assets), 'assets_ex102_absee', accession))
+    
+    if properties:
+        tables.append(Table(_flatten_dict(properties), 'properties_ex102_absee', accession))
+    
+    return tables
 
 # def process_ma(data, accession):
 #     tables = []

datamule/document/table.py

@@ -19,6 +19,10 @@ from .mappings.twentyfivense import *
 from .mappings.twentyfourf2nt import *
 from .mappings.information_table import *
 from .mappings.submission_metadata import *
+from .mappings.ex102_abs import *
+
+from pathlib import Path
+import csv
 # need to check if mappings correctly create new columns
 class Table():
     def __init__(self, data, type,accession):
@@ -27,11 +31,18 @@ class Table():
         self.type = type
         self.data = data
         self.accession = accession
-        self.columns = self.determine_columns()
+        self.columns = self.determine_columns_complete()
+
+    def determine_columns_complete(self):
+        if not self.data:
+            return []
+        return list(set().union(*(row.keys() for row in self.data)))
+
 
     def determine_columns(self):
         if len(self.data) == 0:
             return []
+        
         return self.data[0].keys()
 
     def add_column(self,column_name,value):
@@ -227,7 +238,11 @@ class Table():
             mapping_dict = item_9_24f2nt_dict
         elif self.type == 'signature_info_schedule_a':
             mapping_dict = signature_24f2nt_dict
-
+        # ABS
+        elif self.type == 'assets_ex102_absee':
+            mapping_dict = assets_dict_ex102_abs
+        elif self.type =='properties_ex102_absee':
+            mapping_dict = properties_dict_ex102_abs
         # submission metadata
         elif self.type == 'document_submission_metadata':
             mapping_dict = document_submission_metadata_dict
@@ -250,9 +265,6 @@ class Table():
             for old_key, new_key in mapping_dict.items():
                 if old_key in row:
                     ordered_row[new_key] = row.pop(old_key)
-                else:
-                    # if the old key is not present, set the new key to None
-                    ordered_row[new_key] = None
             
             # Then add any remaining keys that weren't in the mapping
             for key, value in row.items():
@@ -262,4 +274,30 @@ class Table():
             row.clear()
             row.update(ordered_row)
 
-        self.determine_columns()
+        # Update the columns after mapping
+        columns = set(self.columns)
+        # remove the old columns that are now in the mapping
+        columns.difference_update(mapping_dict.keys())
+        # add the new columns from the mapping
+        columns.update(mapping_dict.values())
+        # add the accession column to the columns
+        columns.add('accession')
+
+        self.columns = list(columns)
+
+    def write_csv(self, output_file):
+        output_file = Path(output_file)
+        fieldnames = self.columns
+        
+        # Check if the file already exists
+        if output_file.exists():
+            # Append to existing file without writing header
+            with open(output_file, 'a', newline='') as csvfile:
+                writer = csv.DictWriter(csvfile, fieldnames=fieldnames, quoting=csv.QUOTE_ALL)
+                writer.writerows(self.data)
+        else:
+            # Create new file with header
+            with open(output_file, 'w', newline='') as csvfile:
+                writer = csv.DictWriter(csvfile, fieldnames=fieldnames, quoting=csv.QUOTE_ALL)
+                writer.writeheader()
+                writer.writerows(self.data)

{datamule-1.2.6.dist-info → datamule-1.2.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.2.6
+Version: 1.2.7
 Summary: Making it easier to use SEC filings.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-1.2.6.dist-info → datamule-1.2.7.dist-info}/RECORD

@@ -7,12 +7,13 @@ datamule/portfolio.py,sha256=8fiK-vfZM5-NJSvOEsDR2YDb-2njjzFk6l7BiRyrzOM,7168
 datamule/sheet.py,sha256=TvFqK9eAYuVoJ2uWdAlx5EN6vS9lke-aZf7FqtUiDBc,22304
 datamule/submission.py,sha256=Yh5nG3ioumhl6z30wJdIEmKjDDNSuo0r2xycZSIaeIg,11035
 datamule/document/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/document/document.py,sha256=CQqlHCVwp0TfOjE173jJSh9O-zpYuo4Ixr85vXCJA1E,9977
-datamule/document/processing.py,sha256=Wn2Dpe6vXqVDm1qgIc-nR273wKBQEtstxZbugvuPZdI,28526
-datamule/document/table.py,sha256=TiOQyFp8CLWt-hHHX8BCjKuaBgeb-uO23ekD8QoUsqM,10993
+datamule/document/document.py,sha256=menUFoeWwiY0rJnBkQiqY4NWnO0J17-qs8jFvO_1jiY,9969
+datamule/document/processing.py,sha256=MLgOtVNmsUotfxj5XvQqw5Q3idhvK6FdHcQz3U4ud7s,29333
+datamule/document/table.py,sha256=NdDQh7EWOf2qSx5ZxydCHpaNPd7J7wtVFoooiVzbmgk,12443
 datamule/document/mappings/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/document/mappings/atsn.py,sha256=qkZGNIhyPC3VTTOjQ8-FSCQIhUy4XeSycUGLShxNVCo,17743
 datamule/document/mappings/cfportal.py,sha256=bR9d6DDY0kJ_HGx_hND2y1PNNkZjemYZ2KdyFAcv760,25257
+datamule/document/mappings/ex102_abs.py,sha256=FdGKvteRh_HsYgILF-8o4R6aSsjYwcaLpJxzdru4FTE,3976
 datamule/document/mappings/ex99a_sdr.py,sha256=PNdj9I0ZhNicPObLelNmjp33EgTwzvukqkBDnwxarE0,19
 datamule/document/mappings/ex99c_sdr.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/document/mappings/ex99g_sdr.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -54,7 +55,7 @@ datamule/seclibrary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSu
 datamule/seclibrary/bq.py,sha256=C8sb_rpXTvchprrFLcbRar4Qi0XWW25tnv1YsHSS5o4,18025
 datamule/seclibrary/downloader.py,sha256=PIgz_7ASUTZOHcUZGcD1SmLaGSbq7xe7EiJT0Z7HU4M,13653
 datamule/seclibrary/query.py,sha256=qGuursTERRbOGfoDcYcpo4oWkW3PCBW6x1Qf1Puiak4,7352
-datamule-1.2.6.dist-info/METADATA,sha256=WwrQVyxcEIzhkzy_0WQhkoXHt_nXbmzarCMa2pYl2mw,490
-datamule-1.2.6.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-datamule-1.2.6.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-1.2.6.dist-info/RECORD,,
+datamule-1.2.7.dist-info/METADATA,sha256=EvHa-0eCP6v0GRF6tq4QqnIDQpqM0m9yIXc68BO8Wck,490
+datamule-1.2.7.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+datamule-1.2.7.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-1.2.7.dist-info/RECORD,,