datamule 2.3.4__tar.gz → 2.3.6__tar.gz

{datamule-2.3.4 → datamule-2.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 2.3.4
+Version: 2.3.6
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-2.3.4 → datamule-2.3.6}/datamule/__init__.py

@@ -1,9 +1,9 @@
-from .submission import Submission
-from .portfolio import Portfolio
+from .submission.submission import Submission
+from .portfolio.portfolio import Portfolio
 from .document.document import Document
 from .helper import _load_package_csv, load_package_dataset
 from .config import Config
-from .sheet import Sheet
+from .sheet.sheet import Sheet
 from .index import Index
 from .package_updater import PackageUpdater
 from .utils.format_accession import format_accession
@@ -32,6 +32,8 @@ def _setup_notebook_env():
 # Set up notebook environment
 _setup_notebook_env()
 
+
+# TODO, is this load bearing?
 __all__ = [
     '_load_package_csv',
     'load_package_dataset',

{datamule-2.3.4 → datamule-2.3.6}/datamule/book/s3transfer.py

@@ -8,7 +8,7 @@ from datetime import datetime, timedelta
 from urllib.parse import urlparse
 from tqdm import tqdm
 import logging
-from ..sheet import Sheet
+from ..sheet.sheet import Sheet
 from ..utils.format_accession import format_accession
 
 # Set up logging

{datamule-2.3.4 → datamule-2.3.6}/datamule/document/document.py

@@ -10,7 +10,7 @@ from pathlib import Path
 import webbrowser
 from secsgml.utils import bytes_to_str
 import tempfile
-from .tables.tables import Tables
+from ..tables.tables import Tables
 
 from ..tags.utils import get_cusip_using_regex, get_isin_using_regex, get_figi_using_regex,get_all_tickers, get_full_names,get_full_names_dictionary_lookup, analyze_lm_sentiment_fragment
 from ..utils.pdf import has_extractable_text

{datamule-2.3.4/datamule → datamule-2.3.6/datamule/portfolio}/portfolio.py

@@ -1,19 +1,19 @@
 from pathlib import Path
 from tqdm import tqdm
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from .submission import Submission
-from .sec.submissions.downloader import download as sec_download
-from .sec.submissions.textsearch import filter_text
-from .config import Config
+from ..submission.submission import Submission
+from ..sec.submissions.downloader import download as sec_download
+from ..sec.submissions.textsearch import filter_text
+from ..config import Config
 import os
 import tarfile
 from threading import Lock
-from .helper import _process_cik_and_metadata_filters
-from .datamule.downloader import download as seclibrary_download
-from .sec.xbrl.filter_xbrl import filter_xbrl
-from .sec.submissions.monitor import Monitor
-from .portfolio_compression_utils import CompressionManager
-from .datamule.sec_connector import SecConnector
+from ..helper import _process_cik_and_metadata_filters
+from ..datamule.downloader import download as seclibrary_download
+from ..sec.xbrl.filter_xbrl import filter_xbrl
+from ..sec.submissions.monitor import Monitor
+from .portfolio_compression_utils_legacy import CompressionManager
+from ..datamule.sec_connector import SecConnector
 import shutil
 
 
@@ -31,6 +31,7 @@ class Portfolio:
 
         self.monitor = Monitor()
         
+        
         if self.path.exists():
             self._load_submissions()
             self.submissions_loaded = True
@@ -47,6 +48,7 @@ class Portfolio:
         regular_items = [f for f in self.path.iterdir() if (f.is_dir() or f.suffix=='.tar') and 'batch' not in f.name]
         batch_tars = [f for f in self.path.iterdir() if f.is_file() and 'batch' in f.name and f.suffix == '.tar']
         
+        
         # Load regular submissions (existing logic)
         def load_submission(folder):
             return Submission(folder)
@@ -99,11 +101,12 @@ class Portfolio:
             try:
                 submission = Submission(
                     batch_tar_path=batch_tar_path,
-                    accession_prefix=accession_prefix,
+                    accession=accession_prefix,
                     portfolio_ref=self
                 )
                 submissions.append(submission)
             except Exception as e:
+                print(f"Path: {batch_tar_path}. Exception: {e}")
                 pass
                 #print(f"Path: {batch_tar_path}. Exception: {e}")
             pbar.update(1)  # Update progress for each successful submission
@@ -111,18 +114,6 @@ class Portfolio:
         return submissions
             
 
-    def compress(self, compression=None, compression_level=None, threshold=1048576, max_batch_size=1024*1024*1024):
-        """
-        Compress all individual submissions into batch tar files.
-        
-        Args:
-            compression: None, 'gzip', or 'zstd' for document compression (default: None)
-            compression_level: Compression level, if None uses defaults (gzip=6, zstd=3)
-            threshold: Size threshold for compressing individual documents (default: 1MB)
-            max_batch_size: Maximum size per batch tar file (default: 1GB)
-        """
-        CompressionManager().compress_portfolio(self, compression, compression_level, threshold, max_batch_size, self.MAX_WORKERS)
-
     def decompress(self):
         """
         Decompress all batch tar files back to individual submission directories.

datamule-2.3.4/datamule/portfolio_compression_utils.py → datamule-2.3.6/datamule/portfolio/portfolio_compression_utils_legacy.py

@@ -8,6 +8,8 @@ from tqdm import tqdm
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from secsgml.utils import bytes_to_str, calculate_documents_locations_in_tar
 
+# probably can delete much of this TODO
+
 
 class CompressionManager:
     

{datamule-2.3.4 → datamule-2.3.6}/datamule/seclibrary/bq.py

@@ -2,6 +2,8 @@ import os
 import requests
 import json
 
+# slated for deletion
+
 def get_information_table(
     # Optional filtering parameters
     columns=None,

{datamule-2.3.4/datamule → datamule-2.3.6/datamule/sheet}/sheet.py

@@ -1,10 +1,10 @@
 from pathlib import Path
 import csv
 import os
-from .helper import _process_cik_and_metadata_filters, load_package_dataset
-from .sec.xbrl.downloadcompanyfacts import download_company_facts
-from .datamule.datamule_lookup import datamule_lookup
-from .datamule.datamule_mysql_rds import query_mysql_rds
+from ..helper import _process_cik_and_metadata_filters, load_package_dataset
+from ..sec.xbrl.downloadcompanyfacts import download_company_facts
+from ..datamule.datamule_lookup import datamule_lookup
+from ..datamule.datamule_mysql_rds import query_mysql_rds
 from company_fundamentals.utils import get_fundamental_mappings
 from company_fundamentals import construct_fundamentals
 class Sheet:

{datamule-2.3.4/datamule → datamule-2.3.6/datamule/submission}/submission.py

@@ -1,10 +1,10 @@
 from pathlib import Path
 import json
-from .document.document import Document
+from ..document.document import Document
 from secsgml import parse_sgml_content_into_memory
 from secsgml.parse_sgml import transform_metadata_string
 from secsgml.utils import bytes_to_str
-from .sec.utils import headers
+from ..sec.utils import headers
 import tarfile
 import zstandard as zstd
 import gzip
@@ -12,31 +12,85 @@ import urllib.request
 from secxbrl import parse_inline_xbrl
 from company_fundamentals import construct_fundamentals
 from decimal import Decimal
-from .utils.format_accession import format_accession
-
+from ..utils.format_accession import format_accession
+from .tar_submission import tar_submission
+
+# probably needs rework later
+class FundamentalsAccessor:
+    def __init__(self, submission):
+        self.submission = submission
+        self._cache = {}
+        self._all_data = None
+    
+    def __getattr__(self, category):
+        if category not in self._cache:
+            self._cache[category] = self.submission.parse_fundamentals(categories=[category])
+        return self._cache[category]
+    
+    def _get_all_data(self):
+        if self._all_data is None:
+            self._all_data = self.submission.parse_fundamentals(categories=None)
+        return self._all_data
+    
+    # Make the accessor behave like the underlying data
+    def __getitem__(self, key):
+        return self._get_all_data()[key]
+    
+    def __repr__(self):
+        return repr(self._get_all_data())
+    
+    def __str__(self):
+        return str(self._get_all_data())
+    
+    def __iter__(self):
+        return iter(self._get_all_data())
+    
+    def __len__(self):
+        return len(self._get_all_data()) if self._get_all_data() else 0
+    
+    def __bool__(self):
+        return bool(self._get_all_data())
 
 class Submission:
     def __init__(self, path=None, sgml_content=None, keep_document_types=None,
-                 batch_tar_path=None, accession_prefix=None, portfolio_ref=None,url=None):
+                 batch_tar_path=None, accession=None, portfolio_ref=None,url=None):
         
+        # get accession number
+        # lets just use accesion-prefix, to get around malformed metadata files (1995 has a lot!)
+        if path is not None:
+            self.accession = format_accession(path.stem,'no-dash')
+        elif batch_tar_path is not None:
+            self.accession = format_accession(accession,'no-dash')
+        elif url is not None or sgml_content is not None:
+            if accession is None:
+                raise ValueError("If using url or sgml_content, accession must be specified.")
+            self.accession = format_accession(accession,'no-dash')
+        else:
+            raise ValueError("If this appears, please post an issue: https://github.com/john-friedman/datamule-python/issues.")
+
 
         # declare vars to be filled later
         self._xbrl = None
         self._fundamentals_cache = {}
+        self._tar = None
+        self._tar_compression_type = 'zstd'
+        self._tar_compression_level = 3
+        self._accession_year_2d = None
+        self._documents = None
         
         # Validate parameters
         param_count = sum(x is not None for x in [path, sgml_content, batch_tar_path,url])
         if param_count != 1:
             raise ValueError("Exactly one of path, sgml_content, or batch_tar_path must be provided")
         
-        if batch_tar_path is not None and (accession_prefix is None or portfolio_ref is None):
-            raise ValueError("batch_tar_path requires both accession_prefix and portfolio_ref")
+        if batch_tar_path is not None and (self.accession is None or portfolio_ref is None):
+            raise ValueError("batch_tar_path requires both accession and portfolio_ref")
         
         # Initialize batch tar attributes
         self.batch_tar_path = batch_tar_path
-        self.accession_prefix = accession_prefix
         self.portfolio_ref = portfolio_ref
         
+        # here should set accession either from url or make it a required argument if sgml content
         if url is not None or sgml_content is not None:
             if url is not None:
                 request = urllib.request.Request(url, headers=headers)
@@ -49,17 +103,15 @@ class Submission:
 
             self.path = None
             metadata, raw_documents = parse_sgml_content_into_memory(sgml_content)
-            metadata = bytes_to_str(metadata)
+            metadata = bytes_to_str(metadata,lower=False)
 
             # standardize metadata
             metadata = transform_metadata_string(metadata)
 
             self.metadata = Document(type='submission_metadata', content=metadata, extension='.json',filing_date=None,accession=None,path=None)
-            # code dupe
-            self.accession = self.metadata.content['accession-number']
             self.filing_date= f"{self.metadata.content['filing-date'][:4]}-{self.metadata.content['filing-date'][4:6]}-{self.metadata.content['filing-date'][6:8]}"
     
-            self.documents = []
+            self.documents_obj_list = []
             filtered_metadata_documents = []
 
             for idx,doc in enumerate(self.metadata.content['documents']):
@@ -72,7 +124,7 @@ class Submission:
                 # write as txt if not declared
                 filename = doc.get('filename','.txt')
                 extension = Path(filename).suffix
-                self.documents.append(Document(type=type, content=raw_documents[idx], extension=extension,filing_date=self.filing_date,accession=self.accession))
+                self.documents_obj_list.append(Document(type=type, content=raw_documents[idx], extension=extension,filing_date=self.filing_date,accession=self.accession))
 
                 filtered_metadata_documents.append(doc)
             
@@ -85,24 +137,22 @@ class Submission:
             # Load metadata from batch tar
             with self.portfolio_ref.batch_tar_locks[batch_tar_path]:
                 tar_handle = self.portfolio_ref.batch_tar_handles[batch_tar_path]
-                metadata_obj = tar_handle.extractfile(f'{accession_prefix}/metadata.json')
+                metadata_obj = tar_handle.extractfile(f'{self.accession}/metadata.json')
                 metadata = json.loads(metadata_obj.read().decode('utf-8'))
 
             # Set metadata path using :: notation
-            metadata_path = f"{batch_tar_path}::{accession_prefix}/metadata.json"
+            metadata_path = f"{batch_tar_path}::{self.accession}/metadata.json"
             
             # standardize metadata
             metadata = transform_metadata_string(metadata)
             self.metadata = Document(type='submission_metadata', content=metadata, extension='.json',filing_date=None,accession=None,path=metadata_path)
-
-            # lets just use accesion-prefix, to get around malformed metadata files (1995 has a lot!)
-            self.accession = format_accession(self.accession_prefix,'dash')
             
-            #print(f"s: {self.metadata.content['accession-number']} : {batch_tar_path}")
             self.filing_date= f"{self.metadata.content['filing-date'][:4]}-{self.metadata.content['filing-date'][4:6]}-{self.metadata.content['filing-date'][6:8]}"
 
         elif path is not None:
             self.path = Path(path)  
+
+
             if self.path.suffix == '.tar':
                 with tarfile.open(self.path,'r') as tar:
                     metadata_obj = tar.extractfile('metadata.json')
@@ -118,65 +168,45 @@ class Submission:
             # standardize metadata
             metadata = transform_metadata_string(metadata)
             self.metadata = Document(type='submission_metadata', content=metadata, extension='.json',filing_date=None,accession=None,path=metadata_path)
-            self.accession = self.metadata.content['accession-number']
             self.filing_date= f"{self.metadata.content['filing-date'][:4]}-{self.metadata.content['filing-date'][4:6]}-{self.metadata.content['filing-date'][6:8]}"
 
 
         # booleans
-        self._has_xbrl = any(
+        self._xbrl_bool = any(
                 doc['type'] in ('EX-100.INS', 'EX-101.INS') or 
                 doc.get('filename', '').endswith('_htm.xml')
                 for doc in self.metadata.content['documents']
             )
         
-        self._has_fundamentals = self._has_xbrl
+        self._has_fundamentals = self._xbrl_bool
         
+
+    # TODO rework for better metadata accessing
     def _load_document_by_index(self, idx):
         """Load a document by its index in the metadata documents list."""
         doc = self.metadata.content['documents'][idx]
         
         # If loaded from sgml_content, return pre-loaded document
         if self.path is None and self.batch_tar_path is None:
-            return self.documents[idx]
+            return self.documents_obj_list[idx]
         
         # Get filename from metadata - this is the source of truth
         filename = doc.get('filename')
         if filename is None:
             filename = doc['sequence'] + '.txt'
 
-        # Get the base extension (before any compression extension)
-        # If filename ends with .gz or .zst, the real extension is before that
-        if filename.endswith('.gz'):
-            extension = Path(filename[:-3]).suffix
-            is_compressed = 'gzip'
-        elif filename.endswith('.zst'):
-            extension = Path(filename[:-4]).suffix
-            is_compressed = 'zstd'
-        else:
-            extension = Path(filename).suffix
-            is_compressed = False
-
+        extension = Path(filename).suffix
         # Handle batch tar case
         if self.batch_tar_path is not None:
             with self.portfolio_ref.batch_tar_locks[self.batch_tar_path]:
                 tar_handle = self.portfolio_ref.batch_tar_handles[self.batch_tar_path]
                 
                 # Use exact filename from metadata
-                tar_path = f'{self.accession_prefix}/{filename}'
+                tar_path = f'{self.accession}/{filename}'
                 content = tar_handle.extractfile(tar_path).read()
     
                 
-                # Decompress if needed based on filename extension
-                if is_compressed == 'gzip':
-                    content = gzip.decompress(content)
-                elif is_compressed == 'zstd':
-                    content = zstd.ZstdDecompressor().decompress(content)
-                
-                # Decode text files
-                # if extension in ['.htm', '.html', '.txt', '.xml']:
-                #     content = content.decode('utf-8', errors='replace')
-                
-                document_path = f"{self.batch_tar_path}::{self.accession_prefix}/{filename}"
+                document_path = f"{self.batch_tar_path}::{self.accession}/{filename}"
         
         # Handle regular path case
         else:
@@ -188,27 +218,7 @@ class Submission:
                         content = tar.extractfile(filename).read()
                         actual_filename = filename
                     except:
-                        try:
-                            content = tar.extractfile(filename + '.gz').read()
-                            actual_filename = filename + '.gz'
-                            is_compressed = 'gzip'
-                        except:
-                            try:
-                                content = tar.extractfile(filename + '.zst').read()
-                                actual_filename = filename + '.zst'
-                                is_compressed = 'zstd'
-                            except:
-                                raise FileNotFoundError(f"Document file not found in tar: {filename}")
-                    
-                    # Decompress if compressed
-                    if is_compressed == 'gzip':
-                        content = gzip.decompress(content)
-                    elif is_compressed == 'zstd':
-                        content = zstd.ZstdDecompressor().decompress(content)
-                    
-                    # Decode text files
-                    # if extension in ['.htm', '.html', '.txt', '.xml']:
-                    #     content = content.decode('utf-8', errors='replace')
+                        raise FileNotFoundError(f"Document file not found in tar: {filename}")
                     
                     document_path = f"{self.path}::{actual_filename}"
             
@@ -222,15 +232,6 @@ class Submission:
                 with document_path.open('rb') as f:
                     content = f.read()
                 
-                # Decompress if needed based on filename extension
-                if is_compressed == 'gzip':
-                    content = gzip.decompress(content)
-                elif is_compressed == 'zstd':
-                    content = zstd.ZstdDecompressor().decompress(content)
-                
-                # Decode text files
-                # if extension in ['.htm', '.html', '.txt', '.xml']:
-                #     content = content.decode('utf-8', errors='replace')
 
         return Document(
             type=doc['type'], 
@@ -260,20 +261,24 @@ class Submission:
     def parse_xbrl(self):
         if self._xbrl:
             return
-
+        
+        if not self._xbrl_bool:
+            print(f"Submission: {self.accession} has no xbrl")
+            return
+        
         for idx, doc in enumerate(self.metadata.content['documents']):
             if doc['type'] in ['EX-100.INS','EX-101.INS']:
                 document = self._load_document_by_index(idx)
                 self._xbrl = parse_inline_xbrl(content=document.content,file_type='extracted_inline')
-                return
-
+                return  
+            
             if doc['filename'].endswith('_htm.xml'):
                 document = self._load_document_by_index(idx)
                 self._xbrl = parse_inline_xbrl(content=document.content,file_type='extracted_inline')
                 return
 
     @property
-    def xbrl(self):
+    def xbrl(self): 
         if self._xbrl is None:
             self.parse_xbrl()
         return self._xbrl
@@ -353,20 +358,61 @@ class Submission:
 
     @property
     def fundamentals(self):
-        """Get all fundamental data"""
-        return self.parse_fundamentals(categories=None)
-
-    def __getattr__(self, name):
-        # Check if it's a fundamentals property request
-        if name.endswith('_fundamentals'):
-            category = name.replace('_fundamentals', '')
-            return self.parse_fundamentals(categories=[category])
+        """Access fundamentals via attributes: sub.fundamentals.incomeStatement"""
+        if not hasattr(self, '_fundamentals_accessor'):
+            self._fundamentals_accessor = FundamentalsAccessor(self)
+        return self._fundamentals_accessor
+    
+    @property
+    def tar(self):
+        return self._tar_submission().getvalue()
+    
+    def set_tar_compression(self,compression_type='zstd',level=3):
+        self._tar_compression_type = compression_type
+        self._tar_compression_level = level
+    
+    def _tar_submission(self):
+        if self._tar is not None:
+            return self._tar
+        else:
+            documents_obj_list = self._get_documents_obj_list()
+            self._tar = tar_submission(
+                documents_obj_list=documents_obj_list,
+                metadata=self.metadata.content,
+                compression_type=self._tar_compression_type,
+                level=self._tar_compression_level
+            )
+            return self._tar
         
-        # For any other unknown attribute, try it as a fundamentals category
-        # Let parse_fundamentals handle whether it's valid or not
-        result = self.parse_fundamentals(categories=[name])
-        if result is not None:
-            return result
+    @property
+    def accession_year_2d(self):
+        return self._get_accession_year_2d()
+    
+    def _get_accession_year_2d(self):
+        if self._accession_year_2d is not None:
+            return self._accession_year_2d
+        self._accession_year_2d = format_accession(self.accession,'dash').split('-')[1]
+        return self._accession_year_2d
+    
+    @property
+    def documents(self):
+        return self._get_documents()
+    
+    def _get_documents(self):
+        if self._documents is not None:
+            return self._documents
+        
+        self._documents = self.metadata.content['documents']
+        return self._documents
+
+    def _get_documents_obj_list(self):
+        """Get all documents as Document objects"""
+        if hasattr(self, 'documents_obj_list'):
+            return self.documents_obj_list
+        
+        # Generate documents_obj_list for batch tar and path cases
+        documents_obj_list = []
+        for idx in range(len(self.metadata.content['documents'])):
+            documents_obj_list.append(self._load_document_by_index(idx))
         
-        # Only raise AttributeError if parse_fundamentals returns None/empty
-        raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{name}'")
+        return documents_obj_list

datamule-2.3.6/datamule/submission/tar_submission.py

@@ -0,0 +1,79 @@
+import zstandard as zstd
+from secsgml.utils import calculate_documents_locations_in_tar
+import tarfile
+import io
+import json
+
+# Note: we don't actually need accession at this level. TODO
+
+def compress_content(content, compression_type, level):
+    if compression_type == 'zstd':
+        # Create compressor with specified level
+        compressor = zstd.ZstdCompressor(level=level)
+        
+        # Handle string content
+        # This should never be called
+        if isinstance(content, str):
+            content_bytes = content.encode('utf-8')
+        else:
+            content_bytes = content
+            
+        # Compress and return
+        return compressor.compress(content_bytes)
+    
+    # Return uncompressed if not zstd
+    return content
+
+def compress_content_list(document_tuple_list, compression_type, level):
+    if compression_type is None:
+        return document_tuple_list
+    
+    if level is None:
+        level = 3
+
+    # Create new list to avoid modifying original
+    compressed_list = []
+    for document_tuple in document_tuple_list:
+        content = document_tuple[0]  
+        accession = document_tuple[1]
+        compressed_content = compress_content(content, compression_type, level)
+        compressed_list.append((compressed_content, accession))
+    
+    return compressed_list
+
+def tar_content_list(metadata, document_tuple_list_compressed):
+    # Update metadata with compressed sizes
+    for i, (content, accession) in enumerate(document_tuple_list_compressed):  
+        metadata['documents'][i]['secsgml_size_bytes'] = len(content)
+    
+    metadata = calculate_documents_locations_in_tar(metadata)
+
+    tar_buffer = io.BytesIO()
+    with tarfile.open(fileobj=tar_buffer, mode='w') as tar:  
+        # Add metadata first
+        metadata_json = json.dumps(metadata).encode('utf-8')
+        tarinfo = tarfile.TarInfo(f'metadata.json')
+        tarinfo.size = len(metadata_json)
+        tar.addfile(tarinfo, io.BytesIO(metadata_json))
+        
+        # Add each content
+        for i, (content, accession) in enumerate(document_tuple_list_compressed): 
+            doc = metadata['documents'][i]
+            filename = doc.get('filename', doc['sequence'] + '.txt')
+            
+            tarinfo = tarfile.TarInfo(name=filename)
+            tarinfo.size = len(content)
+            tar.addfile(tarinfo, io.BytesIO(content)) 
+    
+    # Return the tar buffer
+    tar_buffer.seek(0)  # Reset buffer position to beginning
+    return tar_buffer
+
+def tar_submission(metadata, documents_obj_list, compression_type=None, level=None):
+    """Takes a list of documents, compresses them, then tars them."""
+    document_tuple_list = [(doc.content, doc.accession) for doc in documents_obj_list]
+    document_tuple_list_compressed = compress_content_list(document_tuple_list,  # Fixed: correct parameter name
+                                                          compression_type=compression_type, 
+                                                          level=level)
+
+    return tar_content_list(metadata, document_tuple_list_compressed)

{datamule-2.3.4/datamule/document → datamule-2.3.6/datamule}/tables/tables.py

@@ -1,5 +1,5 @@
 from .tables_ownership import config_ownership
-from .tables_13fhr import mapping_13fhr 
+from .tables_13fhr import config_13fhr
 from .tables_informationtable import config_information_table
 from .tables_25nse import config_25nse
 from .tables_npx import config_npx
@@ -18,10 +18,10 @@ all_tables_dict = {
     '4/A' : config_ownership,
     '5' : config_ownership,
     '5/A' : config_ownership,
-    '13F-HR' : mapping_13fhr,
-    '13F-HR/A' : mapping_13fhr,
-    '13F-NT' : mapping_13fhr,
-    '13F-NT/A' : mapping_13fhr,
+    '13F-HR' : config_13fhr,
+    '13F-HR/A' : config_13fhr,
+    '13F-NT' : config_13fhr,
+    '13F-NT/A' : config_13fhr,
     'INFORMATION TABLE' : config_information_table,
     '25-NSE' : config_25nse,
     '25-NSE/A' : config_25nse,
@@ -155,4 +155,4 @@ class Tables():
                     matching_tables.append(table)
                     continue
         
-        return matching_tables
+        return matching_tables

{datamule-2.3.4 → datamule-2.3.6}/datamule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 2.3.4
+Version: 2.3.6
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-2.3.4 → datamule-2.3.6}/datamule.egg-info/SOURCES.txt

@@ -5,10 +5,6 @@ datamule/datasets.py
 datamule/helper.py
 datamule/index.py
 datamule/package_updater.py
-datamule/portfolio.py
-datamule/portfolio_compression_utils.py
-datamule/sheet.py
-datamule/submission.py
 datamule.egg-info/PKG-INFO
 datamule.egg-info/SOURCES.txt
 datamule.egg-info/dependency_links.txt
@@ -17,6 +13,7 @@ datamule.egg-info/top_level.txt
 datamule/book/__init__.py
 datamule/book/book.py
 datamule/book/s3transfer.py
+datamule/cloud/__init__.py
 datamule/data/listed_filer_metadata.csv
 datamule/datamule/__init__.py
 datamule/datamule/datamule_lookup.py
@@ -25,21 +22,13 @@ datamule/datamule/downloader.py
 datamule/datamule/sec_connector.py
 datamule/document/__init__.py
 datamule/document/document.py
-datamule/document/tables/__init__.py
-datamule/document/tables/tables.py
-datamule/document/tables/tables_13fhr.py
-datamule/document/tables/tables_25nse.py
-datamule/document/tables/tables_informationtable.py
-datamule/document/tables/tables_npx.py
-datamule/document/tables/tables_ownership.py
-datamule/document/tables/tables_proxyvotingrecord.py
-datamule/document/tables/tables_sbsef.py
-datamule/document/tables/tables_sdr.py
-datamule/document/tables/utils.py
 datamule/mapping_dicts/__init__.py
 datamule/mapping_dicts/html_mapping_dicts.py
 datamule/mapping_dicts/txt_mapping_dicts.py
 datamule/mapping_dicts/xml_mapping_dicts.py
+datamule/portfolio/__init__.py
+datamule/portfolio/portfolio.py
+datamule/portfolio/portfolio_compression_utils_legacy.py
 datamule/sec/__init__.py
 datamule/sec/utils.py
 datamule/sec/infrastructure/__init__.py
@@ -57,7 +46,22 @@ datamule/sec/xbrl/streamcompanyfacts.py
 datamule/sec/xbrl/xbrlmonitor.py
 datamule/seclibrary/__init__.py
 datamule/seclibrary/bq.py
-datamule/sentiment/__init__.py
+datamule/sheet/__init__.py
+datamule/sheet/sheet.py
+datamule/submission/__init__.py
+datamule/submission/submission.py
+datamule/submission/tar_submission.py
+datamule/tables/__init__.py
+datamule/tables/tables.py
+datamule/tables/tables_13fhr.py
+datamule/tables/tables_25nse.py
+datamule/tables/tables_informationtable.py
+datamule/tables/tables_npx.py
+datamule/tables/tables_ownership.py
+datamule/tables/tables_proxyvotingrecord.py
+datamule/tables/tables_sbsef.py
+datamule/tables/tables_sdr.py
+datamule/tables/utils.py
 datamule/tags/__init__.py
 datamule/tags/config.py
 datamule/tags/dictionaries.py

{datamule-2.3.4 → datamule-2.3.6}/setup.py

@@ -32,7 +32,7 @@ if not os.path.exists(file_path):
 setup(
     name="datamule",
     author="John Friedman",
-    version="2.3.4",
+    version="2.3.6",
     description="Work with SEC submissions at scale.",
     packages=find_packages(include=['datamule', 'datamule.*']),
     url="https://github.com/john-friedman/datamule-python",