epstein-files 1.0.14__py3-none-any.whl → 1.0.16__py3-none-any.whl

epstein_files/__init__.py

@@ -17,11 +17,11 @@ from epstein_files.epstein_files import EpsteinFiles, document_cls
 from epstein_files.documents.document import INFO_PADDING, Document
 from epstein_files.documents.email import Email
 from epstein_files.util.constant.output_files import ALL_EMAILS_PATH, TEXT_MSGS_HTML_PATH, make_clean
-from epstein_files.util.env import args, specified_names
+from epstein_files.util.env import args
 from epstein_files.util.file_helper import coerce_file_path, extract_file_id
 from epstein_files.util.logging import logger
-from epstein_files.util.output import (print_emails, print_json_files, print_json_stats,
-     write_json_metadata, write_urls)
+from epstein_files.util.output import (print_emails_section, print_json_files, print_json_stats,
+     print_other_files_section, print_text_messages_section, write_json_metadata, write_urls)
 from epstein_files.util.rich import build_highlighter, console, print_header, print_panel, write_html
 from epstein_files.util.timer import Timer
 from epstein_files.util.word_count import write_word_counts_html
@@ -49,12 +49,12 @@ def generate_html() -> None:
         exit()
 
     if args.output_texts:
-        epstein_files.print_text_messages_section()
+        print_text_messages_section(epstein_files)
         timer.print_at_checkpoint(f'Printed {len(epstein_files.imessage_logs)} text message logs')
 
     if args.output_emails:
-        emails_printed = print_emails(epstein_files)
-        timer.print_at_checkpoint(f"Printed {emails_printed:,} emails")
+        emails_that_were_printed = print_emails_section(epstein_files)
+        timer.print_at_checkpoint(f"Printed {len(emails_that_were_printed):,} emails")
 
     if args.output_other:
         if args.uninteresting:
@@ -62,7 +62,7 @@ def generate_html() -> None:
         else:
             files = [f for f in epstein_files.other_files if args.all_other_files or f.is_interesting()]
 
-        epstein_files.print_other_files_section(files)
+        print_other_files_section(files, epstein_files)
         timer.print_at_checkpoint(f"Printed {len(files)} other files (skipped {len(epstein_files.other_files) - len(files)})")
 
     # Save output
@@ -86,7 +86,7 @@ def epstein_search():
 
     for search_term in args.positional_args:
         temp_highlighter = build_highlighter(search_term)
-        search_results = epstein_files.docs_matching(search_term, specified_names)
+        search_results = epstein_files.docs_matching(search_term, args.names)
         console.line(2)
         print_panel(f"Found {len(search_results)} documents matching '{search_term}'", padding=(0, 0, 0, 3))
 

epstein_files/documents/document.py

@@ -160,8 +160,8 @@ class Document:
     def file_size(self) -> int:
         return file_size(self.file_path)
 
-    def file_size_str(self) -> str:
-        return file_size_str(self.file_path)
+    def file_size_str(self, decimal_places: int | None = None) -> str:
+        return file_size_str(self.file_path, decimal_places)
 
     def info(self) -> list[Text]:
         """0 to 2 sentences containing the info_txt() as well as any configured description."""
@@ -171,14 +171,14 @@ class Document:
         ])
 
     def info_txt(self) -> Text | None:
-        """Secondary info about this file (recipients, level of certainty, etc). Overload in subclasses."""
+        """Secondary info about this file (description recipients, etc). Overload in subclasses."""
         return None
 
     def is_duplicate(self) -> bool:
         return bool(self.config and self.config.duplicate_of_id)
 
     def is_local_extract_file(self) -> bool:
-        """True if file created by extracting text from a court doc (identifiable from filename e.g. HOUSE_OVERSIGHT_012345_1.txt)."""
+        """True if extracted from other file (identifiable from filename e.g. HOUSE_OVERSIGHT_012345_1.txt)."""
         return is_local_extract_file(self.filename)
 
     def length(self) -> int:
@@ -234,6 +234,7 @@ class Document:
         return text
 
     def sort_key(self) -> tuple[datetime, str, int]:
+        """Sort by timestamp, file_id, then whether or not it's a duplicate file."""
         if self.is_duplicate():
             sort_id = self.config.duplicate_of_id
             dupe_idx = 1
@@ -253,7 +254,7 @@ class Document:
             txt.append(' (', style=SYMBOL_STYLE)
             txt.append(f"{timestamp_str}", style=TIMESTAMP_DIM).append(')', style=SYMBOL_STYLE)
 
-        txt.append(' [').append(key_value_txt('size', Text(self.file_size_str(), style='aquamarine1')))
+        txt.append(' [').append(key_value_txt('size', Text(self.file_size_str(0), style='aquamarine1')))
         txt.append(", ").append(key_value_txt('lines', self.num_lines()))
 
         if self.config and self.config.duplicate_of_id:
@@ -271,6 +272,7 @@ class Document:
         return Panel(Group(*sentences), border_style=self._class_style(), expand=False)
 
     def top_lines(self, n: int = 10) -> str:
+        """First n lines."""
         return '\n'.join(self.lines[0:n])[:MAX_TOP_LINES_LEN]
 
     def warn(self, msg: str) -> None:

epstein_files/documents/email.py

@@ -17,7 +17,7 @@ from epstein_files.documents.document import CLOSE_PROPERTIES_CHAR, INFO_INDENT
 from epstein_files.documents.emails.email_header import (BAD_EMAILER_REGEX, EMAIL_SIMPLE_HEADER_REGEX,
      EMAIL_SIMPLE_HEADER_LINE_BREAK_REGEX, FIELD_NAMES, TIME_REGEX, EmailHeader)
 from epstein_files.util.constant.names import *
-from epstein_files.util.constant.strings import REDACTED, URL_SIGNIFIERS
+from epstein_files.util.constant.strings import REDACTED
 from epstein_files.util.constants import *
 from epstein_files.util.data import (TIMEZONE_INFO, collapse_newlines, escape_single_quotes, extract_last_name,
      flatten, remove_timezone, uniquify)
@@ -41,6 +41,7 @@ LOCAL_EXTRACT_REGEX = re.compile(r"_\d$")
 
 SUPPRESS_LOGS_FOR_AUTHORS = ['Undisclosed recipients:', 'undisclosed-recipients:', 'Multiple Senders Multiple Senders']
 REWRITTEN_HEADER_MSG = "(janky OCR header fields were prettified, check source if something seems off)"
+URL_SIGNIFIERS = ['gclid', 'htm', 'ref=', 'utm']
 APPEARS_IN = 'Appears in'
 MAX_CHARS_TO_PRINT = 4000
 MAX_NUM_HEADER_LINES = 14
@@ -134,6 +135,7 @@ JUNK_EMAILERS = [
 ]
 
 MAILING_LISTS = [
+    CAROLYN_RANGEL,
     INTELLIGENCE_SQUARED,
     'middle.east.update@hotmail.com',
     JP_MORGAN_USGIO,
@@ -341,6 +343,9 @@ class Email(Communication):
             else:
                 for recipient in self.header.recipients():
                     self.recipients.extend(self._emailer_names(recipient))
+
+                if self.author in MAILING_LISTS and (len(self.recipients) == 0 or self.recipients == [self.author]):
+                    self.recipients = [JEFFREY_EPSTEIN]   # Assume mailing list emails are to Epstein
         except Exception as e:
             console.print_exception()
             console.line(2)
@@ -589,7 +594,7 @@ class Email(Communication):
             self._merge_lines(3)  # Merge 4th and 5th rows
         elif self.file_id in '026609 029402 032405 022695'.split():
             self._merge_lines(4)  # Merge 5th and 6th rows
-        elif self.file_id in ['019407', '031980', '030384', '033144', '030999', '033575', '029835', '030381', '033357']:
+        elif self.file_id in ['019407', '031980', '030384', '033144', '030999', '033575', '029835', '030381', '033357', '026924']:
             self._merge_lines(2, 4)
         elif self.file_id in ['029154', '029163']:
             self._merge_lines(2, 5)
@@ -692,7 +697,7 @@ class Email(Communication):
             self.config.description = extracted_description
 
         self.config.is_interesting = self.config.is_interesting or extracted_from_doc_cfg.is_interesting
-        self.warn(f"Constructed synthetic config: {self.config}")
+        self.log(f"Constructed synthetic config: {self.config}")
 
     def __rich_console__(self, console: Console, options: ConsoleOptions) -> RenderResult:
         logger.debug(f"Printing '{self.filename}'...")

epstein_files/documents/imessage/text_message.py

@@ -75,7 +75,7 @@ class TextMessage:
         return msg_txt
 
     def __rich__(self) -> Text:
+        timestamp_txt = Text(f"[{self.timestamp_str}]", style=TIMESTAMP_DIM).append(' ')
         author_style = get_style_for_name(self.author_str if self.author_str.startswith('+') else self.author)
         author_txt = Text(self.author_str, style=author_style)
-        timestamp_txt = Text(f"[{self.timestamp_str}]", style=TIMESTAMP_DIM).append(' ')
         return Text('').append(timestamp_txt).append(author_txt).append(': ', style='dim').append(self._message())

epstein_files/documents/messenger_log.py

@@ -12,7 +12,7 @@ from epstein_files.documents.communication import Communication
 from epstein_files.documents.imessage.text_message import TextMessage
 from epstein_files.util.constant.names import JEFFREY_EPSTEIN, UNKNOWN
 from epstein_files.util.constant.strings import AUTHOR, TIMESTAMP_STYLE
-from epstein_files.util.data import days_between, days_between_str, iso_timestamp, listify, sort_dict
+from epstein_files.util.data import days_between, days_between_str, iso_timestamp, sort_dict
 from epstein_files.util.doc_cfg import Metadata, TextCfg
 from epstein_files.util.highlighted_group import get_style_for_name
 from epstein_files.util.logging import logger
@@ -121,23 +121,22 @@ class MessengerLog(Communication):
         return sender_counts
 
     @classmethod
-    def logs_for(cls, author: str | None | list[str | None], logs: list['MessengerLog']) -> list['MessengerLog']:
-        authors = listify(author)
-        return logs if JEFFREY_EPSTEIN in authors else [log for log in logs if log.author in authors]
-
-    @classmethod
-    def summary_table(cls, imessage_logs: list['MessengerLog']) -> Table:
+    def summary_table(cls, log_files: list['MessengerLog']) -> Table:
         """Build a table summarizing the text messages in 'imessage_logs'."""
-        counts_table = build_table("Text Message Counts By Author")
-        counts_table.add_column(AUTHOR.title(), justify='left', style="steel_blue bold", width=30)
+        author_counts = cls.count_authors(log_files)
+        msg_count = sum([len(log.messages) for log in log_files])
+
+        footer = f"Deanonymized {msg_count - author_counts[None]:,} of {msg_count:,} text messages in"
+        counts_table = build_table("Text Message Counts By Author", caption=f"{footer} {len(log_files)} files")
+        counts_table.add_column(AUTHOR.title(), justify='left', width=30)
         counts_table.add_column('Files', justify='right', style='white')
         counts_table.add_column("Msgs", justify='right')
         counts_table.add_column('First Sent At', justify='center', highlight=True, width=21)
         counts_table.add_column('Last Sent At', justify='center', style=LAST_TIMESTAMP_STYLE, width=21)
         counts_table.add_column('Days', justify='right', style='dim')
 
-        for name, count in sort_dict(cls.count_authors(imessage_logs)):
-            logs = cls.logs_for(name, imessage_logs)
+        for name, count in sort_dict(author_counts):
+            logs = log_files if name == JEFFREY_EPSTEIN else [log for log in log_files if log.author == name]
             first_at = logs[0].first_message_at(name)
             last_at = logs[-1].first_message_at(name)
 

epstein_files/documents/other_file.py

@@ -17,14 +17,15 @@ from rich.text import Text
 from epstein_files.documents.document import CLOSE_PROPERTIES_CHAR, WHITESPACE_REGEX, Document
 from epstein_files.util.constant.strings import *
 from epstein_files.util.constants import *
-from epstein_files.util.doc_cfg import FINANCIAL_REPORTS_AUTHORS, DocCfg, Metadata
+from epstein_files.util.doc_cfg import DocCfg, Metadata
 from epstein_files.util.data import days_between, escape_single_quotes, remove_timezone, sort_dict, uniquify
 from epstein_files.util.file_helper import FILENAME_LENGTH, file_size_to_str
 from epstein_files.util.env import args
 from epstein_files.util.highlighted_group import styled_category
-from epstein_files.util.rich import QUESTION_MARK_TXT, add_cols_to_table, build_table, highlighter
+from epstein_files.util.rich import QUESTION_MARK_TXT, build_table, highlighter
 from epstein_files.util.logging import logger
 
+FIRST_FEW_LINES = 'First Few Lines'
 MAX_DAYS_SPANNED_TO_BE_VALID = 10
 MAX_EXTRACTED_TIMESTAMPS = 100
 MIN_TIMESTAMP = datetime(2000, 1, 1)
@@ -208,6 +209,36 @@ class OtherFile(Document):
         if num_days_spanned > MAX_DAYS_SPANNED_TO_BE_VALID and VAST_HOUSE not in self.text:
             self.log_top_lines(15, msg=timestamps_log_msg, level=logging.DEBUG)
 
+    @staticmethod
+    def count_by_category_table(files: Sequence['OtherFile']) -> Table:
+        counts = defaultdict(int)
+        category_bytes = defaultdict(int)
+
+        for file in files:
+            if file.category() is None:
+                logger.warning(f"file {file.file_id} has no category")
+
+            counts[file.category()] += 1
+            category_bytes[file.category()] += file.file_size()
+
+        table = build_table('Other Files Summary', ['Category', 'Count', 'Has Author', 'No Author', 'Size'])
+        table.columns[0].min_width = 14
+        table.columns[-1].style = 'dim'
+
+        for (category, count) in sort_dict(counts):
+            category_files = [f for f in files if f.category() == category]
+            known_author_count = Document.known_author_count(category_files)
+
+            table.add_row(
+                styled_category(category or UNKNOWN),
+                str(count),
+                str(known_author_count),
+                str(count - known_author_count),
+                file_size_to_str(category_bytes[category]),
+            )
+
+        return table
+
     @staticmethod
     def files_preview_table(files: Sequence['OtherFile']) -> Table:
         """Build a table of OtherFile documents."""
@@ -240,33 +271,3 @@ class OtherFile(Document):
             )
 
         return table
-
-    @staticmethod
-    def count_by_category_table(files: Sequence['OtherFile']) -> Table:
-        counts = defaultdict(int)
-        category_bytes = defaultdict(int)
-
-        for file in files:
-            if file.category() is None:
-                logger.warning(f"file {file.file_id} has no category")
-
-            counts[file.category()] += 1
-            category_bytes[file.category()] += file.file_size()
-
-        table = build_table('Other Files Summary', ['Category', 'Count', 'Has Author', 'No Author', 'Size'])
-        table.columns[0].min_width = 14
-        table.columns[-1].style = 'dim'
-
-        for (category, count) in sort_dict(counts):
-            category_files = [f for f in files if f.category() == category]
-            known_author_count = Document.known_author_count(category_files)
-
-            table.add_row(
-                styled_category(category or UNKNOWN),
-                str(count),
-                str(known_author_count),
-                str(count - known_author_count),
-                file_size_to_str(category_bytes[category]),
-            )
-
-        return table

epstein_files/epstein_files.py

@@ -25,11 +25,11 @@ from epstein_files.util.constant.urls import (EPSTEIN_MEDIA, EPSTEIN_WEB, JMAIL,
 from epstein_files.util.constants import *
 from epstein_files.util.data import days_between, dict_sets_to_lists, json_safe, listify, sort_dict
 from epstein_files.util.doc_cfg import EmailCfg, Metadata
-from epstein_files.util.env import DOCS_DIR, args, logger, specified_names
+from epstein_files.util.env import DOCS_DIR, args, logger
 from epstein_files.util.file_helper import file_size_str
 from epstein_files.util.highlighted_group import HIGHLIGHTED_NAMES, HighlightedNames, get_info_for_name, get_style_for_name
 from epstein_files.util.rich import (DEFAULT_NAME_STYLE, LAST_TIMESTAMP_STYLE, NA_TXT, add_cols_to_table,
-     print_all_files_page_link, build_table, console, highlighter, link_text_obj, link_markup, print_author_header, print_centered,
+     print_other_page_link, build_table, console, highlighter, link_text_obj, link_markup, print_author_header, print_centered,
      print_panel, print_section_header, vertically_pad)
 from epstein_files.util.search_result import SearchResult
 from epstein_files.util.timer import Timer
@@ -201,9 +201,6 @@ class EpsteinFiles:
 
         return docs
 
-    def imessage_logs_for(self, author: str | None | list[str | None]) -> Sequence[MessengerLog]:
-        return MessengerLog.logs_for(author, self.imessage_logs)
-
     def json_metadata(self) -> str:
         """Create a JSON string containing metadata for all the files."""
         metadata = {
@@ -290,44 +287,6 @@ class EpsteinFiles:
         console.print(_build_signature_table(self.email_authors_to_device_signatures, (AUTHOR, DEVICE_SIGNATURE)))
         console.print(_build_signature_table(self.email_device_signatures_to_authors, (DEVICE_SIGNATURE, AUTHOR), ', '))
 
-    def print_other_files_section(self, files: list[OtherFile]) -> None:
-        """Returns the OtherFile objects that were interesting enough to print."""
-        category_table = OtherFile.count_by_category_table(files)
-        other_files_preview_table = OtherFile.files_preview_table(files)
-        header_pfx = '' if args.all_other_files else 'Selected '
-        print_section_header(f"{FIRST_FEW_LINES} of {len(files)} {header_pfx}Files That Are Neither Emails Nor Text Messages")
-
-        if args.all_other_files:
-            console.line(1)
-        else:
-            print_all_files_page_link(self)
-            console.line(2)
-
-            for table in [category_table, other_files_preview_table]:
-                table.title = f"{header_pfx}{table.title}"
-
-        print_centered(category_table)
-        console.line(2)
-        console.print(other_files_preview_table)
-
-    def print_text_messages_section(self) -> None:
-        """Print summary table and stats for text messages."""
-        print_section_header('All of His Text Messages')
-        print_centered("(conversations are sorted chronologically based on timestamp of first message)\n", style='gray30')
-        authors: list[str | None] = specified_names if specified_names else [JEFFREY_EPSTEIN]
-        log_files = self.imessage_logs_for(authors)
-
-        for log_file in log_files:
-            console.print(Padding(log_file))
-            console.line(2)
-
-        print_centered(MessengerLog.summary_table(self.imessage_logs))
-        text_summary_msg = f"\nDeanonymized {Document.known_author_count(self.imessage_logs)} of "
-        text_summary_msg += f"{len(self.imessage_logs)} {TEXT_MESSAGE} logs found in {len(self.all_files):,} files."
-        console.print(text_summary_msg)
-        imessage_msg_count = sum([len(log.messages) for log in self.imessage_logs])
-        console.print(f"Found {imessage_msg_count} text messages in {len(self.imessage_logs)} iMessage log files.")
-
     def table_of_emailers(self) -> Table:
         attributed_emails = [e for e in self.non_duplicate_emails() if e.author]
         footer = f"Identified authors of {len(attributed_emails):,} out of {len(self.non_duplicate_emails()):,} emails."

epstein_files/util/constant/names.py

@@ -38,6 +38,7 @@ BRAD_EDWARDS = 'Brad Edwards'
 BRAD_KARP = 'Brad Karp'
 BRAD_WECHSLER = 'Brad Wechsler'
 BORIS_NIKOLIC = 'Boris Nikolic'
+CAROLYN_RANGEL = 'Carolyn Rangel'
 CECILE_DE_JONGH = 'Cecile de Jongh'
 CECILIA_STEEN = 'Cecilia Steen'
 CELINA_DUBIN = 'Celina Dubin'
@@ -187,9 +188,11 @@ VIRGINIA_GIUFFRE = 'Virginia Giuffre'
 
 # Organizations
 BOFA = 'BofA'
+BOFA_MERRILL = f'{BOFA} / Merrill Lynch'
 CNN = 'CNN'
 DEUTSCHE_BANK = 'Deutsche Bank'
 ELECTRON_CAPITAL_PARTNERS = 'Electron Capital Partners'
+EPSTEIN_FOUNDATION = 'Jeffrey Epstein VI Foundation'
 GOLDMAN_SACHS = 'Goldman Sachs'
 GOLDMAN_INVESTMENT_MGMT = f'{GOLDMAN_SACHS} Investment Management Division'
 HARVARD = 'Harvard'

epstein_files/util/constant/strings.py

@@ -57,12 +57,10 @@ TIMESTAMP_DIM = f"turquoise4 dim"
 AUTHOR = 'author'
 DEFAULT = 'default'
 EVERYONE = 'everyone'
-FIRST_FEW_LINES = 'First Few Lines'
 HOUSE_OVERSIGHT_PREFIX = 'HOUSE_OVERSIGHT_'
 JSON = 'json'
 NA = 'n/a'
 REDACTED = '<REDACTED>'
-URL_SIGNIFIERS = ['gclid', 'htm', 'ref=', 'utm']
 QUESTION_MARKS = '(???)'
 
 # Regexes

epstein_files/util/constant/urls.py

@@ -6,6 +6,7 @@ from inflection import parameterize
 from rich.text import Text
 
 from epstein_files.util.constant.output_files import *
+from epstein_files.util.env import args
 from epstein_files.util.file_helper import coerce_file_stem
 
 # Style stuff
@@ -49,7 +50,7 @@ DOC_LINK_BASE_URLS: dict[ExternalSite, str] = {
 }
 
 
-epsteinify_api_url = lambda file_id: f"{EPSTEINIFY_URL}/api/documents/HOUSE_OVERSIGHT_{file_id}"
+epsteinify_api_url = lambda file_stem: f"{EPSTEINIFY_URL}/api/documents/{file_stem}"
 epsteinify_doc_link_markup = lambda filename_or_id, style = TEXT_LINK: external_doc_link_markup(EPSTEINIFY, filename_or_id, style)
 epsteinify_doc_link_txt = lambda filename_or_id, style = TEXT_LINK: Text.from_markup(external_doc_link_markup(filename_or_id, style))
 epsteinify_doc_url = lambda file_stem: build_doc_url(DOC_LINK_BASE_URLS[EPSTEINIFY], file_stem)
@@ -66,8 +67,6 @@ epstein_web_search_url = lambda s: f"{EPSTEIN_WEB_URL}/?ewmfileq={urllib.parse.q
 
 rollcall_doc_url = lambda file_stem: build_doc_url(DOC_LINK_BASE_URLS[ROLLCALL], file_stem, 'title')
 
-search_archive_url = lambda txt: f"{COURIER_NEWSROOM_ARCHIVE_URL}&q={urllib.parse.quote(txt)}&p=1"
-search_coffeezilla_url = lambda txt: f"{COFFEEZILLA_ARCHIVE_URL}&q={urllib.parse.quote(txt)}&p=1"
 search_jmail_url = lambda txt: f"{JMAIL_URL}/search?q={urllib.parse.quote(txt)}"
 search_twitter_url = lambda txt: f"https://x.com/search?q={urllib.parse.quote(txt)}&src=typed_query&f=live"
 
@@ -103,8 +102,12 @@ def link_text_obj(url: str, link_text: str | None = None, style: str = ARCHIVE_L
     return Text.from_markup(link_markup(url, link_text, style))
 
 
-def search_coffeezilla_link(text: str, link_txt: str, style: str = ARCHIVE_LINK_COLOR) -> Text:
-    return link_text_obj(search_coffeezilla_url(text), link_txt or text, style)
+def other_site_type() -> SiteType:
+    return TEXT_MESSAGE if args.all_emails else EMAIL
+
+
+def other_site_url() -> str:
+    return SITE_URLS[other_site_type()]
 
 
 CRYPTADAMUS_TWITTER = link_markup('https://x.com/cryptadamist', '@cryptadamist')