epstein-files 1.0.1__py3-none-any.whl → 1.0.3__py3-none-any.whl

epstein_files/epstein_files.py

@@ -19,14 +19,15 @@ from epstein_files.documents.emails.email_header import AUTHOR
 from epstein_files.documents.json_file import JsonFile
 from epstein_files.documents.messenger_log import MSG_REGEX, MessengerLog
 from epstein_files.documents.other_file import OtherFile
+from epstein_files.util.constant.output_files import PICKLED_PATH
 from epstein_files.util.constant.strings import *
-from epstein_files.util.constant.urls import (EPSTEIN_WEB, JMAIL, epsteinify_name_url, epstein_web_person_url,
-     search_jmail_url, search_twitter_url)
+from epstein_files.util.constant.urls import (EPSTEIN_MEDIA, EPSTEIN_WEB, JMAIL, epstein_media_person_url,
+     epsteinify_name_url, epstein_web_person_url, search_jmail_url, search_twitter_url)
 from epstein_files.util.constants import *
 from epstein_files.util.data import dict_sets_to_lists, json_safe, sort_dict
-from epstein_files.util.doc_cfg import EmailCfg
+from epstein_files.util.doc_cfg import EmailCfg, Metadata
 from epstein_files.util.env import args, logger
-from epstein_files.util.file_helper import DOCS_DIR, PICKLED_PATH, file_size_str
+from epstein_files.util.file_helper import DOCS_DIR, file_size_str
 from epstein_files.util.highlighted_group import get_info_for_name, get_style_for_name
 from epstein_files.util.rich import (DEFAULT_NAME_STYLE, NA_TXT, add_cols_to_table, console, highlighter,
      link_text_obj, link_markup, print_author_header, print_centered, print_other_site_link, print_panel,
@@ -37,7 +38,7 @@ from epstein_files.util.timer import Timer
 DEVICE_SIGNATURE = 'Device Signature'
 DEVICE_SIGNATURE_PADDING = (1, 0)
 NOT_INCLUDED_EMAILERS = [e.lower() for e in (USELESS_EMAILERS + [JEFFREY_EPSTEIN])]
-SLOW_FILE_SECONDS = 0.4
+SLOW_FILE_SECONDS = 1.0
 
 INVALID_FOR_EPSTEIN_WEB = JUNK_EMAILERS + KRASSNER_RECIPIENTS + [
     'ACT for America',
@@ -54,6 +55,7 @@ class EpsteinFiles:
     imessage_logs: list[MessengerLog] = field(default_factory=list)
     json_files: list[JsonFile] = field(default_factory=list)
     other_files: list[OtherFile] = field(default_factory=list)
+    timer: Timer = field(default_factory=lambda: Timer())
 
     # Analytics / calculations
     email_author_counts: dict[str | None, int] = field(default_factory=lambda: defaultdict(int))
@@ -66,6 +68,7 @@ class EpsteinFiles:
         """Iterate through files and build appropriate objects."""
         self.all_files = [f for f in DOCS_DIR.iterdir() if f.is_file() and not f.name.startswith('.')]
         documents = []
+        file_type_count = defaultdict(int)
 
         # Read through and classify all the files
         for file_arg in self.all_files:
@@ -73,12 +76,13 @@ class EpsteinFiles:
             document = Document(file_arg)
 
             if document.length == 0:
-                logger.warning(f"Skipping empty file: {document}")
+                logger.warning(f"Skipping empty file: {document}]")
                 continue
 
             cls = document_cls(document)
             documents.append(cls(file_arg, text=document.text))
             logger.info(str(documents[-1]))
+            file_type_count[cls.__name__] += 1
 
             if doc_timer.seconds_since_start() > SLOW_FILE_SECONDS:
                 doc_timer.print_at_checkpoint(f"Slow file: {documents[-1]} processed")
@@ -90,17 +94,18 @@ class EpsteinFiles:
         self._tally_email_data()
 
     @classmethod
-    def get_files(cls, timer: Timer | None = None) -> 'EpsteinFiles':
+    def get_files(cls, timer: Timer | None = None, use_pickled: bool = False) -> 'EpsteinFiles':
         """Alternate constructor that reads/writes a pickled version of the data ('timer' arg is for logging)."""
         timer = timer or Timer()
 
-        if (args.pickled and PICKLED_PATH.exists()) and not args.overwrite_pickle:
+        if ((args.pickled or use_pickled) and PICKLED_PATH.exists()) and not args.overwrite_pickle:
             with gzip.open(PICKLED_PATH, 'rb') as file:
                 epstein_files = pickle.load(file)
                 timer.print_at_checkpoint(f"Loaded {len(epstein_files.all_files):,} documents from '{PICKLED_PATH}' ({file_size_str(PICKLED_PATH)})")
+                epstein_files.timer = timer
                 return epstein_files
 
-        epstein_files = EpsteinFiles()
+        epstein_files = EpsteinFiles(timer=timer)
 
         if args.overwrite_pickle or not PICKLED_PATH.exists():
             with gzip.open(PICKLED_PATH, 'wb') as file:
@@ -192,42 +197,39 @@ class EpsteinFiles:
     def imessage_logs_for(self, author: str | None | list[str | None]) -> Sequence[MessengerLog]:
         return MessengerLog.logs_for(author, self.imessage_logs)
 
-    def identified_imessage_log_count(self) -> int:
-        return len([log for log in self.imessage_logs if log.author])
-
     def json_metadata(self) -> str:
+        """Create a JSON string containing metadata for all the files."""
         metadata = {
-            EMAIL_CLASS: [json_safe(doc.metadata()) for doc in self.emails],
-            MESSENGER_LOG_CLASS: [json_safe(doc.metadata()) for doc in self.imessage_logs],
-            OTHER_FILE_CLASS: [json_safe(doc.metadata()) for doc in self.other_files],
+            EMAIL_CLASS: _sorted_metadata(self.emails),
+            JSON_FILE_CLASS: _sorted_metadata(self.json_files),
+            MESSENGER_LOG_CLASS: _sorted_metadata(self.imessage_logs),
+            OTHER_FILE_CLASS: _sorted_metadata(self.non_json_other_files()),
         }
 
         return json.dumps(metadata, indent=4, sort_keys=True)
 
-    def print_files_summary(self) -> None:
-        other_files = [doc for doc in self.other_files if not isinstance(doc, JsonFile)]
-        dupes = defaultdict(int)
-
-        for doc in self.all_documents():
-            if doc.is_duplicate:
-                dupes[doc.class_name()] += 1
+    def non_json_other_files(self) -> list[OtherFile]:
+        return [doc for doc in self.other_files if not isinstance(doc, JsonFile)]
 
+    def print_files_summary(self) -> None:
         table = Table(title='Summary of Document Types')
         add_cols_to_table(table, ['File Type', 'Files', 'Author Known', 'Author Unknown', 'Duplicates'])
 
-        def add_row(label: str, docs: list, known: int | None = None, dupes: int | None = None):
+        def add_row(label: str, docs: list):
+            known = None if isinstance(docs[0], JsonFile) else Document.known_author_count(docs)
+
             table.add_row(
                 label,
                 f"{len(docs):,}",
-                f"{known:,}" if known else NA_TXT,
-                f"{len(docs) - known:,}" if known else NA_TXT,
-                f"{dupes:,}" if dupes else NA_TXT,
+                f"{known:,}" if known is not None else NA_TXT,
+                f"{len(docs) - known:,}" if known is not None else NA_TXT,
+                f"{len([d for d in docs if d.is_duplicate])}",
             )
 
-        add_row('iMessage Logs', self.imessage_logs, self.identified_imessage_log_count())
-        add_row('Emails', self.emails, len([e for e in self.emails if e.author]), dupes[EMAIL_CLASS])
-        add_row('JSON Data', self.json_files, dupes=0)
-        add_row('Other', other_files, dupes=dupes[OTHER_FILE_CLASS])
+        add_row('iMessage Logs', self.imessage_logs)
+        add_row('Emails', self.emails)
+        add_row('JSON Data', self.json_files)
+        add_row('Other', self.non_json_other_files())
         console.print(Align.center(table))
         console.line()
 
@@ -272,7 +274,7 @@ class EpsteinFiles:
     def print_emailer_counts_table(self) -> None:
         footer = f"Identified authors of {self.attributed_email_count():,} emails out of {len(self.emails):,}."
         counts_table = Table(title=f"Email Counts", caption=footer, header_style="bold")
-        add_cols_to_table(counts_table, ['Name', 'Count', 'Sent', "Recv'd", JMAIL, EPSTEIN_WEB, 'Twitter'])
+        add_cols_to_table(counts_table, ['Name', 'Count', 'Sent', "Recv'd", JMAIL, EPSTEIN_MEDIA, EPSTEIN_WEB, 'Twitter'])
 
         emailer_counts = {
             emailer: self.email_author_counts[emailer] + self.email_recipient_counts[emailer]
@@ -288,7 +290,8 @@ class EpsteinFiles:
                 str(self.email_author_counts[p]),
                 str(self.email_recipient_counts[p]),
                 '' if p is None else link_text_obj(search_jmail_url(p), JMAIL),
-                '' if not is_ok_for_epstein_web(p) else link_text_obj(epstein_web_person_url(p), EPSTEIN_WEB.lower()),
+                '' if not is_ok_for_epstein_web(p) else link_text_obj(epstein_media_person_url(p), EPSTEIN_MEDIA),
+                '' if not is_ok_for_epstein_web(p) else link_text_obj(epstein_web_person_url(p), EPSTEIN_WEB),
                 '' if p is None else link_text_obj(search_twitter_url(p), 'search X'),
             )
 
@@ -297,7 +300,7 @@ class EpsteinFiles:
     def print_imessage_summary(self) -> None:
         """Print summary table and stats for text messages."""
         console.print(MessengerLog.summary_table(self.imessage_logs))
-        text_summary_msg = f"\nDeanonymized {self.identified_imessage_log_count()} of "
+        text_summary_msg = f"\nDeanonymized {Document.known_author_count(self.imessage_logs)} of "
         text_summary_msg += f"{len(self.imessage_logs)} {TEXT_MESSAGE} logs found in {len(self.all_files):,} files."
         console.print(text_summary_msg)
         imessage_msg_count = sum([len(log.messages()) for log in self.imessage_logs])
@@ -357,6 +360,18 @@ def build_signature_table(keyed_sets: dict[str, set[str]], cols: tuple[str, str]
     return Padding(table, DEVICE_SIGNATURE_PADDING)
 
 
+def count_by_month(docs: Sequence[Document]) -> dict[str | None, int]:
+    counts: dict[str | None, int] = defaultdict(int)
+
+    for doc in docs:
+        if doc.timestamp:
+            counts[doc.timestamp.date().isoformat()[0:7]] += 1
+        else:
+            counts[None] += 1
+
+    return counts
+
+
 def document_cls(document: Document) -> Type[Document]:
     search_area = document.text[0:5000]  # Limit search area to avoid pointless scans of huge files
 
@@ -382,13 +397,6 @@ def is_ok_for_epstein_web(name: str | None) -> bool:
     return True
 
 
-def count_by_month(docs: Sequence[Document]) -> dict[str | None, int]:
-    counts: dict[str | None, int] = defaultdict(int)
-
-    for doc in docs:
-        if doc.timestamp:
-            counts[doc.timestamp.date().isoformat()[0:7]] += 1
-        else:
-            counts[None] += 1
-
-    return counts
+def _sorted_metadata(docs: Sequence[Document]) -> list[Metadata]:
+    docs_sorted_by_id = sorted(docs, key=lambda d: d.file_id)
+    return [json_safe(d.metadata()) for d in docs_sorted_by_id]

epstein_files/util/constant/names.py

@@ -1,6 +1,5 @@
 from epstein_files.util.constant.strings import QUESTION_MARKS, remove_question_marks
 
-
 UNKNOWN = '(unknown)'
 
 # Texting Names
@@ -170,6 +169,7 @@ ZUBAIR_KHAN = 'Zubair Khan'
 
 # No communications but name is in the files
 BILL_GATES = 'Bill Gates'
+DONALD_TRUMP = 'Donald Trump'
 ELON_MUSK = 'Elon Musk'
 HENRY_HOLT = 'Henry Holt'  # Actually a company?
 IVANKA = 'Ivanka'
@@ -195,16 +195,13 @@ INSIGHTS_POD = f"InsightsPod"  # Zubair bots
 NEXT_MANAGEMENT = 'Next Management LLC'
 JP_MORGAN = 'JP Morgan'
 OSBORNE_LLP = f"{IAN_OSBORNE} & Partners LLP"  # Ian Osborne's PR firm
+TRUMP_ORG = 'Trump Organization'
 UBS = 'UBS'
 
-# Locations
-PALM_BEACH = 'Palm Beach'
-VIRGIN_ISLANDS = 'Virgin Islands'
-
 # First and last names that should be made part of a highlighting regex for emailers
 NAMES_TO_NOT_HIGHLIGHT: list[str] = [name.lower() for name in [
-    'Al', 'Alfredo', 'Allen', 'Alex', 'Alexander', 'Amanda', 'Andres', 'Andrew',
-    'Bard', 'Barry', 'Bill', 'Black', 'Brad', 'Bruce',
+    'Al', 'Alan', 'Alfredo', 'Allen', 'Alex', 'Alexander', 'Amanda', 'Andres', 'Andrew',
+    'Bard', 'Barry', 'Bill', 'Black', 'Boris', 'Brad', 'Bruce',
     'Carolyn', 'Chris', 'Christina',
     'Dan', 'Daniel', 'Danny', 'Darren', 'Dave', 'David',
     'Ed', 'Edward', 'Edwards', 'Epstein', 'Eric', 'Erika', 'Etienne',
@@ -214,10 +211,10 @@ NAMES_TO_NOT_HIGHLIGHT: list[str] = [name.lower() for name in [
     'Ian',
     'Jack', 'James', 'Jay', 'Jean', 'Jeff', 'Jeffrey', 'Jennifer', 'Jeremy', 'jessica', 'Joel', 'John', 'Jon', 'Jonathan', 'Joseph', 'Jr',
     'Kahn', 'Katherine', 'Ken', 'Kevin',
-    'Leon', 'Lesley', 'Linda', 'Link', 'Lisa',
+    'Larry', 'Leon', 'Lesley', 'Linda', 'Link', 'Lisa',
     'Mann', 'Marc', 'Marie', 'Mark', 'Martin', 'Melanie', 'Michael', 'Mike', 'Miller', 'Mitchell', 'Miles', 'Morris', 'Moskowitz',
     'Nancy', 'Neal', 'New',
-    'Paul', 'Paula', 'Pen', 'Peter', 'Philip',
+    'Paul', 'Paula', 'Pen', 'Peter', 'Philip', 'Prince',
     'Randall', 'Reid', 'Richard', 'Robert', 'Rodriguez', 'Roger', 'Rosenberg', 'Ross', 'Roth', 'Rubin',
     'Scott', 'Sean', 'Stanley', 'Stern', 'Stephen', 'Steve', 'Steven', 'Stone', 'Susan',
     'The', 'Thomas', 'Tim', 'Tom', 'Tyler',
@@ -227,25 +224,25 @@ NAMES_TO_NOT_HIGHLIGHT: list[str] = [name.lower() for name in [
 ]]
 
 # Names to color white in the word counts
-OTHER_NAMES = """
-    aaron albert alberto alec alex alexandra alice allen anderson andre andres ann anna anne ariana arthur
-    baldwin barack barbro barry ben benjamin berger bert binant bob bonner boyden brad bradley brady branson bruce bruno burton
-    chapman charles charlie chris christopher clint cohen colin collins conway
-    dave davis dean debra deborah dennis diana diane diaz dickinson dixon dominique don dylan
-    ed edmond elizabeth emily entwistle erik erika etienne evelyn
+OTHER_NAMES = NAMES_TO_NOT_HIGHLIGHT + """
+    aaron albert alberto alec alexandra alice anderson andre ann anna anne ariana arthur
+    baldwin barack ben benjamin berger bert binant bob bonner boyden bradley brady branson bruno bryant burton
+    chapman charles charlie christopher clint cohen colin collins conway
+    davis dean debra deborah dennis diana diane diaz dickinson dixon dominique don dylan
+    edmond elizabeth emily entwistle erik evelyn
     ferguson flachsbart francis franco frank
-    gardner gary geoff geoffrey george gilbert goldberg gonzalez gould graham greene guarino gwyneth
+    gardner gary geoff geoffrey gilbert gloria goldberg gonzalez gould graham greene guarino gwyneth
     hancock harold harrison harry helen hirsch hofstadter horowitz hussein
     isaac isaacson
-    jamie james jane janet jason jen jennifer jim joe joel johnson jones julie justin
-    kate kathy kelly kevin kim kruger kyle
-    leonard lenny lieberman louis lynch lynn
-    marcus marianne matt matthew melissa michele michelle mike mitchell moore moscowitz
+    jamie jane janet jason jen jim joe johnson jones josh julie justin
+    kate kathy kelly kim kruger kyle
+    leo leonard lenny leslie lieberman louis lynch lynn
+    marcus marianne matt matthew melissa michele michelle moore moscowitz
     nicole nussbaum
-    paul paula paulson philip philippe
-    rafael ray richardson rob robin rodriguez ron rudolph ryan
-    sara sarah seligman serge sergey silverman sloman smith snowden sorkin stanley steele stevie stewart susan
-    ted theresa thompson tiffany tim timothy tom
+    paulson philippe
+    rafael ray richardson rob robin ron rudolph ryan
+    sara sarah seligman serge sergey silverman sloman smith snowden sorkin steele stevie stewart
+    ted theresa thompson tiffany timothy
     valeria
     walter warren weinstein weiss william
     zach zack

epstein_files/util/constant/output_files.py

@@ -0,0 +1,29 @@
+from pathlib import Path
+
+PICKLED_PATH = Path("the_epstein_files.pkl.gz")
+
+EPSTEIN_FILES_NOV_2025 = 'epstein_files_nov_2025'
+URLS_ENV = '.urls.env'
+
+HTML_DIR = Path('docs')
+ALL_EMAILS_PATH = HTML_DIR.joinpath(f'all_emails_{EPSTEIN_FILES_NOV_2025}.html')
+JSON_METADATA_PATH = HTML_DIR.joinpath(f'file_metadata_{EPSTEIN_FILES_NOV_2025}.json')
+TEXT_MSGS_HTML_PATH = HTML_DIR.joinpath('index.html')
+WORD_COUNT_HTML_PATH = HTML_DIR.joinpath(f'communication_word_count_{EPSTEIN_FILES_NOV_2025}.html')
+# EPSTEIN_WORD_COUNT_HTML_PATH = HTML_DIR.joinpath('epstein_texts_and_emails_word_count.html')
+
+BUILD_ARTIFACTS = [
+    ALL_EMAILS_PATH,
+    # EPSTEIN_WORD_COUNT_HTML_PATH,
+    JSON_METADATA_PATH,
+    TEXT_MSGS_HTML_PATH,
+    WORD_COUNT_HTML_PATH,
+]
+
+
+def make_clean() -> None:
+    """Delete all build artifacts."""
+    for build_file in BUILD_ARTIFACTS:
+        if build_file.exists():
+            print(f"Removing build file '{build_file}'...")
+            build_file.unlink()

epstein_files/util/constant/strings.py

@@ -30,6 +30,10 @@ REPUTATION = 'reputation'
 SOCIAL = 'social'
 SPEECH = 'speech'
 
+# Locations
+PALM_BEACH = 'Palm Beach'
+VIRGIN_ISLANDS = 'Virgin Islands'
+
 # Publications
 BBC = 'BBC'
 BLOOMBERG = 'Bloomberg'
@@ -38,10 +42,12 @@ DAILY_MAIL = 'Daily Mail'
 DAILY_TELEGRAPH = "Daily Telegraph"
 LA_TIMES = 'LA Times'
 MIAMI_HERALD = 'Miami Herald'
-NYT_ARTICLE = 'NYT article about'
-NYT_COLUMN = 'NYT column about'
+NYT = "New York Times"
+PALM_BEACH_DAILY_NEWS = f'{PALM_BEACH} Daily News'
+PALM_BEACH_POST = f'{PALM_BEACH} Post'
 THE_REAL_DEAL = 'The Real Deal'
 WAPO = 'WaPo'
+VI_DAILY_NEWS = f'{VIRGIN_ISLANDS} Daily News'
 
 # Site types
 EMAIL = 'email'

epstein_files/util/constant/urls.py

@@ -5,8 +5,9 @@ from typing import Literal
 from inflection import parameterize
 from rich.text import Text
 
+from epstein_files.util.constant.output_files import *
 from epstein_files.util.constant.strings import EMAIL, TEXT_MESSAGE, SiteType
-from epstein_files.util.file_helper import JSON_METADATA_PATH, WORD_COUNT_HTML_PATH, coerce_file_stem
+from epstein_files.util.file_helper import coerce_file_stem
 
 # Style stuff
 ARCHIVE_LINK_COLOR = 'slate_blue3'
@@ -21,15 +22,17 @@ EPSTEINIFY = 'epsteinify'
 JMAIL = 'Jmail'
 
 
-# Cryptadamus URLs
+# Deployment URLS
+# NOTE: don't rename these variables without changing deploy.sh!
 GH_PAGES_BASE_URL = 'https://michelcrypt4d4mus.github.io'
-TEXT_MSGS_BASE_URL = f"{GH_PAGES_BASE_URL}/epstein_text_messages"
-JSON_METADATA_URL = f'{TEXT_MSGS_BASE_URL}/{JSON_METADATA_PATH.name}'
-WORD_COUNT_URL = f'{TEXT_MSGS_BASE_URL}/{WORD_COUNT_HTML_PATH.name}'
+TEXT_MSGS_URL = f"{GH_PAGES_BASE_URL}/epstein_text_messages"
+ALL_EMAILS_URL = f'{TEXT_MSGS_URL}/{ALL_EMAILS_PATH.name}'
+JSON_METADATA_URL = f'{TEXT_MSGS_URL}/{JSON_METADATA_PATH.name}'
+WORD_COUNT_URL = f'{TEXT_MSGS_URL}/{WORD_COUNT_HTML_PATH.name}'
 
 SITE_URLS: dict[SiteType, str] = {
-    EMAIL: f'{GH_PAGES_BASE_URL}/epstein_emails_house_oversight/',  # TODO should just be same repo
-    TEXT_MESSAGE: TEXT_MSGS_BASE_URL,
+    EMAIL: ALL_EMAILS_URL,
+    TEXT_MESSAGE: TEXT_MSGS_URL,
 }
 
 GH_PROJECT_URL = 'https://github.com/michelcrypt4d4mus/epstein_text_messages'
@@ -68,6 +71,7 @@ epsteinify_name_url = lambda name: f"{EPSTEINIFY_URL}/?name={urllib.parse.quote(
 epstein_media_doc_url = lambda file_stem: build_doc_url(DOC_LINK_BASE_URLS[EPSTEIN_MEDIA], file_stem, True)
 epstein_media_doc_link_markup = lambda filename_or_id, style = TEXT_LINK: external_doc_link_markup(EPSTEIN_MEDIA, filename_or_id, style)
 epstein_media_doc_link_txt = lambda filename_or_id, style = TEXT_LINK: Text.from_markup(epstein_media_doc_link_markup(filename_or_id, style))
+epstein_media_person_url = lambda person: f"{EPSTEIN_MEDIA_URL}/people/{parameterize(person)}"
 
 epstein_web_doc_url = lambda file_stem: f"{DOC_LINK_BASE_URLS[EPSTEIN_WEB]}/{file_stem}.jpg"
 epstein_web_person_url = lambda person: f"{EPSTEIN_WEB_URL}/{parameterize(person)}"