epstein-files 1.0.0__py3-none-any.whl → 1.0.2__py3-none-any.whl

epstein_files/__init__.py

@@ -10,185 +10,125 @@ from sys import exit
 
 from dotenv import load_dotenv
 load_dotenv()
+
+from rich.markup import escape
 from rich.padding import Padding
+from rich.panel import Panel
 
+from epstein_files.epstein_files import EpsteinFiles, document_cls
+from epstein_files.documents.document import INFO_PADDING, Document
 from epstein_files.documents.email import Email
-from epstein_files.epstein_files import EpsteinFiles, count_by_month
 from epstein_files.util.constant.html import *
 from epstein_files.util.constant.names import *
-from epstein_files.util.constant.strings import EMAIL_CLASS, MESSENGER_LOG_CLASS
-from epstein_files.util.data import Timer, dict_sets_to_lists, flatten
-from epstein_files.util.env import specified_names, args
-from epstein_files.util.file_helper import GH_PAGES_HTML_PATH
-from epstein_files.util.rich import *
-
-PRINT_COLOR_KEY_EVERY_N_EMAILS = 150
-
-# Order matters (will be order of output)
-PEOPLE_WHOSE_EMAILS_SHOULD_BE_PRINTED: list[str | None] = [
-    JEREMY_RUBIN,
-    AL_SECKEL,
-    JOI_ITO,
-    JABOR_Y,
-    STEVEN_SINOFSKY,
-    DANIEL_SIAD,
-    JEAN_LUC_BRUNEL,
-    STEVEN_HOFFENBERG,
-    EHUD_BARAK,
-    MARTIN_NOWAK,
-    MASHA_DROKOVA,
-    RENATA_BOLOTOVA,
-    STEVE_BANNON,
-    OLIVIER_COLOM,
-    BORIS_NIKOLIC,
-    PRINCE_ANDREW,
-    JIDE_ZEITLIN,
-    DAVID_STERN,
-    MOHAMED_WAHEED_HASSAN,
-    JENNIFER_JACQUET,
-    None,
-]
-
-# Order matters (will be order of output)
-PEOPLE_WHOSE_EMAILS_SHOULD_BE_TABLES: list[str | None] = [
-    GHISLAINE_MAXWELL,
-    LEON_BLACK,
-    LANDON_THOMAS,
-    KATHRYN_RUEMMLER,
-    DARREN_INDYKE,
-    RICHARD_KAHN,
-    TYLER_SHEARS,
-    SULTAN_BIN_SULAYEM,
-    DEEPAK_CHOPRA,
-    ARIANE_DE_ROTHSCHILD,
-    TOM_PRITZKER,
-]
+from epstein_files.util.constant.output_files import ALL_EMAILS_PATH, TEXT_MSGS_HTML_PATH, make_clean
+from epstein_files.util.env import args, specified_names
+from epstein_files.util.file_helper import coerce_file_path, extract_file_id
+from epstein_files.util.logging import logger
+from epstein_files.util.output import print_emails, print_json_metadata, print_json_stats, print_text_messages, write_urls
+from epstein_files.util.rich import build_highlighter, console, print_header, print_panel, write_html
+from epstein_files.util.timer import Timer
 
 
 def generate_html() -> None:
+    if args.make_clean:
+        make_clean()
+        exit()
+
     timer = Timer()
     epstein_files = EpsteinFiles.get_files(timer)
+
+    if args.json_metadata:
+        print_json_metadata(epstein_files)
+        exit()
+
     print_header(epstein_files)
 
     if args.colors_only:
         exit()
 
-    # Text messages section
     if args.output_texts:
         print_text_messages(epstein_files)
-        timer.print_at_checkpoint(f'Printed {len(epstein_files.imessage_logs):,} text message logs')
+        timer.print_at_checkpoint(f'Printed {len(epstein_files.imessage_logs)} text message logs')
 
-    # Emails section
     if args.output_emails:
         emails_printed = print_emails(epstein_files)
         timer.print_at_checkpoint(f"Printed {emails_printed:,} emails")
 
     if args.output_other_files:
-        epstein_files.print_other_files_table()
-        timer.print_at_checkpoint(f"Printed {len(epstein_files.other_files):,} other files")
-    else:
-        logger.warning(f"Skipping other files section...")
+        files_printed = epstein_files.print_other_files_table()
+        timer.print_at_checkpoint(f"Printed {len(files_printed)} other files")
 
     # Save output
-    write_html(GH_PAGES_HTML_PATH)
-    logger.warning(f"Total time: {timer.seconds_since_start()}")
+    write_html(ALL_EMAILS_PATH if args.all_emails else TEXT_MSGS_HTML_PATH)
+    logger.warning(f"Total time: {timer.seconds_since_start_str()}")
 
     # JSON stats (mostly used for building pytest checks)
     if args.json_stats:
-        console.line(5)
         print_json_stats(epstein_files)
 
 
-def print_emails(epstein_files: EpsteinFiles) -> int:
-    """Returns number of emails printed."""
-    print_section_header(('Selections from ' if not args.all_emails else '') + 'His Emails')
-    print_other_site_link(is_header=False)
-
-    if len(specified_names) == 0:
-        epstein_files.print_emailer_counts_table()
-
-    emailers_to_print: list[str | None]
-    emailer_tables: list[str | None] = []
-    emails_that_were_printed: list[Email] = []
-    num_emails_printed_since_last_color_key = 0
+def epstein_diff():
+    """Diff the cleaned up text of two files."""
+    Document.diff_files(args.positional_args)
 
-    if args.all_emails:
-        console.print('Email conversations are sorted chronologically based on time of the first email.')
-        emailers_to_print = sorted(epstein_files.all_emailers(), key=lambda e: epstein_files.earliest_email_at(e))
-        print_numbered_list_of_emailers(emailers_to_print, epstein_files)
-    else:
-        if len(specified_names) > 0:
-            emailers_to_print = specified_names
-        else:
-            emailers_to_print = PEOPLE_WHOSE_EMAILS_SHOULD_BE_PRINTED
-
-        console.print('Email conversations grouped by counterparty can be found in the order listed below.')
-        print_numbered_list_of_emailers(emailers_to_print)
-        console.print("\nAfter that there's tables linking to (but not displaying) all known emails for each of these people:")
-
-        if len(specified_names) > 0:
-            if args.all_email_tables:
-                emailer_tables = sorted(epstein_files.all_emailers(), key=lambda e: epstein_files.earliest_email_at(e))
-            else:
-                emailer_tables = PEOPLE_WHOSE_EMAILS_SHOULD_BE_TABLES
-
-            print_numbered_list_of_emailers(emailer_tables)
 
-    for author in emailers_to_print:
-        newly_printed_emails = epstein_files.print_emails_for(author)
-        emails_that_were_printed.extend(newly_printed_emails)
-        num_emails_printed_since_last_color_key += len(newly_printed_emails)
+def epstein_search():
+    """Search the cleaned up text of the files."""
+    _assert_positional_args()
+    epstein_files = EpsteinFiles.get_files(use_pickled=True)
 
-        # Print color key every once in a while
-        if num_emails_printed_since_last_color_key > PRINT_COLOR_KEY_EVERY_N_EMAILS:
-            print_color_key()
-            num_emails_printed_since_last_color_key = 0
-
-    if len(emailer_tables) > 0 and len(specified_names) == 0:
-        print_author_header(f"Email Tables for {len(emailer_tables)} Other People", 'white')
+    for search_term in args.positional_args:
+        temp_highlighter = build_highlighter(search_term)
+        search_results = epstein_files.docs_matching(search_term, specified_names)
+        console.line(2)
+        print_panel(f"Found {len(search_results)} documents matching '{search_term}'", padding=(0, 0, 0, 3))
 
-        for name in emailer_tables:
-            epstein_files.print_emails_table_for(name)
+        for search_result in search_results:
+            console.line()
 
-    if len(specified_names) == 0:
-        epstein_files.print_email_device_info()
+            if args.whole_file:
+                console.print(search_result.document)
+            else:
+                console.print(search_result.document.description_panel())
 
-    logger.warning(f"Rewrote {len(Email.rewritten_header_ids)} headers of {len(epstein_files.emails)} emails")
+                for matching_line in search_result.lines:
+                    line_txt = matching_line.__rich__()
+                    console.print(Padding(temp_highlighter(line_txt), INFO_PADDING), style='gray37')
 
-    if args.all_emails:
-        email_ids_that_were_printed = set([email.file_id for email in emails_that_were_printed])
-        logger.warning(f"Printed {len(emails_that_were_printed)} emails of {len(email_ids_that_were_printed)} unique file IDs.")
 
-        for email in epstein_files.emails:
-            if email.file_id not in email_ids_that_were_printed and not email.is_duplicate:
-                logger.warning(f"Failed to print {email.description()}")
+def epstein_show():
+    """Show the color highlighted file. If --raw arg is passed, show the raw text of the file as well."""
+    _assert_positional_args()
+    ids = [extract_file_id(arg) for arg in args.positional_args]
+    console.line()
 
-    return len(emails_that_were_printed)
+    if args.pickled:
+        epstein_files = EpsteinFiles.get_files(use_pickled=True)
+        docs = epstein_files.get_documents_by_id(ids)
+    else:
+        raw_docs = [Document(coerce_file_path(id)) for id in ids]
+        docs = [document_cls(doc)(doc.file_path) for doc in raw_docs]
 
+    for doc in docs:
+        console.line()
+        console.print(doc)
 
-def print_text_messages(epstein_files: EpsteinFiles) -> None:
-    print_section_header('Text Messages')
-    print_centered("(conversations are sorted chronologically based on timestamp of first message)\n", style='gray30')
+        if args.raw:
+            console.line()
+            console.print(Panel(f"*** {doc.url_slug} RAW ***", expand=False, style=doc._border_style()))
+            console.print(escape(doc.raw_text()))
 
-    if len(specified_names) == 0:
-        log_files = epstein_files.imessage_logs
-    else:
-        log_files = flatten([epstein_files.imessage_logs_for(name) for name in specified_names])
+            if isinstance(doc, Email):
+                console.line()
+                console.print(Panel(f"*** {doc.url_slug} actual_text ***", expand=False, style=doc._border_style()))
+                console.print(escape(doc._actual_text()))
 
-    for log_file in log_files:
-        console.print(Padding(log_file))
-        console.line(2)
 
-    epstein_files.print_imessage_summary()
+def epstein_dump_urls() -> None:
+    write_urls()
 
 
-def print_json_stats(epstein_files: EpsteinFiles) -> None:
-    console.print(Panel('JSON Stats Dump', expand=True, style='reverse bold'), '\n')
-    print_json(f"{MESSENGER_LOG_CLASS} Sender Counts", epstein_files.imessage_sender_counts(), skip_falsey=True)
-    print_json(f"{EMAIL_CLASS} Author Counts", epstein_files.email_author_counts, skip_falsey=True)
-    print_json(f"{EMAIL_CLASS} Recipient Counts", epstein_files.email_recipient_counts, skip_falsey=True)
-    print_json("Email signature_substitution_countss", epstein_files.email_signature_substitution_counts(), skip_falsey=True)
-    print_json("email_author_device_signatures", dict_sets_to_lists(epstein_files.email_authors_to_device_signatures))
-    print_json("email_sent_from_devices", dict_sets_to_lists(epstein_files.email_device_signatures_to_authors))
-    print_json("email_unknown_recipient_file_ids", epstein_files.email_unknown_recipient_file_ids())
-    print_json("count_by_month", count_by_month(epstein_files.all_documents()))
+def _assert_positional_args():
+    if not args.positional_args:
+        console.print(f"\n  ERROR: No positional args!\n", style='red1')
+        exit(1)

epstein_files/documents/communication.py

@@ -8,7 +8,7 @@ from rich.text import Text
 from epstein_files.documents.document import CLOSE_PROPERTIES_CHAR, Document
 from epstein_files.util.constant.names import UNKNOWN
 from epstein_files.util.constants import FALLBACK_TIMESTAMP
-from epstein_files.util.file_cfg import MessageCfg
+from epstein_files.util.doc_cfg import CommunicationCfg
 from epstein_files.util.highlighted_group import get_style_for_name
 from epstein_files.util.rich import key_value_txt
 
@@ -20,7 +20,7 @@ class Communication(Document):
     """Superclass for Email and MessengerLog."""
     author_style: str = 'white'
     author_txt: Text = field(init=False)
-    config: MessageCfg | None = None
+    config: CommunicationCfg | None = None
     timestamp: datetime = FALLBACK_TIMESTAMP  # TODO this default sucks (though it never happens)
 
     def __post_init__(self):
@@ -31,22 +31,22 @@ class Communication(Document):
     def author_or_unknown(self) -> str:
         return self.author or UNKNOWN
 
-    def description(self) -> Text:
-        return self._description().append(CLOSE_PROPERTIES_CHAR)
-
-    def is_attribution_uncertain(self) -> bool | None:
-        return self.config and self.config.is_attribution_uncertain
+    def is_attribution_uncertain(self) -> bool:
+        return bool(self.config and self.config.is_attribution_uncertain)
 
     def raw_document_link_txt(self, _style: str = '', include_alt_link: bool = True) -> Text:
         """Overrides super() method to apply self.author_style."""
         return super().raw_document_link_txt(self.author_style, include_alt_link=include_alt_link)
 
+    def summary(self) -> Text:
+        return self._summary().append(CLOSE_PROPERTIES_CHAR)
+
     def timestamp_without_seconds(self) -> str:
         return TIMESTAMP_SECONDS_REGEX.sub('', str(self.timestamp))
 
-    def _description(self) -> Text:
+    def _summary(self) -> Text:
         """One line summary mostly for logging."""
-        txt = super().description().append(', ')
+        txt = super().summary().append(', ')
         return txt.append(key_value_txt('author', Text(f"'{self.author_or_unknown()}'", style=self.author_style)))