epstein-files 1.2.5__tar.gz → 1.5.0__tar.gz

{epstein_files-1.2.5 → epstein_files-1.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: epstein-files
-Version: 1.2.5
+Version: 1.5.0
 Summary: Tools for working with the Jeffrey Epstein documents released in November 2025.
 Home-page: https://michelcrypt4d4mus.github.io/epstein_text_messages/
 License: GPL-3.0-or-later
@@ -20,6 +20,7 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: cairosvg (>=2.8.2,<3.0.0)
 Requires-Dist: datefinder (>=0.7.3,<0.8.0)
 Requires-Dist: inflection (>=0.5.1,<0.6.0)
+Requires-Dist: pdfalyzer[extract] (>=1.19.6,<2.0.0)
 Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
 Requires-Dist: python-dotenv (>=1.2.1,<2.0.0)
 Requires-Dist: requests (>=2.32.5,<3.0.0)
@@ -47,6 +48,7 @@ Description-Content-Type: text/markdown
 #### Installation
 1. Requires you have a local copy of the OCR text files from the House Oversight document release in a directory `/path/to/epstein/ocr_txt_files`. You can download those OCR text files from [the Congressional Google Drive folder](https://drive.google.com/drive/folders/1ldncvdqIf6miiskDp_EDuGSDAaI_fJx8) (make sure you grab both the `001/` and `002/` folders).
 1. Use `poetry install` for easiest time installing. `pip install epstein-files` should also work, though `pipx install epstein-files` is usually better.
+1. (Optional) If you want to work with the documents released by DOJ on January 30th 2026 you'll need to also download the PDF collections from [the DOJ site](https://www.justice.gov/epstein/doj-disclosures) (they're in the "Epstein Files Transparency Act" section) and OCR them or find another way to get the OCR text.
 
 
 #### Command Line Tools
@@ -56,6 +58,13 @@ You need to set the `EPSTEIN_DOCS_DIR` environment variable with the path to the
 EPSTEIN_DOCS_DIR=/path/to/epstein/ocr_txt_files epstein_generate --help
 ```
 
+To work with the January 2026 DOJ documents you'll also need to set the `EPSTEIN_DOJ_TXTS_20260130_DIR` env var to point at folders full of OCR extracted texts from the raw DOJ PDFs. If you have the PDFs but not the text files there's [a script](scripts/extract_doj_pdfs.py) that can help you take care of that.
+
+```bash
+EPSTEIN_DOCS_DIR=/path/to/epstein/ocr_txt_files EPSTEIN_DOJ_TXTS_20260130_DIR=/path/to/doj/files epstein_generate --help
+```
+
+
 All the tools that come with the package require `EPSTEIN_DOCS_DIR` to be set. These are the available tools:
 
 ```bash
@@ -63,9 +72,9 @@ All the tools that come with the package require `EPSTEIN_DOCS_DIR` to be set. T
 epstein_generate
 
 # Search for a string:
-epstein_search Bannon
+epstein_grep Bannon
 # Or a regex:
-epstein_search '\bSteve\s*Bannon|Jeffrey\s*Epstein\b'
+epstein_grep '\bSteve\s*Bannon|Jeffrey\s*Epstein\b'
 
 # Show a file with color highlighting of keywords:
 epstein_show 030999
@@ -123,3 +132,7 @@ for file in epstein_files.other_files:
 # Everyone Who Sent or Received an Email in the November Document Dump
 ![emails](https://github.com/michelcrypt4d4mus/epstein_text_messages/raw/master/docs/emailers_info_table.png)
 
+
+# TODO List
+See [TODO.md](TODO.md).
+

{epstein_files-1.2.5 → epstein_files-1.5.0}/README.md

@@ -13,6 +13,7 @@
 #### Installation
 1. Requires you have a local copy of the OCR text files from the House Oversight document release in a directory `/path/to/epstein/ocr_txt_files`. You can download those OCR text files from [the Congressional Google Drive folder](https://drive.google.com/drive/folders/1ldncvdqIf6miiskDp_EDuGSDAaI_fJx8) (make sure you grab both the `001/` and `002/` folders).
 1. Use `poetry install` for easiest time installing. `pip install epstein-files` should also work, though `pipx install epstein-files` is usually better.
+1. (Optional) If you want to work with the documents released by DOJ on January 30th 2026 you'll need to also download the PDF collections from [the DOJ site](https://www.justice.gov/epstein/doj-disclosures) (they're in the "Epstein Files Transparency Act" section) and OCR them or find another way to get the OCR text.
 
 
 #### Command Line Tools
@@ -22,6 +23,13 @@ You need to set the `EPSTEIN_DOCS_DIR` environment variable with the path to the
 EPSTEIN_DOCS_DIR=/path/to/epstein/ocr_txt_files epstein_generate --help
 ```
 
+To work with the January 2026 DOJ documents you'll also need to set the `EPSTEIN_DOJ_TXTS_20260130_DIR` env var to point at folders full of OCR extracted texts from the raw DOJ PDFs. If you have the PDFs but not the text files there's [a script](scripts/extract_doj_pdfs.py) that can help you take care of that.
+
+```bash
+EPSTEIN_DOCS_DIR=/path/to/epstein/ocr_txt_files EPSTEIN_DOJ_TXTS_20260130_DIR=/path/to/doj/files epstein_generate --help
+```
+
+
 All the tools that come with the package require `EPSTEIN_DOCS_DIR` to be set. These are the available tools:
 
 ```bash
@@ -29,9 +37,9 @@ All the tools that come with the package require `EPSTEIN_DOCS_DIR` to be set. T
 epstein_generate
 
 # Search for a string:
-epstein_search Bannon
+epstein_grep Bannon
 # Or a regex:
-epstein_search '\bSteve\s*Bannon|Jeffrey\s*Epstein\b'
+epstein_grep '\bSteve\s*Bannon|Jeffrey\s*Epstein\b'
 
 # Show a file with color highlighting of keywords:
 epstein_show 030999
@@ -88,3 +96,7 @@ for file in epstein_files.other_files:
 
 # Everyone Who Sent or Received an Email in the November Document Dump
 ![emails](https://github.com/michelcrypt4d4mus/epstein_text_messages/raw/master/docs/emailers_info_table.png)
+
+
+# TODO List
+See [TODO.md](TODO.md).

{epstein_files-1.2.5 → epstein_files-1.5.0}/epstein_files/__init__.py

@@ -4,6 +4,7 @@ Reformat Epstein text message files for readability and count email senders.
 
     Run: 'EPSTEIN_DOCS_DIR=/path/to/TXT epstein_generate'
 """
+import re
 from sys import exit
 
 from dotenv import load_dotenv
@@ -15,20 +16,21 @@ from rich.text import Text
 
 from epstein_files.epstein_files import EpsteinFiles, document_cls
 from epstein_files.documents.document import INFO_PADDING, Document
+from epstein_files.documents.doj_file import DojFile
 from epstein_files.documents.email import Email
 from epstein_files.documents.messenger_log import MessengerLog
 from epstein_files.documents.other_file import OtherFile
 from epstein_files.util.constant.output_files import make_clean
-from epstein_files.util.constant.strings import ID_REGEX
+from epstein_files.util.constant.strings import HOUSE_OVERSIGHT_NOV_2025_ID_REGEX
 from epstein_files.util.data import flatten
 from epstein_files.util.env import args
 from epstein_files.util.file_helper import coerce_file_path, extract_file_id
 from epstein_files.util.logging import exit_with_error, logger
-from epstein_files.util.output import (print_emails_section, print_json_files, print_json_stats,
+from epstein_files.util.output import (print_doj_files, print_emails_section, print_json_files, print_stats,
      print_other_files_section, print_text_messages_section, print_email_timeline, print_emailers_info,
      print_json_metadata, write_urls)
-from epstein_files.util.rich import (build_highlighter, console, highlighter, print_color_key, print_title_page_header,
-     print_title_page_tables, print_subtitle_panel, write_html)
+from epstein_files.util.rich import (build_highlighter, console, highlighter, print_color_key, print_json,
+     print_title_page_header, print_title_page_tables, print_subtitle_panel, write_html)
 from epstein_files.util.timer import Timer
 from epstein_files.util.word_count import write_word_counts_html
 
@@ -62,6 +64,10 @@ def generate_html() -> None:
     if args.colors_only:
         exit()
 
+    if args.output_doj_files:
+        printed_doj_files = print_doj_files(epstein_files)
+        timer.log_section_complete('DojFile', epstein_files.doj_files, printed_doj_files)
+
     if args.output_texts:
         printed_logs = print_text_messages_section(epstein_files)
         timer.log_section_complete('MessengerLog', epstein_files.imessage_logs, printed_logs)
@@ -83,9 +89,8 @@ def generate_html() -> None:
     if args.debug:
         highlighter.print_highlight_counts(console)
 
-    # JSON stats (mostly used for building pytest checks)
-    if args.json_stats:
-        print_json_stats(epstein_files)
+    if args.stats:
+        print_stats(epstein_files)  # Used for building pytest checks
 
 
 def epstein_diff():
@@ -93,11 +98,11 @@ def epstein_diff():
     Document.diff_files(args.positional_args)
 
 
-def epstein_search():
+def epstein_grep():
     """Search the cleaned up text of the files."""
     epstein_files = EpsteinFiles.get_files()
 
-    if ID_REGEX.match(args.positional_args[0]):
+    if HOUSE_OVERSIGHT_NOV_2025_ID_REGEX.match(args.positional_args[0]):
         logger.warning(f"'{args.positional_args[0]}' seems to be an ID, running epstein_show instead...")
         epstein_show()
         return
@@ -106,26 +111,41 @@ def epstein_search():
         temp_highlighter = build_highlighter(search_term)
         search_results = epstein_files.docs_matching(search_term, args.names)
         print_subtitle_panel(f"Found {len(search_results)} documents matching '{search_term}'")
+        last_document = None
 
         for search_result in search_results:
-            document = search_result.document
+            doc = search_result.document
+            lines = search_result.lines
 
-            if (isinstance(document, Email) and not args.output_emails) \
-                    or (isinstance(document, OtherFile) and not args.output_other) \
-                    or (isinstance(document, MessengerLog) and not args.output_texts):
-                document.warn(f"{type(document).__name__} Skipping search result...")
+            if (isinstance(doc, Email) and not args.output_emails) \
+                    or (isinstance(doc, (DojFile, OtherFile)) and not args.output_other) \
+                    or (isinstance(doc, MessengerLog) and not args.output_texts):
+                doc.log(f"{type(doc).__name__} Skipping search result...")
                 continue
+            elif isinstance(doc, Email) and args.email_body:
+                lines = [l for l in search_result.lines if l.line_number > doc.header.num_header_rows]
+
+                if not lines:
+                    doc.log(f"None of the matches for '{search_term}' seem to be in the body of the email")
+                    continue
+
+            if doc.is_duplicate:
+                if last_document and not last_document.is_duplicate:
+                    console.line()
 
-            if args.whole_file:
-                console.print(document)
+                last_document = doc
+                console.print(doc.duplicate_file_txt)
+            elif args.whole_file:
+                console.print(doc)
             else:
-                console.print(document.summary_panel())
+                console.print(doc.summary_panel)
 
-                for matching_line in search_result.lines:
+                for matching_line in lines:
                     line_txt = matching_line.__rich__()
                     console.print(Padding(temp_highlighter(line_txt), INFO_PADDING), style='gray37')
 
             console.line()
+            console.print(doc.local_path_and_url + '\n', style='dim')
 
 
 def epstein_show():
@@ -138,23 +158,35 @@ def epstein_show():
             people = EpsteinFiles.get_files().person_objs(args.names)
             raw_docs = [doc for doc in flatten([p.emails for p in people])]
         else:
-            ids = [extract_file_id(arg) for arg in args.positional_args]
-            raw_docs = [Document(coerce_file_path(id)) for id in ids]
+            ids = [extract_file_id(arg.strip().strip('_')) for arg in args.positional_args]
+            logger.info(f"extracted IDs: {ids}")
+            raw_docs = [Document.from_file_id(id) for id in ids]
+            logger.info(f"raw docs: {raw_docs}")
 
+        # Rebuild the Document objs so we can see result of latest processing
         docs = Document.sort_by_timestamp([document_cls(doc)(doc.file_path) for doc in raw_docs])
+        logger.info(f"Document types: {[doc._class_name for doc in docs]}")
     except Exception as e:
+        console.print_exception()
         exit_with_error(str(e))
 
     for doc in docs:
         console.print('\n', doc, '\n')
 
         if args.raw:
-            console.print(Panel(Text("RAW: ").append(doc.summary()), expand=False, style=doc._border_style()))
+            console.print(Panel(Text("RAW: ").append(doc.summary()), expand=False, style=doc.border_style))
             console.print(escape(doc.raw_text()), '\n')
 
             if isinstance(doc, Email):
-                console.print(Panel(Text("actual_text: ").append(doc.summary()), expand=False, style=doc._border_style()))
-                console.print(escape(doc._actual_text()), '\n')
+                console.print(Panel(Text("actual_text: ").append(doc.summary()), expand=False, style=doc.border_style))
+                console.print(escape(doc._extract_actual_text()), '\n')
+                metadata = doc.metadata
+                metadata['is_fwded_article'] = doc.is_fwded_article
+                metadata['is_word_count_worthy'] = doc.is_word_count_worthy
+                metadata['_is_first_for_user'] = doc._is_first_for_user
+                print_json(f"{doc.file_id} Metadata", metadata)
+
+        console.print(doc.local_path_and_url, style='dim')
 
 
 def epstein_word_count() -> None:

{epstein_files-1.2.5 → epstein_files-1.5.0}/epstein_files/documents/communication.py

@@ -21,26 +21,30 @@ class Communication(Document):
     config: CommunicationCfg | None = None
     timestamp: datetime = FALLBACK_TIMESTAMP  # TODO this default sucks (though it never happens)
 
+    @property
     def author_or_unknown(self) -> str:
         return self.author or UNKNOWN
 
+    @property
     def author_style(self) -> str:
         return get_style_for_name(self.author)
 
+    @property
     def author_txt(self) -> Text:
         return styled_name(self.author)
 
+    @property
+    def timestamp_without_seconds(self) -> str:
+        return TIMESTAMP_SECONDS_REGEX.sub('', str(self.timestamp))
+
     def external_links_txt(self, _style: str = '', include_alt_links: bool = True) -> Text:
         """Overrides super() method to apply self.author_style."""
-        return super().external_links_txt(self.author_style(), include_alt_links=include_alt_links)
+        return super().external_links_txt(self.author_style, include_alt_links=include_alt_links)
 
     def summary(self) -> Text:
         return self._summary().append(CLOSE_PROPERTIES_CHAR)
 
-    def timestamp_without_seconds(self) -> str:
-        return TIMESTAMP_SECONDS_REGEX.sub('', str(self.timestamp))
-
     def _summary(self) -> Text:
         """One line summary mostly for logging."""
         txt = super().summary().append(', ')
-        return txt.append(key_value_txt('author', Text(f"'{self.author_or_unknown()}'", style=self.author_style())))
+        return txt.append(key_value_txt('author', Text(f"'{self.author_or_unknown}'", style=self.author_style)))