epstein-files 1.2.1__py3-none-any.whl → 1.4.1__py3-none-any.whl

epstein_files/__init__.py

@@ -16,15 +16,19 @@ from rich.text import Text
 from epstein_files.epstein_files import EpsteinFiles, document_cls
 from epstein_files.documents.document import INFO_PADDING, Document
 from epstein_files.documents.email import Email
+from epstein_files.documents.messenger_log import MessengerLog
+from epstein_files.documents.other_file import OtherFile
 from epstein_files.util.constant.output_files import make_clean
+from epstein_files.util.constant.strings import ID_REGEX
+from epstein_files.util.data import flatten
 from epstein_files.util.env import args
 from epstein_files.util.file_helper import coerce_file_path, extract_file_id
 from epstein_files.util.logging import exit_with_error, logger
 from epstein_files.util.output import (print_emails_section, print_json_files, print_json_stats,
      print_other_files_section, print_text_messages_section, print_email_timeline, print_emailers_info,
      print_json_metadata, write_urls)
-from epstein_files.util.rich import (build_highlighter, console, print_color_key, print_title_page_header,
-     print_title_page_tables, print_subtitle_panel, write_html)
+from epstein_files.util.rich import (build_highlighter, console, highlighter, print_color_key, print_json,
+     print_title_page_header, print_title_page_tables, print_subtitle_panel, write_html)
 from epstein_files.util.timer import Timer
 from epstein_files.util.word_count import write_word_counts_html
 
@@ -76,6 +80,9 @@ def generate_html() -> None:
     write_html(args.build)
     logger.warning(f"Total time: {timer.seconds_since_start_str()}")
 
+    if args.debug:
+        highlighter.print_highlight_counts(console)
+
     # JSON stats (mostly used for building pytest checks)
     if args.json_stats:
         print_json_stats(epstein_files)
@@ -86,27 +93,54 @@ def epstein_diff():
     Document.diff_files(args.positional_args)
 
 
-def epstein_search():
+def epstein_grep():
     """Search the cleaned up text of the files."""
     epstein_files = EpsteinFiles.get_files()
 
+    if ID_REGEX.match(args.positional_args[0]):
+        logger.warning(f"'{args.positional_args[0]}' seems to be an ID, running epstein_show instead...")
+        epstein_show()
+        return
+
     for search_term in args.positional_args:
         temp_highlighter = build_highlighter(search_term)
         search_results = epstein_files.docs_matching(search_term, args.names)
         print_subtitle_panel(f"Found {len(search_results)} documents matching '{search_term}'")
+        last_document = None
 
         for search_result in search_results:
-            console.line()
-
-            if args.whole_file:
-                console.print(search_result.document)
+            doc = search_result.document
+            lines = search_result.lines
+
+            if (isinstance(doc, Email) and not args.output_emails) \
+                    or (isinstance(doc, OtherFile) and not args.output_other) \
+                    or (isinstance(doc, MessengerLog) and not args.output_texts):
+                doc.log(f"{type(doc).__name__} Skipping search result...")
+                continue
+            elif isinstance(doc, Email) and args.email_body:
+                lines = [l for l in search_result.lines if l.line_number > doc.header.num_header_rows]
+
+                if not lines:
+                    doc.log(f"None of the matches for '{search_term}' seem to be in the body of the email")
+                    continue
+
+            if doc.is_duplicate():
+                if last_document and not last_document.is_duplicate():
+                    console.line()
+
+                last_document = doc
+                console.print(doc.duplicate_file_txt())
+            elif args.whole_file:
+                console.print(doc)
             else:
-                console.print(search_result.document.summary_panel())
+                console.print(doc.summary_panel())
 
-                for matching_line in search_result.lines:
+                for matching_line in lines:
                     line_txt = matching_line.__rich__()
                     console.print(Padding(temp_highlighter(line_txt), INFO_PADDING), style='gray37')
 
+            console.line()
+
 
 def epstein_show():
     """Show the color highlighted file. If --raw arg is passed, show the raw text of the file as well."""
@@ -114,8 +148,13 @@ def epstein_show():
     console.line()
 
     try:
-        ids = [extract_file_id(arg) for arg in args.positional_args]
-        raw_docs = [Document(coerce_file_path(id)) for id in ids]
+        if args.names:
+            people = EpsteinFiles.get_files().person_objs(args.names)
+            raw_docs = [doc for doc in flatten([p.emails for p in people])]
+        else:
+            ids = [extract_file_id(arg.strip().strip('_')) for arg in args.positional_args]
+            raw_docs = [Document(coerce_file_path(id)) for id in ids]
+
         docs = Document.sort_by_timestamp([document_cls(doc)(doc.file_path) for doc in raw_docs])
     except Exception as e:
         exit_with_error(str(e))
@@ -130,6 +169,11 @@ def epstein_show():
             if isinstance(doc, Email):
                 console.print(Panel(Text("actual_text: ").append(doc.summary()), expand=False, style=doc._border_style()))
                 console.print(escape(doc._actual_text()), '\n')
+                metadata = doc.metadata()
+                metadata['is_fwded_article'] = doc.is_fwded_article()
+                metadata['is_word_count_worthy'] = doc.is_word_count_worthy()
+                metadata['_is_first_for_user'] = doc._is_first_for_user
+                print_json(f"{doc.file_id} Metadata", metadata)
 
 
 def epstein_word_count() -> None:

epstein_files/documents/document.py

@@ -197,6 +197,9 @@ class Document:
     def is_duplicate(self) -> bool:
         return bool(self.duplicate_of_id())
 
+    def is_interesting(self) -> bool:
+        return bool(self.config and self.config.is_interesting)
+
     def is_local_extract_file(self) -> bool:
         """True if extracted from other file (identifiable from filename e.g. HOUSE_OVERSIGHT_012345_1.txt)."""
         return is_local_extract_file(self.filename)
@@ -267,11 +270,11 @@ class Document:
             txt.append(' (', style=SYMBOL_STYLE)
             txt.append(f"{timestamp_str}", style=TIMESTAMP_DIM).append(')', style=SYMBOL_STYLE)
 
-        txt.append(' [').append(key_value_txt('size', Text(self.file_size_str(0), style='aquamarine1')))
+        txt.append(' [').append(key_value_txt('size', Text(str(self.length()), style='aquamarine1')))
         txt.append(", ").append(key_value_txt('lines', self.num_lines()))
 
         if self.config and self.config.duplicate_of_id:
-            txt.append(", ").append(key_value_txt('dupe_of', Text(self.config.duplicate_of_id, style='magenta')))
+            txt.append(", ").append(key_value_txt('dupe_of', Text(self.config.duplicate_of_id, style='cyan dim')))
 
         return txt
 
@@ -435,6 +438,14 @@ class Document:
         """Count of how many Document objects have an author attribution."""
         return len([doc for doc in docs if doc.author])
 
+    @staticmethod
+    def sort_by_id(docs: Sequence['DocumentType']) -> list['DocumentType']:
+        return sorted(docs, key=lambda d: d.file_id)
+
+    @staticmethod
+    def sort_by_length(docs: Sequence['DocumentType']) -> list['DocumentType']:
+        return sorted(docs, key=lambda d: d.file_size(), reverse=True)
+
     @staticmethod
     def sort_by_timestamp(docs: Sequence['DocumentType']) -> list['DocumentType']:
         return sorted(docs, key=lambda doc: doc.timestamp_sort_key())