contaix 0.0.2__tar.gz

contaix-0.0.2/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) [year] [fullname]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

contaix-0.0.2/PKG-INFO

@@ -0,0 +1,207 @@
+Metadata-Version: 2.4
+Name: contaix
+Version: 0.0.2
+Summary: Tools to make contexts for AI
+Home-page: https://github.com/thorwhalen/contaix
+Author: Thor Whalen
+License: mit
+Platform: any
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: dol
+Requires-Dist: lkj
+Requires-Dist: pypdf
+Requires-Dist: mammoth
+Requires-Dist: pandas
+Requires-Dist: pptx
+Requires-Dist: html2text
+Requires-Dist: nbconvert
+Requires-Dist: nbformat
+Provides-Extra: pdf
+Requires-Dist: pypdf; extra == "pdf"
+Provides-Extra: office
+Requires-Dist: mammoth; extra == "office"
+Requires-Dist: pandas; extra == "office"
+Requires-Dist: pptx; extra == "office"
+Provides-Extra: html
+Requires-Dist: html2text; extra == "html"
+Provides-Extra: jupyter
+Requires-Dist: nbconvert; extra == "jupyter"
+Requires-Dist: nbformat; extra == "jupyter"
+Provides-Extra: all
+Requires-Dist: pypdf; extra == "all"
+Requires-Dist: mammoth; extra == "all"
+Requires-Dist: pandas; extra == "all"
+Requires-Dist: pptx; extra == "all"
+Requires-Dist: html2text; extra == "all"
+Requires-Dist: nbconvert; extra == "all"
+Requires-Dist: nbformat; extra == "all"
+Requires-Dist: python-magic; extra == "all"
+Dynamic: license-file
+
+# contaix
+
+Tools to make contexts for AI
+
+To install:	```pip install contaix```
+
+
+# Markdown Conversion
+
+This module provides tools for converting various file formats to Markdown. It supports common formats such as PDF, Word, Excel, PowerPoint, HTML, and Jupyter notebooks.
+
+## Key Features
+
+- Convert files to Markdown from bytes with format auto-detection
+- Batch convert multiple files
+- Customize output filenames and processing
+- Extensible converter system
+
+## Basic Usage
+
+### Converting a Single File
+
+The primary function `bytes_to_markdown` converts a file's bytes to Markdown text:
+
+```python
+from contaix import bytes_to_markdown
+
+# Convert with explicit format
+pdf_bytes = get_file_bytes('document.pdf')
+markdown_text = bytes_to_markdown(pdf_bytes, "pdf")
+
+# Or let the function detect the format from filename
+markdown_text = bytes_to_markdown(file_bytes, input_format=None, key="document.docx")
+
+# Or analyze the content to detect format (when no information is available)
+markdown_text = bytes_to_markdown(
+    unknown_bytes, 
+    input_format=None, 
+    key=None, 
+    try_bytes_detection=True
+)
+```
+
+### Converting Multiple Files
+
+Use `bytes_store_to_markdown_store` to process multiple files at once:
+
+```python
+from contaix import bytes_store_to_markdown_store
+from dol import Files
+
+# Convert all files in a directory
+src_files = Files('/path/to/documents')
+target_store = {}
+bytes_store_to_markdown_store(src_files, target_store)
+
+# Now target_store contains {"file1.docx.md": "converted markdown...", ...}
+```
+
+## Advanced Usage
+
+### Selective Conversion
+
+If you only want to convert specific file types:
+
+```python
+# Filter to only include certain file types
+filtered_files = {k: v for k, v in src_files.items() 
+                 if k.endswith('.docx') or k.endswith('.pdf')}
+bytes_store_to_markdown_store(filtered_files, target_store)
+```
+
+### Custom Output Naming
+
+You can control how output filenames are generated:
+
+```python
+def custom_key_transform(key):
+    # Remove the extension and add "-markdown.md"
+    base_name = os.path.splitext(key)[0]
+    return f"{base_name}-markdown.md"
+
+bytes_store_to_markdown_store(
+    src_files, 
+    target_store, 
+    old_to_new_key=custom_key_transform
+)
+```
+
+### Content Aggregation
+
+After conversion, you might want to combine all the markdown into a single document:
+
+```python
+def aggregate_content(store):
+    """Combine all markdown content into a single document with headers."""
+    result = "# Combined Markdown Document\n\n"
+    
+    for filename, content in store.items():
+        result += f"## {filename}\n\n{content}\n\n---\n\n"
+        
+    return result
+
+combined_markdown = bytes_store_to_markdown_store(
+    src_files, 
+    {}, 
+    target_store_egress=aggregate_content
+)
+```
+
+### Custom Converters
+
+You can extend the system with your own converters:
+
+```python
+def custom_txt_converter(b):
+    """A custom converter for text files."""
+    text = b.decode('utf-8', errors='ignore')
+    lines = text.split('\n')
+    result = f"# Custom Converted Text File\n\n"
+    
+    for line in lines:
+        if line.strip():
+            result += f"> {line}\n"
+    
+    return result
+
+custom_converters = {"txt": custom_txt_converter}
+
+bytes_store_to_markdown_store(
+    src_files,
+    target_store,
+    converters=custom_converters
+)
+```
+
+## Format Detection Logic
+
+The `bytes_to_markdown` function employs a prioritized strategy to find the right converter:
+
+1. **Explicit Format**: If you provide `input_format`, it uses that directly
+2. **Filename-Based**: If `input_format` is None but `key` (filename) is provided, it extracts the format from the extension
+3. **Content-Based**: If `try_bytes_detection` is True, it analyzes the bytes to determine the format
+4. **Fallback**: If no converter is found through the above methods, it uses the fallback converter
+
+This flexible approach means you can control how formats are detected based on your specific needs.
+
+## Performance Considerations
+
+- If you know the file format in advance, specifying `input_format` will be faster
+- To disable content-based detection (for better performance), set `try_bytes_detection=False`
+- When processing large batches, filtering to only include supported formats can improve efficiency
+
+## Supported Formats
+
+The module currently supports the following formats:
+
+- PDF (`.pdf`)
+- Microsoft Word (`.docx`, `.doc`)
+- Microsoft Excel (`.xlsx`, `.xls`)
+- Microsoft PowerPoint (`.pptx`, `.ppt`)
+- HTML (`.html`)
+- Jupyter Notebooks (`.ipynb`)
+- Plain text (`.txt`, `.md`)
+
+Additional formats can be supported by adding custom converters.

contaix-0.0.2/README.md

@@ -0,0 +1,166 @@
+# contaix
+
+Tools to make contexts for AI
+
+To install:	```pip install contaix```
+
+
+# Markdown Conversion
+
+This module provides tools for converting various file formats to Markdown. It supports common formats such as PDF, Word, Excel, PowerPoint, HTML, and Jupyter notebooks.
+
+## Key Features
+
+- Convert files to Markdown from bytes with format auto-detection
+- Batch convert multiple files
+- Customize output filenames and processing
+- Extensible converter system
+
+## Basic Usage
+
+### Converting a Single File
+
+The primary function `bytes_to_markdown` converts a file's bytes to Markdown text:
+
+```python
+from contaix import bytes_to_markdown
+
+# Convert with explicit format
+pdf_bytes = get_file_bytes('document.pdf')
+markdown_text = bytes_to_markdown(pdf_bytes, "pdf")
+
+# Or let the function detect the format from filename
+markdown_text = bytes_to_markdown(file_bytes, input_format=None, key="document.docx")
+
+# Or analyze the content to detect format (when no information is available)
+markdown_text = bytes_to_markdown(
+    unknown_bytes, 
+    input_format=None, 
+    key=None, 
+    try_bytes_detection=True
+)
+```
+
+### Converting Multiple Files
+
+Use `bytes_store_to_markdown_store` to process multiple files at once:
+
+```python
+from contaix import bytes_store_to_markdown_store
+from dol import Files
+
+# Convert all files in a directory
+src_files = Files('/path/to/documents')
+target_store = {}
+bytes_store_to_markdown_store(src_files, target_store)
+
+# Now target_store contains {"file1.docx.md": "converted markdown...", ...}
+```
+
+## Advanced Usage
+
+### Selective Conversion
+
+If you only want to convert specific file types:
+
+```python
+# Filter to only include certain file types
+filtered_files = {k: v for k, v in src_files.items() 
+                 if k.endswith('.docx') or k.endswith('.pdf')}
+bytes_store_to_markdown_store(filtered_files, target_store)
+```
+
+### Custom Output Naming
+
+You can control how output filenames are generated:
+
+```python
+def custom_key_transform(key):
+    # Remove the extension and add "-markdown.md"
+    base_name = os.path.splitext(key)[0]
+    return f"{base_name}-markdown.md"
+
+bytes_store_to_markdown_store(
+    src_files, 
+    target_store, 
+    old_to_new_key=custom_key_transform
+)
+```
+
+### Content Aggregation
+
+After conversion, you might want to combine all the markdown into a single document:
+
+```python
+def aggregate_content(store):
+    """Combine all markdown content into a single document with headers."""
+    result = "# Combined Markdown Document\n\n"
+    
+    for filename, content in store.items():
+        result += f"## {filename}\n\n{content}\n\n---\n\n"
+        
+    return result
+
+combined_markdown = bytes_store_to_markdown_store(
+    src_files, 
+    {}, 
+    target_store_egress=aggregate_content
+)
+```
+
+### Custom Converters
+
+You can extend the system with your own converters:
+
+```python
+def custom_txt_converter(b):
+    """A custom converter for text files."""
+    text = b.decode('utf-8', errors='ignore')
+    lines = text.split('\n')
+    result = f"# Custom Converted Text File\n\n"
+    
+    for line in lines:
+        if line.strip():
+            result += f"> {line}\n"
+    
+    return result
+
+custom_converters = {"txt": custom_txt_converter}
+
+bytes_store_to_markdown_store(
+    src_files,
+    target_store,
+    converters=custom_converters
+)
+```
+
+## Format Detection Logic
+
+The `bytes_to_markdown` function employs a prioritized strategy to find the right converter:
+
+1. **Explicit Format**: If you provide `input_format`, it uses that directly
+2. **Filename-Based**: If `input_format` is None but `key` (filename) is provided, it extracts the format from the extension
+3. **Content-Based**: If `try_bytes_detection` is True, it analyzes the bytes to determine the format
+4. **Fallback**: If no converter is found through the above methods, it uses the fallback converter
+
+This flexible approach means you can control how formats are detected based on your specific needs.
+
+## Performance Considerations
+
+- If you know the file format in advance, specifying `input_format` will be faster
+- To disable content-based detection (for better performance), set `try_bytes_detection=False`
+- When processing large batches, filtering to only include supported formats can improve efficiency
+
+## Supported Formats
+
+The module currently supports the following formats:
+
+- PDF (`.pdf`)
+- Microsoft Word (`.docx`, `.doc`)
+- Microsoft Excel (`.xlsx`, `.xls`)
+- Microsoft PowerPoint (`.pptx`, `.ppt`)
+- HTML (`.html`)
+- Jupyter Notebooks (`.ipynb`)
+- Plain text (`.txt`, `.md`)
+
+Additional formats can be supported by adding custom converters.

contaix-0.0.2/contaix/__init__.py

@@ -0,0 +1,49 @@
+"""
+Tools for working with AI contexts
+"""
+
+"""
+Tools to make contexts (knowledge bases for AI agents)
+
+Examples:
+
+Download all articles from a markdown string and save them as PDF files:
+
+>>> download_articles(md_string)  # doctest: +SKIP
+
+Verify URLs in a markdown string by checking their status codes
+(useful when trying to verify if AI hallucinated the urls)
+x`
+>>> verify_urls(md_string)  # doctest: +SKIP
+
+Make an md file with all the code in a directory:
+
+>>> md_string = code_aggregate(package_or_folder_or_github_url)  # doctest: +SKIP
+
+"""
+
+# --------------------------------------------------------------------------------------
+# General utilities
+
+from contaix.util import (
+    identity,
+    fullpath,
+    url_to_contents,
+    save_to_file_and_return_file,
+)
+from contaix.aggregation import aggregate_store
+from contaix.code import code_aggregate, PackageCodeContexts
+from contaix.urls import (
+    extract_urls,
+    verify_urls,
+    download_articles,
+    download_articles_by_section,
+)
+from contaix.markdown import (
+    bytes_to_markdown,
+    bytes_store_to_markdown_store,
+    dflt_converters,
+    add_dflt_converter,
+    truncate_text,
+    notebook_to_markdown,
+)

contaix-0.0.2/contaix/aggregation.py

@@ -0,0 +1,86 @@
+"""Tools for aggregating contexts"""
+
+from typing import Optional, Mapping, Iterable, Union, Callable
+from dol import TextFiles, filt_iter, wrap_kvs, Pipe, store_aggregate
+from contaix.util import identity
+
+
+def aggregate_store(
+    store: Mapping,
+    *,
+    min_number_of_duplicated_lines: Optional[int] = None,
+    max_num_characters: Optional[int] = None,
+    exclude: Optional[Iterable] = None,
+    chk_size=None,
+    egress: Optional[Union[str, Callable]] = None,
+    **store_aggregate_kwargs,
+):
+    """
+    Aggregates and processes text files from a store.
+
+    Parameters:
+        store: The source store (e.g., TextFiles instance)
+        min_number_of_duplicated_lines (int): If not None, minimum block size for deduplication.
+        max_num_characters (int): Maximum number of characters per file.
+        exclude (set): Set of filenames to exclude.
+        chk_size (int): Chunk size for aggregation. If None, won't chunk
+        egress (str or callable): Template for output filenames or function to call on each chunk
+        **store_aggregate_kwargs: Additional keyword arguments to pass to store_aggregate.
+    """
+    if exclude is None:
+        exclude = set()
+
+    wrappers = []
+
+    if exclude is not None:
+        wrappers.append(filt_iter(filt=lambda x: x not in exclude))
+
+    if min_number_of_duplicated_lines is not None:
+        from hg import deduplicate_string_lines
+
+        remove_duplicate_lines = wrap_kvs(
+            value_decoder=lambda v: deduplicate_string_lines(
+                v,
+                min_block_size=min_number_of_duplicated_lines,
+                return_removed_blocks=False,
+            )
+        )
+    else:
+        remove_duplicate_lines = identity
+
+    if max_num_characters is not None:
+        wrappers.append(wrap_kvs(value_decoder=lambda v: v[:max_num_characters]))
+
+    capped_num_characters = wrap_kvs(value_decoder=lambda v: v[:max_num_characters])
+
+    if len(wrappers) > 0:
+        store_wrap = Pipe(
+            filt_iter(filt=lambda x: x.endswith(".md") and x not in exclude),
+            remove_duplicate_lines,
+            capped_num_characters,
+        )
+    else:
+        store_wrap = identity
+
+    wrapped_store = store_wrap(store)
+
+    if isinstance(egress, str):
+        output_template = egress
+        egress = output_template.format
+    else:
+        assert egress is None or callable(
+            egress
+        ), "egress must be None, str, or callable"
+
+    if isinstance(chk_size, int):
+        from lkj.chunking import chunk_iterable
+
+        chunks = chunk_iterable(wrapped_store, chk_size)
+        if egress is not None:
+            egress = "store_aggregate_{:02.0f}.md"
+        for i, sub_store in enumerate(chunks, 1):
+            store_aggregate(sub_store, egress=egress(i), **store_aggregate_kwargs)
+    else:
+        if egress is None:
+            egress = identity
+        return store_aggregate(wrapped_store, egress=egress, **store_aggregate_kwargs)