kopipasta 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

kopipasta/main.py

@@ -1,9 +1,12 @@
 #!/usr/bin/env python3
 import os
 import argparse
+import re
 import pyperclip
 import fnmatch
 
+import requests
+
 def read_gitignore():
     default_ignore_patterns = [
         '.git', 'node_modules', 'venv', '.venv', 'dist', '.idea', '__pycache__',
@@ -49,6 +52,9 @@ def get_human_readable_size(size):
             return f"{size:.2f} {unit}"
         size /= 1024.0
 
+def is_large_file(file_path, threshold=102400):  # 100 KB threshold
+    return os.path.getsize(file_path) > threshold
+
 def get_project_structure(ignore_patterns):
     tree = []
     for root, dirs, files in os.walk('.'):
@@ -94,6 +100,17 @@ def get_language_for_file(file_path):
     }
     return language_map.get(extension, '')
 
+def get_file_snippet(file_path, max_lines=50, max_bytes=4096):
+    snippet = ""
+    byte_count = 0
+    with open(file_path, 'r') as file:
+        for i, line in enumerate(file):
+            if i >= max_lines or byte_count >= max_bytes:
+                break
+            snippet += line
+            byte_count += len(line.encode('utf-8'))
+    return snippet
+
 def select_files_in_directory(directory, ignore_patterns, current_char_count=0):
     files = [f for f in os.listdir(directory)
              if os.path.isfile(os.path.join(directory, f)) and not is_ignored(os.path.join(directory, f), ignore_patterns) and not is_binary(os.path.join(directory, f))]
@@ -115,10 +132,26 @@ def select_files_in_directory(directory, ignore_patterns, current_char_count=0):
         print_char_count(current_char_count)
         choice = input("(y)es add all / (n)o ignore all / (s)elect individually / (q)uit? ").lower()
         if choice == 'y':
+            selected_files = []
             for file in files:
-                current_char_count += os.path.getsize(os.path.join(directory, file))
+                file_path = os.path.join(directory, file)
+                if is_large_file(file_path):
+                    while True:
+                        snippet_choice = input(f"{file} is large. Use (f)ull content or (s)nippet? ").lower()
+                        if snippet_choice in ['f', 's']:
+                            break
+                        print("Invalid choice. Please enter 'f' or 's'.")
+                    if snippet_choice == 's':
+                        selected_files.append((file, True))
+                        current_char_count += len(get_file_snippet(file_path))
+                    else:
+                        selected_files.append((file, False))
+                        current_char_count += os.path.getsize(file_path)
+                else:
+                    selected_files.append((file, False))
+                    current_char_count += os.path.getsize(file_path)
             print(f"Added all files from {directory}")
-            return files, current_char_count
+            return selected_files, current_char_count
         elif choice == 'n':
             print(f"Ignored all files from {directory}")
             return [], current_char_count
@@ -135,8 +168,21 @@ def select_files_in_directory(directory, ignore_patterns, current_char_count=0):
                         print_char_count(current_char_count)
                     file_choice = input(f"{file} ({file_size_readable}, ~{file_char_estimate} chars, ~{file_token_estimate} tokens) (y/n/q)? ").lower()
                     if file_choice == 'y':
-                        selected_files.append(file)
-                        current_char_count += file_char_estimate
+                        if is_large_file(file_path):
+                            while True:
+                                snippet_choice = input(f"{file} is large. Use (f)ull content or (s)nippet? ").lower()
+                                if snippet_choice in ['f', 's']:
+                                    break
+                                print("Invalid choice. Please enter 'f' or 's'.")
+                            if snippet_choice == 's':
+                                selected_files.append((file, True))
+                                current_char_count += len(get_file_snippet(file_path))
+                            else:
+                                selected_files.append((file, False))
+                                current_char_count += file_char_estimate
+                        else:
+                            selected_files.append((file, False))
+                            current_char_count += file_char_estimate
                         break
                     elif file_choice == 'n':
                         break
@@ -165,24 +211,89 @@ def process_directory(directory, ignore_patterns, current_char_count=0):
             continue
 
         selected_files, current_char_count = select_files_in_directory(root, ignore_patterns, current_char_count)
-        full_paths = [os.path.join(root, f) for f in selected_files]
+        full_paths = [(os.path.join(root, f), use_snippet) for f, use_snippet in selected_files]
         files_to_include.extend(full_paths)
         processed_dirs.add(root)
 
     return files_to_include, processed_dirs, current_char_count
 
-def generate_prompt(files_to_include, ignore_patterns):
+def fetch_web_content(url):
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        full_content = response.text
+        snippet = full_content[:1000] if len(full_content) > 10000 else full_content
+        return full_content, snippet
+    except requests.RequestException as e:
+        print(f"Error fetching content from {url}: {e}")
+        return None, None
+
+def read_env_file():
+    env_vars = {}
+    if os.path.exists('.env'):
+        with open('.env', 'r') as env_file:
+            for line in env_file:
+                line = line.strip()
+                if line and not line.startswith('#'):
+                    key, value = line.split('=', 1)
+                    env_vars[key.strip()] = value.strip()
+    return env_vars
+
+def detect_env_variables(content, env_vars):
+    detected_vars = []
+    for key, value in env_vars.items():
+        if value in content:
+            detected_vars.append((key, value))
+    return detected_vars
+
+def handle_env_variables(content, env_vars):
+    detected_vars = detect_env_variables(content, env_vars)
+    if not detected_vars:
+        return content
+
+    print("Detected environment variables:")
+    for key, value in detected_vars:
+        print(f"- {key}={value}")
+    
+    for key, value in detected_vars:
+        while True:
+            choice = input(f"How would you like to handle {key}? (m)ask / (s)kip / (k)eep: ").lower()
+            if choice in ['m', 's', 'k']:
+                break
+            print("Invalid choice. Please enter 'm', 's', or 'k'.")
+        
+        if choice == 'm':
+            content = content.replace(value, '*' * len(value))
+        elif choice == 's':
+            content = content.replace(value, "[REDACTED]")
+        # If 'k', we don't modify the content
+
+    return content
+
+def generate_prompt(files_to_include, ignore_patterns, web_contents, env_vars):
     prompt = "# Project Overview\n\n"
     prompt += "## Project Structure\n\n"
     prompt += "```\n"
     prompt += get_project_structure(ignore_patterns)
     prompt += "\n```\n\n"
     prompt += "## File Contents\n\n"
-    for file in files_to_include:
+    for file, use_snippet in files_to_include:
         relative_path = get_relative_path(file)
         language = get_language_for_file(file)
-        file_content = f"### {relative_path}\n\n```{language}\n{read_file_contents(file)}\n```\n\n"
-        prompt += file_content
+        if use_snippet:
+            file_content = get_file_snippet(file)
+            prompt += f"### {relative_path} (snippet)\n\n```{language}\n{file_content}\n```\n\n"
+        else:
+            file_content = read_file_contents(file)
+            file_content = handle_env_variables(file_content, env_vars)
+            prompt += f"### {relative_path}\n\n```{language}\n{file_content}\n```\n\n"
+    
+    if web_contents:
+        prompt += "## Web Content\n\n"
+        for url, (full_content, snippet) in web_contents.items():
+            content = handle_env_variables(snippet if len(full_content) > 10000 else full_content, env_vars)
+            prompt += f"### {url}{' (snippet)' if len(full_content) > 10000 else ''}\n\n```\n{content}\n```\n\n"
+    
     prompt += "## Task Instructions\n\n"
     task_instructions = input("Enter the task instructions: ")
     prompt += f"{task_instructions}\n\n"
@@ -200,21 +311,34 @@ def print_char_count(count):
     print(f"\rCurrent prompt size: {count} characters (~ {token_estimate} tokens)", flush=True)
 
 def main():
-    parser = argparse.ArgumentParser(description="Generate a prompt with project structure and file contents.")
-    parser.add_argument('inputs', nargs='+', help='Files or directories to include in the prompt')
+    parser = argparse.ArgumentParser(description="Generate a prompt with project structure, file contents, and web content.")
+    parser.add_argument('inputs', nargs='+', help='Files, directories, or URLs to include in the prompt')
     args = parser.parse_args()
 
     ignore_patterns = read_gitignore()
+    env_vars = read_env_file()
 
     files_to_include = []
     processed_dirs = set()
+    web_contents = {}
     current_char_count = 0
 
     for input_path in args.inputs:
-        if os.path.isfile(input_path):
+        if input_path.startswith(('http://', 'https://')):
+            full_content, snippet = fetch_web_content(input_path)
+            if full_content:
+                web_contents[input_path] = (full_content, snippet)
+                current_char_count += len(snippet if len(full_content) > 10000 else full_content)
+                print(f"Added web content from: {input_path}")
+        elif os.path.isfile(input_path):
             if not is_ignored(input_path, ignore_patterns) and not is_binary(input_path):
-                files_to_include.append(input_path)
-                print(f"Added file: {input_path}")
+                use_snippet = is_large_file(input_path)
+                files_to_include.append((input_path, use_snippet))
+                if use_snippet:
+                    current_char_count += len(get_file_snippet(input_path))
+                else:
+                    current_char_count += os.path.getsize(input_path)
+                print(f"Added file: {input_path}{' (snippet)' if use_snippet else ''}")
             else:
                 print(f"Ignored file: {input_path}")
         elif os.path.isdir(input_path):
@@ -222,17 +346,17 @@ def main():
             files_to_include.extend(dir_files)
             processed_dirs.update(dir_processed)
         else:
-            print(f"Warning: {input_path} is not a valid file or directory. Skipping.")
+            print(f"Warning: {input_path} is not a valid file, directory, or URL. Skipping.")
 
-    if not files_to_include:
-        print("No files were selected. Exiting.")
+    if not files_to_include and not web_contents:
+        print("No files or web content were selected. Exiting.")
         return
 
-    print("\nFile selection complete.")
+    print("\nFile and web content selection complete.")
     print_char_count(current_char_count)
-    print(f"Summary: Added {len(files_to_include)} files from {len(processed_dirs)} directories.")
+    print(f"Summary: Added {len(files_to_include)} files from {len(processed_dirs)} directories and {len(web_contents)} web sources.")
 
-    prompt = generate_prompt(files_to_include, ignore_patterns)
+    prompt = generate_prompt(files_to_include, ignore_patterns, web_contents, env_vars)
     print("\n\nGenerated prompt:")
     print(prompt)
 

kopipasta-0.4.0.dist-info/METADATA

@@ -0,0 +1,198 @@
+Metadata-Version: 2.1
+Name: kopipasta
+Version: 0.4.0
+Summary: A CLI tool to generate prompts with project structure and file contents
+Home-page: https://github.com/mkorpela/kopipasta
+Author: Mikko Korpela
+Author-email: mikko.korpela@gmail.com
+License: MIT
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pyperclip==1.9.0
+Requires-Dist: requests==2.32.3
+
+# kopipasta
+
+A CLI tool to generate prompts with project structure, file contents, and web content, while handling environment variables securely and offering snippets for large files.
+
+<img src="kopipasta.jpg" alt="kopipasta" width="300">
+
+## Installation
+
+You can install kopipasta using pipx (or pip):
+
+```
+pipx install kopipasta
+```
+
+## Usage
+
+To use kopipasta, run the following command in your terminal:
+
+```
+kopipasta [files_or_directories_or_urls]
+```
+
+Replace `[files_or_directories_or_urls]` with the paths to the files or directories you want to include in the prompt, as well as any web URLs you want to fetch content from.
+
+Example:
+```
+kopipasta src/ config.json https://example.com/api-docs
+```
+
+This will generate a prompt including:
+- The project structure
+- Contents of the specified files and directories (with snippet options for large files)
+- Content fetched from the provided URLs (with snippet options for large content)
+- Handling of environment variables found in a `.env` file (if present)
+
+Files and directories typically excluded in version control (based on common .gitignore patterns) are ignored.
+
+The generated prompt will be displayed in the console and automatically copied to your clipboard.
+
+## Features
+
+- Generates a structured prompt with project overview, file contents, web content, and task instructions
+- Offers snippet options for large files (>100 KB) and web content (>10,000 characters)
+- Fetches and includes content from web URLs
+- Detects and securely handles environment variables from a `.env` file
+- Ignores files and directories based on common .gitignore patterns
+- Allows interactive selection of files to include
+- Automatically copies the generated prompt to the clipboard
+
+## Environment Variable Handling
+
+If a `.env` file is present in the current directory, kopipasta will:
+1. Read and store the environment variables
+2. Detect these variables in file contents and web content
+3. Prompt you to choose how to handle each detected variable:
+   - (m)ask: Replace the value with asterisks
+   - (s)kip: Replace the value with "[REDACTED]"
+   - (k)eep: Leave the value as-is
+
+This ensures sensitive information is handled securely in the generated prompt.
+
+## Snippet Functionality
+
+For large files (>100 KB) and web content (>10,000 characters), kopipasta offers a snippet option:
+
+- For files: The first 50 lines or 4 KB (4,096 bytes), whichever comes first
+- For web content: The first 1,000 characters
+
+This helps manage the overall prompt size while still providing useful information about the content structure.
+
+## Example output
+
+```bash
+❯ kopipasta . https://example.com/api-docs
+
+Directory: .
+Files:
+- __init__.py
+- main.py (120 KB, ~120000 chars, ~30000 tokens)
+- large_data.csv (5 MB, ~5000000 chars, ~1250000 tokens)
+- .env
+
+(y)es add all / (n)o ignore all / (s)elect individually / (q)uit? y
+main.py is large. Use (f)ull content or (s)nippet? s
+large_data.csv is large. Use (f)ull content or (s)nippet? s
+Added all files from .
+Added web content from: https://example.com/api-docs
+
+File and web content selection complete.
+Current prompt size: 10500 characters (~ 2625 tokens)
+Summary: Added 3 files from 1 directory and 1 web source.
+
+Detected environment variables:
+- API_KEY=12345abcde
+
+How would you like to handle API_KEY? (m)ask / (k)eep: m
+
+Enter the task instructions: Implement new API endpoint
+
+Generated prompt:
+# Project Overview
+
+## Project Structure
+
+```
+|-- ./
+    |-- __init__.py
+    |-- main.py
+    |-- large_data.csv
+    |-- .env
+```
+
+## File Contents
+
+### __init__.py
+
+```python
+# Initialize package
+```
+
+### main.py (snippet)
+
+```python
+import os
+import pandas as pd
+
+API_KEY = os.getenv('API_KEY')
+
+def process_data(file_path):
+    df = pd.read_csv(file_path)
+    # Rest of the function...
+
+# More code...
+```
+
+### large_data.csv (snippet)
+
+```
+id,name,value
+1,John,100
+2,Jane,200
+3,Bob,150
+4,Alice,300
+# ... (first 50 lines or 4 KB)
+```
+
+## Web Content
+
+### https://example.com/api-docs (snippet)
+
+```
+API Documentation
+Endpoint: /api/v1/data
+Method: GET
+Authorization: Bearer **********
+...
+```
+
+## Task Instructions
+
+Implement new API endpoint
+
+## Task Analysis and Planning
+
+Before starting, explain the task back to me in your own words. Ask for any clarifications if needed. Once you're clear, ask to proceed.
+
+Then, outline a plan for the task. Finally, use your plan to complete the task.
+
+Prompt has been copied to clipboard. Final size: 1500 characters (~ 375 tokens)
+```
+
+## License
+
+This project is licensed under the MIT License.

kopipasta-0.4.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+kopipasta/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+kopipasta/main.py,sha256=5-zAPtdF4PU5xbMFJk5x5lmVAAXSIQ0FoKko4znch3k,15829
+kopipasta-0.4.0.dist-info/LICENSE,sha256=xw4C9TAU7LFu4r_MwSbky90uzkzNtRwAo3c51IWR8lk,1091
+kopipasta-0.4.0.dist-info/METADATA,sha256=8xv0cFWAnXYKS1_gUUCycrL_KGhWG1OUML0la5YAhiE,5431
+kopipasta-0.4.0.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+kopipasta-0.4.0.dist-info/entry_points.txt,sha256=but54qDNz1-F8fVvGstq_QID5tHjczP7bO7rWLFkc6Y,50
+kopipasta-0.4.0.dist-info/top_level.txt,sha256=iXohixMuCdw8UjGDUp0ouICLYBDrx207sgZIJ9lxn0o,10
+kopipasta-0.4.0.dist-info/RECORD,,

{kopipasta-0.2.0.dist-info → kopipasta-0.4.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (74.1.2)
+Generator: setuptools (75.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

kopipasta-0.2.0.dist-info/METADATA

@@ -1,122 +0,0 @@
-Metadata-Version: 2.1
-Name: kopipasta
-Version: 0.2.0
-Summary: A CLI tool to generate prompts with project structure and file contents
-Home-page: https://github.com/mkorpela/kopipasta
-Author: Mikko Korpela
-Author-email: mikko.korpela@gmail.com
-License: MIT
-Classifier: Development Status :: 3 - Alpha
-Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Requires-Python: >=3.8
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: pyperclip ==1.9.0
-
-
-# kopipasta
-
-A CLI tool to generate prompts with project structure and file contents.
-
-<img src="kopipasta.jpg" alt="kopipasta" width="300">
-
-## Installation
-
-You can install kopipasta using pipx (or pip):
-
-```
-pipx install kopipasta
-```
-
-## Usage
-
-To use kopipasta, run the following command in your terminal:
-
-```
-kopipasta [files_or_directories]
-```
-
-Replace `[files_or_directories]` with the paths to the files or directories you want to include in the prompt.
-
-Example:
-```
-kopipasta src/ config.json
-```
-
-This will generate a prompt including the project structure and contents of the specified files and directories, ignoring files and directories typically excluded in version control (based on common .gitignore patterns).
-
-The generated prompt will be displayed in the console and automatically copied to your clipboard.
-
-## Features
-
-- Generates a structured prompt with project overview, file contents, and task instructions
-- Ignores files and directories based on common .gitignore patterns
-- Allows interactive selection of files to include
-- Automatically copies the generated prompt to the clipboard
-
-## Example output
-
-```bash
-    ❯ kopipasta .
-
-    Directory: .
-    Files:
-    - __init__.py
-    - main.py
-
-    (y)es add all / (n)o ignore all / (s)elect individually / (q)uit? s
-    __init__.py (y/n/q)? y
-    main.py (y/n/q)? n
-    Added 1 files from .
-
-    File selection complete.
-    Summary: Added 1 files from 1 directories.
-    Enter the task instructions: Do my work
-
-    Generated prompt:
-    # Project Overview
-
-    ## Summary of Included Files
-
-    - __init__.py
-
-    ## Project Structure
-
-    ```
-    |-- ./
-        |-- __init__.py
-        |-- main.py
-    ```
-
-    ## File Contents
-
-    ### __init__.py
-
-    ```python
-
-    ```
-
-    ## Task Instructions
-
-    Do my work
-
-    ## Task Analysis and Planning
-
-    Before starting, explain the task back to me in your own words. Ask for any clarifications if needed. Once you're clear, ask to proceed.
-
-    Then, outline a plan for the task. Finally, use your plan to complete the task.
-
-    Prompt has been copied to clipboard.
-```
-
-## License
-
-This project is licensed under the MIT License.

kopipasta-0.2.0.dist-info/RECORD

@@ -1,8 +0,0 @@
-kopipasta/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-kopipasta/main.py,sha256=-3K3Pnv6auCjsQGwxTip8GlBZD2RiPAj9h7h_zeMbSc,10268
-kopipasta-0.2.0.dist-info/LICENSE,sha256=xw4C9TAU7LFu4r_MwSbky90uzkzNtRwAo3c51IWR8lk,1091
-kopipasta-0.2.0.dist-info/METADATA,sha256=hGK30wSKHEPuLh1DRibm6k1iq9Zsx-Dg7uGTO8bBBRo,3033
-kopipasta-0.2.0.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
-kopipasta-0.2.0.dist-info/entry_points.txt,sha256=but54qDNz1-F8fVvGstq_QID5tHjczP7bO7rWLFkc6Y,50
-kopipasta-0.2.0.dist-info/top_level.txt,sha256=iXohixMuCdw8UjGDUp0ouICLYBDrx207sgZIJ9lxn0o,10
-kopipasta-0.2.0.dist-info/RECORD,,