coauthor 0.0.5__py3-none-any.whl

coauthor/__init__.py

@@ -0,0 +1,5 @@
+"""C2 Platform Coauthor Package"""
+
+from __future__ import annotations
+
+__version__ = "0.0.5"

coauthor/main.py

@@ -0,0 +1,81 @@
+"""
+This module serves as the entry point for processing command-line arguments
+related to configuration and workflow initialization. It configures logging,
+retrieves configuration details, and initializes workflows based on the parsed
+command-line arguments.
+"""
+
+import argparse
+import logging
+import os
+from coauthor.utils.logger import Logger
+from coauthor.utils.config import get_config
+from coauthor.modules.workflow import initialize_workflows
+
+
+def main():
+    """
+    Parses command-line arguments, sets up logging, retrieves configuration,
+    and initializes workflows.
+
+    The function uses argparse to handle command-line arguments, sets up a logger based
+    on the provided or default log level and log file path, and retrieves configuration
+    details. It then initializes workflows using the collected configuration and logger.
+
+    Command-line arguments:
+
+    --config_path: Path to the configuration file.
+
+    --profile: Name of profile (e.g., obsidian, python, hugo).
+
+    --watch: Flag to enable watch mode.
+    --scan: Flag to enable scan mode.
+    --debug: Flag to set log level to DEBUG.
+    --log_file: Path to the log file.
+    """
+    parser = argparse.ArgumentParser(description="Process arguments to select steps")
+    parser.add_argument(
+        "--config_path",
+        type=str,
+        help="Path to the configuration file",
+    )
+    parser.add_argument(
+        "--profile",
+        type=str,
+        help="Name of profile e.g. obsidian, python, hugo",
+    )
+    parser.add_argument(
+        "--watch",
+        action="store_true",
+        help="Flag to enable watch mode",
+    )
+    parser.add_argument(
+        "--scan",
+        action="store_true",
+        help="Flag to enable scan mode",
+    )
+    parser.add_argument(
+        "--debug",
+        action="store_true",
+        help="Flag to set log level to DEBUG",
+    )
+    parser.add_argument(
+        "--log_file",
+        type=str,
+        help="Path to the log file",
+    )
+    args = parser.parse_args()
+
+    log_level = logging.DEBUG if args.debug else logging.INFO
+    if args.log_file:
+        log_file = args.log_file
+    else:
+        log_file = f"{os.getcwd()}/coauthor.log"
+    logger = Logger(__name__, log_file=log_file, level=log_level)
+    config = get_config(logger=logger, args=args)
+    config["args"] = args
+    initialize_workflows(config, logger)
+
+
+if __name__ == "__main__":
+    main()

coauthor/modules/__init__.py

@@ -0,0 +1,3 @@
+"""C2 Platform Coauthor Modules Package"""
+
+from __future__ import annotations

coauthor/modules/ai.py

@@ -0,0 +1,177 @@
+import os
+import traceback
+import yaml
+import datetime
+from openai import OpenAI
+from coauthor.utils.match_utils import file_submit_to_ai, path_new_replace
+from coauthor.utils.jinja import render_template, template_exists
+
+
+def load_system_message(agent_system_path, logger):
+    """Loads the system message from a string or a file."""
+    logger.info(f"System message from file {agent_system_path}")
+    if os.path.exists(agent_system_path):
+        try:
+            with open(agent_system_path, "r", encoding="utf-8") as file:
+                system_message = file.read()
+            logger.info(f"Loaded system message from file: {agent_system_path}")
+            return system_message
+        except Exception as error:
+            logger.error(f"Error reading system message file: {error}")
+            raise
+    else:
+        raise FileNotFoundError(f"The system path {agent_system_path} does not exist")
+
+
+def write_response_to_yaml(config, messages, model, response, logger, duration=None):
+    """Writes response data to respective markdown and YAML files."""
+    task = config["current-task"]
+    workflow = config["current-workflow"]
+    coauthor_ai_log_dir = os.getenv("COAUTHOR_AI_LOG_DIR")
+    if not coauthor_ai_log_dir:
+        logger.debug("Environment variable COAUTHOR_AI_LOG_DIR not set")
+        return
+
+    counter_file_path = os.path.join(coauthor_ai_log_dir, ".ai-prompt-counter")
+    if not os.path.exists(counter_file_path):
+        message_id = 1
+        with open(counter_file_path, "w", encoding="utf-8") as counter_file:
+            counter_file.write(str(message_id))
+    else:
+        with open(counter_file_path, "r+", encoding="utf-8") as counter_file:
+            message_id = int(counter_file.read())
+            message_id += 1
+            counter_file.seek(0)
+            counter_file.write(str(message_id))
+
+    # Create directory path for markdown files
+    markdown_dir = os.path.join(coauthor_ai_log_dir, f"{workflow['name']}/{task['id']}")
+    os.makedirs(markdown_dir, exist_ok=True)
+
+    # Write each message to an individual markdown file
+    for msg in messages:
+        md_file_path = os.path.join(markdown_dir, f"{message_id}-{msg['role']}.md")
+        with open(md_file_path, "w", encoding="utf-8") as md_file:
+            md_file.write(msg["content"])
+        logger.info(f"Message written to Markdown file: {md_file_path}")
+
+    yaml_file_path = os.path.join(markdown_dir, f"{message_id}.yml")
+    task["ai-log-file"] = yaml_file_path
+
+    data = {
+        "messages": [{"role": msg["role"], "tokens": len(msg["content"].split())} for msg in messages],  # Count words
+        "model": model,
+        "response": response,
+        "task": task,
+        "date": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+        "duration": duration,  # Add the duration to the data
+    }
+    try:
+        with open(yaml_file_path, "w", encoding="utf-8") as file:
+            yaml.dump(data, file, default_flow_style=False, allow_unicode=True)
+        logger.info(f"Response written to YAML file: {yaml_file_path}")
+    except Exception as error:
+        logger.error(f"Error writing response to YAML file: {error}")
+        raise
+
+
+def create_chat_completion(config, client, messages, logger):
+    """Create a chat completion using OpenAI client."""
+    model = config["agent"]["model"]
+    try:
+        start_time = datetime.datetime.now()  # Record start time
+        response = client.chat.completions.create(
+            messages=messages,
+            model=model,
+        )
+        end_time = datetime.datetime.now()  # Record end time
+        duration = (end_time - start_time).seconds  # Calculate duration
+
+        content = response.choices[0].message.content.strip()
+
+        write_response_to_yaml(config, messages, model, content, logger, duration)
+
+        if content.startswith("```") and content.endswith("```"):
+            content_lines = content.splitlines()[1:-1]
+            content = "\n".join(content_lines).strip()
+        return content
+
+    except Exception as error:
+        logger.error(f"Error creating chat completion: {error}")
+        logger.error(traceback.format_exc())
+        raise
+
+
+def prompt_template_path(config, filename):
+    task = config["current-task"]
+    workflow = config["current-workflow"]
+    return f"{workflow['name']}/{task['id']}/{filename}"
+
+
+def user_template_path(config, filename="user.md"):
+    return prompt_template_path(config, filename)
+
+
+def system_template_path(config, filename="system.md"):
+    return prompt_template_path(config, filename)
+
+
+def process_with_openai_agent(config, logger):
+    """Submit content to OpenAI API for processing."""
+    task = config["current-task"]
+    logger.info(f"Processing content using AI for task {task['id']}")
+    agent = config["agent"]
+    logger.debug(f"agent: {agent}")
+
+    if "api_key" in agent:
+        api_key = agent["api_key"]
+    else:
+        api_key = os.getenv(agent["api_key_var"])
+    if "api_url" in agent:
+        api_url = agent["api_url"]
+    else:
+        api_url = os.getenv(agent["api_url_var"])
+
+    client = OpenAI(
+        api_key=api_key,
+        base_url=api_url,
+    )
+
+    message_system_and_user = {}
+    for system_or_user in ["system", "user"]:
+        path_template = prompt_template_path(config, f"{system_or_user}.md")
+        if template_exists(task, path_template, config, logger):
+            message_system_and_user[system_or_user] = render_template(task, path_template, config, logger)
+        elif system_or_user in task:
+            message_system_and_user[system_or_user] = task[system_or_user]
+        else:
+            if "path-modify-event" in task and system_or_user == "user" and "user" not in task:
+                logger.info(f"Using the file {task['path-modify-event']} as the user message")
+                message_system_and_user[system_or_user] = file_submit_to_ai(config, logger)
+            else:
+                raise ValueError(
+                    f'An AI task should have a key "{system_or_user}" or a template with path "{path_template}"'
+                )
+
+    # Log only the first line of the messages
+    user_message = message_system_and_user["user"].splitlines()[0] if message_system_and_user["user"] else ""
+    system_message = message_system_and_user["system"].splitlines()[0] if message_system_and_user["system"] else ""
+    logger.debug(f"user_message: {user_message}")
+    logger.debug(f"system_message: {system_message}")
+
+    messages = [
+        {
+            "role": "system",
+            "content": message_system_and_user["system"],
+        },
+        {"role": "user", "content": message_system_and_user["user"]},
+    ]
+
+    return create_chat_completion(config, client, messages, logger)
+
+
+def process_file_with_openai_agent(config, logger):
+    """Submit file to OpenAI API for processing."""
+    task = config["current-task"]
+    task["response"] = process_with_openai_agent(config, logger)
+    return task["response"]

coauthor/modules/file_processor.py

@@ -0,0 +1,64 @@
+import re
+import time
+from coauthor.utils.match_utils import file_path_match
+
+
+def pong(config, logger):
+    path = config["current-task"]["path-modify-event"]
+    logger.info("Running pong file processor " + path)
+    time.sleep(3)
+    with open(path, "r") as file:
+        file_contents = file.read()
+    if file_contents != "pong":
+        logger.info(f'Updating {path} to "pong"')
+        with open(path, "w") as f:
+            f.write("pong")
+
+
+def regex_replace_in_file(config, logger):
+    task = config["current-task"]
+    path = task["path-modify-event"]
+    path_match = file_path_match(config, logger)
+    if not path_match:
+        logger.debug(f"regex_replace_in_file: no path match for {path}")
+        return False
+    patterns = task["patterns"]
+
+    # Open the file and read the content
+    with open(path, "r", encoding="utf-8") as file:
+        content = file.read()
+
+    original_content = content
+
+    # Apply each regex pattern and handle internal matches/replacement if needed
+    for pattern_set in patterns:
+        logger.debug(f"regex_replace_in_file: {path}, patterns: {pattern_set}")
+        pattern = pattern_set["regex"]
+
+        # Check for internal modifications based on run-time conditions
+        internal_regex = pattern_set.get("internal_regex")
+        internal_replace = pattern_set.get("internal_replace")
+        if internal_regex and internal_replace:
+            content = re.sub(
+                pattern,
+                lambda match: re.sub(
+                    internal_regex,
+                    internal_replace,
+                    match.group(0),
+                ),
+                content,
+            )
+        else:
+            replace = pattern_set["replace"]
+            content = re.sub(pattern, replace, content)
+
+    if content == original_content:
+        logger.debug("regex_replace_in_file: no content was changed")
+        return False
+
+    logger.info(f"Regex patterns changed file {path}, patterns: {patterns}")
+    task["content"] = content
+    with open(path, "w", encoding="utf-8") as file:
+        file.write(content)
+        time.sleep(3)
+    return True

coauthor/modules/file_scanner.py

@@ -0,0 +1,45 @@
+import os
+import time
+from coauthor.modules.ai import process_file_with_openai_agent
+
+from coauthor.utils.workflow_utils import get_workflows_that_scan
+from coauthor.modules.file_processor import regex_replace_in_file, pong
+from coauthor.modules.workflow_tasks import write_file, read_file
+
+task_type_functions = {
+    "process_file_with_openai_agent": process_file_with_openai_agent,
+    "regex_replace_in_file": regex_replace_in_file,
+    "pong": pong,
+    "write_file": write_file,
+    "read_file": read_file,
+}
+
+
+def scan(config, logger):
+    workflows_that_scan = get_workflows_that_scan(config, logger)
+    logger.debug(f"workflows_that_scan: {workflows_that_scan}")
+    for workflow in workflows_that_scan:
+        scan_directories = workflow["scan"]["filesystem"]["paths"]
+        logger.info(f"Workflow {workflow['name']}: scan directories {', '.join(scan_directories)}")
+
+        wd_to_path = {}
+        for directory in scan_directories:
+            for root, _, files in os.walk(directory):
+                for filename in files:
+                    path = os.path.join(root, filename)
+                    handle_workflow_scan_file(path, workflow, config, logger)
+
+
+def handle_workflow_scan_file(path, workflow, config, logger):
+    logger.info(f"Processing file {path}")
+    for task in workflow["tasks"]:
+        logger.debug(f"task: {task}")
+        if task["type"] in task_type_functions:
+            logger.debug(f"Workflow: {workflow['name']}, Task: {task['id']} → {path}")
+            if "path-modify-event" not in task:
+                task["path-modify-event"] = path
+            config["current-task"] = task
+            config["current-workflow"] = workflow
+            task_type_functions[task["type"]](config, logger)
+        else:
+            raise ValueError(f'Unsupported task_type: {task["type"]}')

coauthor/modules/file_watcher.py

@@ -0,0 +1,173 @@
+import os
+import time
+from inotify_simple import INotify, flags
+
+from coauthor.utils.workflow_utils import (
+    get_workflows_that_watch,
+    get_workflows_that_scan,
+    get_all_scan_directories_from_workflows,
+    get_all_watch_directories_from_workflows,
+)
+from coauthor.utils.match_utils import file_path_match, file_content_match
+from coauthor.modules.ai import process_file_with_openai_agent
+from coauthor.modules.file_processor import regex_replace_in_file, pong
+from coauthor.modules.workflow_tasks import write_file, read_file
+
+task_type_functions = {
+    "process_file_with_openai_agent": process_file_with_openai_agent,
+    "regex_replace_in_file": regex_replace_in_file,
+    "pong": pong,
+    "write_file": write_file,
+    "read_file": read_file,
+}
+
+# Initialize a dictionary to store the last modification time of each file based
+# on workflow and task
+last_modification_times = {}
+
+
+def add_watch_recursive(inotify, directory):
+    """Recursively add watches on all subdirectories,
+    ignoring certain directories."""
+    wd_to_path = {}
+    for root, dirs, files in os.walk(directory):
+        # Ignore specific directories like __pycache__  # TODO .obsidian directory
+        dirs[:] = [d for d in dirs if d != "__pycache__"]
+        wd = inotify.add_watch(root, flags.CREATE | flags.MODIFY)
+        wd_to_path[wd] = root
+    return wd_to_path
+
+
+def watch(config, logger):
+    watch_directory(config, logger)
+
+
+def handle_inotify_event(event, wd_to_path, inotify, config, logger):
+    """Handle inotify events by accurately determining the file changed.
+
+    For MODIFY events, due to some editors' behavior of replacing files rather than directly modifying them,
+    this function identifies the most recently updated file in the event's directory as the file affected by the event.
+    """
+    directory = wd_to_path.get(event.wd, "")
+    logger.debug(
+        f"Inotify Event: directory={directory}, event_mask={event.mask}, flags.from_mask={flags.from_mask(event.mask)}"
+    )
+
+    # Find the most recently modified file in the directory, ignoring hidden files
+    def get_recently_modified_file(file_path_inotify, dir_path, logger):
+        all_files = [
+            os.path.join(dir_path, f)
+            for f in os.listdir(dir_path)
+            if os.path.isfile(os.path.join(dir_path, f)) and not f.startswith(".")
+        ]
+        logger.debug(f"all_files: {', '.join(all_files)}")
+        if file_path_inotify in all_files:
+            return file_path_inotify
+        return max(all_files, key=os.path.getmtime) if all_files else None
+
+    file_path_inotify = os.path.join(directory, event.name)
+    file_path = get_recently_modified_file(file_path_inotify, directory, logger)
+
+    # Check if it is a directory and add watch if new directory is created
+    if flags.CREATE in flags.from_mask(event.mask) and os.path.isdir(file_path_inotify):
+        logger.info(f"Watching new directory: {file_path_inotify}")
+        wd = inotify.add_watch(file_path_inotify, flags.CREATE | flags.MODIFY | flags.CLOSE_WRITE)
+        wd_to_path[wd] = file_path_inotify
+
+    if file_path_inotify != file_path:
+        logger.warning(f"file_path_inotify: {file_path_inotify} is not equal to file_path: {file_path}")
+        logger.debug(" this can occur depending on how editors write changes to files")
+        logger.debug(" For example Gedit uses a temporary file .goutputstream-G1SHX2")
+        file_path_selected = file_path
+        time.sleep(2)  # allow Gedit some time to finish updating file
+    else:
+        file_path_selected = file_path_inotify
+
+    ignore_extensions = config.get("watch-ignore-file-extensions", [])
+    if flags.MODIFY in flags.from_mask(event.mask) and file_path_selected:
+        file_extension = os.path.splitext(file_path_selected)[1]
+        if file_extension not in ignore_extensions:
+            logger.info(f"MODIFY event: file_path_inotify: {file_path_selected}")
+            handle_workflow_file_modify_event(file_path_selected, config, logger)
+
+    # Stop the process if `stop` file is created
+    if os.path.basename(file_path_selected) == "stop":
+        logger.info("Stop file found!")
+        os.remove(file_path_selected)
+        return True
+
+    return False
+
+
+def watch_directory(config, logger):
+    watch_directories = get_all_watch_directories_from_workflows(config, logger)
+    inotify = INotify()
+
+    logger.info(f"Watching directories recursively: {', '.join(watch_directories)}")
+
+    wd_to_path = {}
+    for directory in watch_directories:
+        wd_to_path.update(add_watch_recursive(inotify, directory))
+
+    try:
+        while True:
+            for event in inotify.read():
+                if handle_inotify_event(event, wd_to_path, inotify, config, logger):
+                    return
+    except KeyboardInterrupt:
+        print("Stopping directory watch")
+    finally:
+        for wd in wd_to_path.keys():
+            inotify.rm_watch(wd)
+
+
+def handle_workflow_file_modify_event(path, config, logger):
+    workflows_that_watch = get_workflows_that_watch(config, logger)
+    logger.debug(f"get_workflows_that_watch: workflows_that_watch: {workflows_that_watch}")
+    for workflow in workflows_that_watch:
+        config["current-workflow"] = workflow
+        logger.debug(f"workflow: {workflow}")
+        for task in workflow["tasks"]:
+            config["current-task"] = task
+            task["path-modify-event"] = path
+            path_match = file_path_match(config, logger)
+            if "content_patterns" in workflow:
+                content_match = file_content_match(config, logger)
+                logger.debug(f'Workflow has no "content_patterns", so "content_match" is True')
+            else:
+                content_match = True
+            if not path_match or not content_match:
+                logger.debug(f"No path or content match on {path}")
+                continue
+            logger.info(f"Path and content match on {path}")
+            if task["type"] in task_type_functions:
+                logger.info(f"Workflow: {workflow['name']}, Task: {task['id']} → {path}")
+                if not ignore_rapid_modify_event(path, config, logger):
+                    task_type_functions[task["type"]](config, logger)
+            else:
+                raise ValueError(f'Unsupported task_type: {task["type"]}')
+
+
+# Some tools / IDE depending on configuraton  / extensions / plugins might cause
+# multiple modify events. To distinquishd from technical save from user save
+# we can set a time limit and ignore the same modify event
+def ignore_rapid_modify_event(path, config, logger):
+    current_time = time.time()
+    workflow = config["current-workflow"]
+    task = config["current-task"]
+    # Determine unique key based on file path, workflow and task id
+    key = (path, workflow["name"], task["id"])
+    last_time = last_modification_times.get(key, 0)
+
+    modify_event_limit = 3
+    if "modify_event_limit" in task:
+        modify_event_limit = task["modify_event_limit"]
+
+    # If the time since the last modification event is less than
+    # modify_event_limit, ignore this event
+    if current_time - last_time < modify_event_limit:
+        logger.info(f"  Ignoring rapid MODIFY event for (modify_event_limit: {modify_event_limit}).")
+        return True
+    last_modification_times[key] = current_time
+    logger.debug(f"  NOT ignoring rapid MODIFY event for (modify_event_limit: {modify_event_limit}).")
+    return False

coauthor/modules/workflow.py

@@ -0,0 +1,32 @@
+# from coauthor.modules.file_watcher import watch
+# from coauthor.modules.file_scanner import scan
+# from coauthor.modules.ai import process_file_with_openai_agent
+# from coauthor.modules.file_processor import pong, regex_replace_in_file
+
+# from coauthor.modules.workflow_tasks import read_file, write_file
+from coauthor.modules.file_scanner import scan
+from coauthor.modules.file_watcher import watch
+from coauthor.utils.workflow_utils import (
+    get_all_scan_directories_from_workflows,
+    get_all_watch_directories_from_workflows,
+)
+
+
+def initialize_workflows(config, logger, trigger_scan=False):
+    if not "workflows" in config:
+        logger.warning("No workflows in config, nothing to do")
+        return
+    args = config.get("args", None)
+    if (args and args.scan) or trigger_scan:
+        logger.info("Scan mode enabled with --scan")
+        scan_directories = get_all_scan_directories_from_workflows(config, logger)
+        logger.info(f"scan_directories: {', '.join(scan_directories)}")
+        if len(scan_directories) > 0:
+            scan(config, logger)
+        else:
+            logger.debug(f"No scan directories!")
+    if args and args.watch:
+        watch_directories = get_all_watch_directories_from_workflows(config, logger)
+        logger.info("Watch mode enabled with --watch")
+        if len(watch_directories) > 0:
+            watch(config, logger)

coauthor/modules/workflow_tasks.py

@@ -0,0 +1,32 @@
+from coauthor.utils.jinja import render_content
+import os
+
+
+def read_file(config, logger):
+    logger.debug(f"config: {config}")
+    task = config["current-task"]
+    path = task["path-modify-event"]
+    logger.info(f"Reading file from {path}")
+    with open(path, "r", encoding="utf-8") as file:
+        content = file.read()
+    task["content"] = content
+
+
+def write_file(config, logger):
+    logger.debug(f"config: {config}")
+    task = config["current-task"]
+    if "path" in task:
+        path = render_content(task, task["path"], config, logger)
+    else:
+        path = task["path-modify-event"]
+
+    directory = os.path.dirname(path)
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+        logger.warning(f"Directory {directory} did not exist, created it.")
+
+    logger.info(f"Writing to file at {path}")
+
+    content = render_content(task, task["content"], config, logger)
+    with open(path, "w", encoding="utf-8") as file:
+        file.write(content)