PyPI - mindroot - Versions diffs - 9.11.0__py3-none-any.whl → 9.13.0__py3-none-any.whl - Mend - Supply Chain Defender

mindroot 9.11.0py3-none-any.whl → 9.13.0py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of mindroot might be problematic. Click here for more details.

Files changed (417) hide show

mindroot/docs/_build/html/_modules/mindroot/lib/token_counter.html ADDED Viewed

@@ -0,0 +1,326 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" data-content_root="../../../">
+<head>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>mindroot.lib.token_counter &mdash; MindRoot 9.10.0 documentation</title>
+      <link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=b86133f3" />
+      <link rel="stylesheet" type="text/css" href="../../../_static/css/theme.css?v=e59714d7" />
+      <script src="../../../_static/jquery.js?v=5d32c60e"></script>
+      <script src="../../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
+      <script src="../../../_static/documentation_options.js?v=3c16008f"></script>
+      <script src="../../../_static/doctools.js?v=9bcbadda"></script>
+      <script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
+    <script src="../../../_static/js/theme.js"></script>
+    <link rel="index" title="Index" href="../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../search.html" />
+</head>
+<body class="wy-body-for-nav">
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search"  style="background: #343131" >
+          <a href="../../../index.html" class="icon icon-home">
+            MindRoot
+          </a>
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <!-- Local TOC -->
+              <div class="local-toc"></div>
+        </div>
+      </div>
+    </nav>
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu"  style="background: #343131" >
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../index.html">MindRoot</a>
+      </nav>
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+  <ul class="wy-breadcrumbs">
+      <li><a href="../../../index.html" class="icon icon-home" aria-label="Home"></a></li>
+          <li class="breadcrumb-item"><a href="../../index.html">Module code</a></li>
+      <li class="breadcrumb-item active">mindroot.lib.token_counter</li>
+      <li class="wy-breadcrumbs-aside">
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+  <h1>Source code for mindroot.lib.token_counter</h1><div class="highlight"><pre>
+<span></span><span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">time</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">asyncio</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">aiofiles</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">aiofiles.os</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">Dict</span><span class="p">,</span> <span class="n">List</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">mindroot.lib.chatlog</span><span class="w"> </span><span class="kn">import</span> <span class="n">ChatLog</span>
+<div class="viewcode-block" id="find_chatlog_file">
+<a class="viewcode-back" href="../../../source/mindroot.lib.html#mindroot.lib.token_counter.find_chatlog_file">[docs]</a>
+<span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">find_chatlog_file</span><span class="p">(</span><span class="n">log_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Find a chatlog file by its log_id.</span>
+<span class="sd">    </span>
+<span class="sd">    Args:</span>
+<span class="sd">        log_id: The log ID to search for</span>
+<span class="sd">        </span>
+<span class="sd">    Returns:</span>
+<span class="sd">        The full path to the chatlog file if found, None otherwise</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">chat_dir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;CHATLOG_DIR&#39;</span><span class="p">,</span> <span class="s1">&#39;data/chat&#39;</span><span class="p">)</span>
+    <span class="c1"># Use os.walk to search through all subdirectories</span>
+    <span class="k">for</span> <span class="n">root</span><span class="p">,</span> <span class="n">dirs</span><span class="p">,</span> <span class="n">files</span> <span class="ow">in</span> <span class="k">await</span> <span class="n">asyncio</span><span class="o">.</span><span class="n">to_thread</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">walk</span><span class="p">,</span> <span class="n">chat_dir</span><span class="p">):</span>
+        <span class="k">for</span> <span class="n">file</span> <span class="ow">in</span> <span class="n">files</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">file</span> <span class="o">==</span> <span class="sa">f</span><span class="s2">&quot;chatlog_</span><span class="si">{</span><span class="n">log_id</span><span class="si">}</span><span class="s2">.json&quot;</span><span class="p">:</span>
+                <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">root</span><span class="p">,</span> <span class="n">file</span><span class="p">)</span>
+    <span class="k">return</span> <span class="kc">None</span></div>
+<div class="viewcode-block" id="extract_delegate_task_log_ids">
+<a class="viewcode-back" href="../../../source/mindroot.lib.html#mindroot.lib.token_counter.extract_delegate_task_log_ids">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">extract_delegate_task_log_ids</span><span class="p">(</span><span class="n">messages</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Dict</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Extract log IDs from delegate_task commands in messages.</span>
+<span class="sd">    </span>
+<span class="sd">    Args:</span>
+<span class="sd">        messages: List of chat messages</span>
+<span class="sd">        </span>
+<span class="sd">    Returns:</span>
+<span class="sd">        List of log IDs found in delegate_task commands</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">log_ids</span> <span class="o">=</span> <span class="p">[]</span>
+    <span class="k">for</span> <span class="n">message</span> <span class="ow">in</span> <span class="n">messages</span><span class="p">:</span>
+        <span class="k">if</span> <span class="n">message</span><span class="p">[</span><span class="s1">&#39;role&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;assistant&#39;</span><span class="p">:</span>
+            <span class="n">content</span> <span class="o">=</span> <span class="n">message</span><span class="p">[</span><span class="s1">&#39;content&#39;</span><span class="p">]</span>
+            <span class="c1"># Handle both string and list content formats</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
+                <span class="n">text</span> <span class="o">=</span> <span class="n">content</span>
+            <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="nb">list</span><span class="p">)</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span> <span class="ow">and</span> <span class="s1">&#39;text&#39;</span> <span class="ow">in</span> <span class="n">content</span><span class="p">[</span><span class="mi">0</span><span class="p">]:</span>
+                <span class="n">text</span> <span class="o">=</span> <span class="n">content</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">&#39;text&#39;</span><span class="p">]</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">continue</span>
+            <span class="c1"># Try to parse as JSON</span>
+            <span class="k">try</span><span class="p">:</span>
+                <span class="n">commands</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
+                <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">commands</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+                    <span class="n">commands</span> <span class="o">=</span> <span class="p">[</span><span class="n">commands</span><span class="p">]</span>
+                <span class="k">for</span> <span class="n">cmd</span> <span class="ow">in</span> <span class="n">commands</span><span class="p">:</span>
+                    <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">cmd</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+                        <span class="k">if</span> <span class="n">key</span> <span class="o">==</span> <span class="s1">&#39;delegate_task&#39;</span> <span class="ow">and</span> <span class="s1">&#39;log_id&#39;</span> <span class="ow">in</span> <span class="n">value</span><span class="p">:</span>
+                            <span class="n">log_ids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">value</span><span class="p">[</span><span class="s1">&#39;log_id&#39;</span><span class="p">])</span>
+            <span class="k">except</span> <span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">JSONDecodeError</span><span class="p">,</span> <span class="ne">TypeError</span><span class="p">,</span> <span class="ne">KeyError</span><span class="p">):</span>
+                <span class="c1"># If not JSON, try regex to find log_ids in delegate_task commands</span>
+                <span class="n">matches</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="sa">r</span><span class="s1">&#39;&quot;delegate_task&quot;\s*:\s*{\s*&quot;log_id&quot;\s*:\s*&quot;([^&quot;]+)&quot;&#39;</span><span class="p">,</span> <span class="n">text</span><span class="p">)</span>
+                <span class="n">log_ids</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">matches</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">log_ids</span></div>
+<div class="viewcode-block" id="get_cache_dir">
+<a class="viewcode-back" href="../../../source/mindroot.lib.html#mindroot.lib.token_counter.get_cache_dir">[docs]</a>
+<span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">get_cache_dir</span><span class="p">()</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Get the directory for token count cache files.</span>
+<span class="sd">    Creates the directory if it doesn&#39;t exist.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">cache_dir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;TOKEN_CACHE_DIR&#39;</span><span class="p">,</span> <span class="s1">&#39;data/token_cache&#39;</span><span class="p">)</span>
+    <span class="k">if</span> <span class="ow">not</span> <span class="k">await</span> <span class="n">aiofiles</span><span class="o">.</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">cache_dir</span><span class="p">):</span>
+        <span class="k">await</span> <span class="n">aiofiles</span><span class="o">.</span><span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">cache_dir</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">cache_dir</span></div>
+<div class="viewcode-block" id="get_cache_path">
+<a class="viewcode-back" href="../../../source/mindroot.lib.html#mindroot.lib.token_counter.get_cache_path">[docs]</a>
+<span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">get_cache_path</span><span class="p">(</span><span class="n">log_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Get the path to the cache file for a specific log_id.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">cache_dir</span> <span class="o">=</span> <span class="k">await</span> <span class="n">get_cache_dir</span><span class="p">()</span>
+    <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">cache_dir</span><span class="p">,</span> <span class="sa">f</span><span class="s2">&quot;tokens_</span><span class="si">{</span><span class="n">log_id</span><span class="si">}</span><span class="s2">.json&quot;</span><span class="p">)</span></div>
+<div class="viewcode-block" id="get_cached_token_counts">
+<a class="viewcode-back" href="../../../source/mindroot.lib.html#mindroot.lib.token_counter.get_cached_token_counts">[docs]</a>
+<span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">get_cached_token_counts</span><span class="p">(</span><span class="n">log_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">log_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Get cached token counts if available and valid.</span>
+<span class="sd">    </span>
+<span class="sd">    Args:</span>
+<span class="sd">        log_id: The log ID</span>
+<span class="sd">        log_path: Path to the actual log file</span>
+<span class="sd">        </span>
+<span class="sd">    Returns:</span>
+<span class="sd">        Cached token counts if valid, None otherwise</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">cache_path</span> <span class="o">=</span> <span class="k">await</span> <span class="n">get_cache_path</span><span class="p">(</span><span class="n">log_id</span><span class="p">)</span>
+    <span class="c1"># If cache doesn&#39;t exist, return None</span>
+    <span class="k">if</span> <span class="ow">not</span> <span class="k">await</span> <span class="n">aiofiles</span><span class="o">.</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">cache_path</span><span class="p">):</span>
+        <span class="k">return</span> <span class="kc">None</span>
+    <span class="k">try</span><span class="p">:</span>
+        <span class="c1"># Get modification times</span>
+        <span class="n">log_mtime</span> <span class="o">=</span> <span class="k">await</span> <span class="n">aiofiles</span><span class="o">.</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">getmtime</span><span class="p">(</span><span class="n">log_path</span><span class="p">)</span>
+        <span class="n">cache_mtime</span> <span class="o">=</span> <span class="k">await</span> <span class="n">aiofiles</span><span class="o">.</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">getmtime</span><span class="p">(</span><span class="n">cache_path</span><span class="p">)</span>
+        <span class="n">current_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
+        <span class="c1"># If log was modified after cache was created, cache is invalid</span>
+        <span class="k">if</span> <span class="n">log_mtime</span> <span class="o">&gt;</span> <span class="n">cache_mtime</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">None</span>
+        <span class="c1"># Don&#39;t recalculate sooner than 3 minutes after last calculation</span>
+        <span class="k">if</span> <span class="n">current_time</span> <span class="o">-</span> <span class="n">cache_mtime</span> <span class="o">&lt;</span> <span class="mi">180</span><span class="p">:</span>  <span class="c1"># 3 minutes in seconds</span>
+            <span class="k">async</span> <span class="k">with</span> <span class="n">aiofiles</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">cache_path</span><span class="p">,</span> <span class="s1">&#39;r&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
+                <span class="n">content</span> <span class="o">=</span> <span class="k">await</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
+                <span class="k">return</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">content</span><span class="p">)</span>
+        <span class="c1"># For logs that haven&#39;t been modified in over an hour, consider them &quot;finished&quot;</span>
+        <span class="c1"># and use the cache regardless of when it was last calculated</span>
+        <span class="k">if</span> <span class="n">current_time</span> <span class="o">-</span> <span class="n">log_mtime</span> <span class="o">&gt;</span> <span class="mi">3600</span><span class="p">:</span>  <span class="c1"># 1 hour in seconds</span>
+            <span class="k">async</span> <span class="k">with</span> <span class="n">aiofiles</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">cache_path</span><span class="p">,</span> <span class="s1">&#39;r&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
+                <span class="n">content</span> <span class="o">=</span> <span class="k">await</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
+                <span class="k">return</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">content</span><span class="p">)</span>
+    <span class="k">except</span> <span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">JSONDecodeError</span><span class="p">,</span> <span class="ne">IOError</span><span class="p">)</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+        <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Error reading token cache: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+    <span class="k">return</span> <span class="kc">None</span></div>
+<div class="viewcode-block" id="save_token_counts_to_cache">
+<a class="viewcode-back" href="../../../source/mindroot.lib.html#mindroot.lib.token_counter.save_token_counts_to_cache">[docs]</a>
+<span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">save_token_counts_to_cache</span><span class="p">(</span><span class="n">log_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">token_counts</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Save token counts to cache.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">cache_path</span> <span class="o">=</span> <span class="k">await</span> <span class="n">get_cache_path</span><span class="p">(</span><span class="n">log_id</span><span class="p">)</span>
+    <span class="k">async</span> <span class="k">with</span> <span class="n">aiofiles</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">cache_path</span><span class="p">,</span> <span class="s1">&#39;w&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
+        <span class="k">await</span> <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">token_counts</span><span class="p">))</span></div>
+<div class="viewcode-block" id="count_tokens_for_log_id">
+<a class="viewcode-back" href="../../../source/mindroot.lib.html#mindroot.lib.token_counter.count_tokens_for_log_id">[docs]</a>
+<span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">count_tokens_for_log_id</span><span class="p">(</span><span class="n">log_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Count tokens for a chat log identified by log_id, including any delegated tasks.</span>
+<span class="sd">    </span>
+<span class="sd">    Args:</span>
+<span class="sd">        log_id: The log ID to count tokens for</span>
+<span class="sd">        </span>
+<span class="sd">    Returns:</span>
+<span class="sd">        Dictionary with token counts or None if log not found</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="c1"># Find the chatlog file</span>
+    <span class="n">chatlog_path</span> <span class="o">=</span> <span class="k">await</span> <span class="n">find_chatlog_file</span><span class="p">(</span><span class="n">log_id</span><span class="p">)</span>
+    <span class="k">if</span> <span class="ow">not</span> <span class="n">chatlog_path</span><span class="p">:</span>
+        <span class="k">return</span> <span class="kc">None</span>
+    <span class="c1"># Check cache first</span>
+    <span class="n">cached_counts</span> <span class="o">=</span> <span class="k">await</span> <span class="n">get_cached_token_counts</span><span class="p">(</span><span class="n">log_id</span><span class="p">,</span> <span class="n">chatlog_path</span><span class="p">)</span>
+    <span class="k">if</span> <span class="n">cached_counts</span><span class="p">:</span>
+        <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Using cached token counts for </span><span class="si">{</span><span class="n">log_id</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">cached_counts</span>
+    <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Calculating token counts for </span><span class="si">{</span><span class="n">log_id</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+    <span class="c1"># Load the chat log</span>
+    <span class="k">async</span> <span class="k">with</span> <span class="n">aiofiles</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">chatlog_path</span><span class="p">,</span> <span class="s1">&#39;r&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
+        <span class="n">content</span> <span class="o">=</span> <span class="k">await</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
+        <span class="n">log_data</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">content</span><span class="p">)</span>
+    <span class="c1"># Create a temporary ChatLog instance to count tokens</span>
+    <span class="n">temp_log</span> <span class="o">=</span> <span class="n">ChatLog</span><span class="p">(</span><span class="n">log_id</span><span class="o">=</span><span class="n">log_id</span><span class="p">,</span> <span class="n">user</span><span class="o">=</span><span class="s2">&quot;system&quot;</span><span class="p">,</span> <span class="n">agent</span><span class="o">=</span><span class="n">log_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;agent&#39;</span><span class="p">,</span> <span class="s1">&#39;unknown&#39;</span><span class="p">))</span>
+    <span class="n">temp_log</span><span class="o">.</span><span class="n">messages</span> <span class="o">=</span> <span class="n">log_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;messages&#39;</span><span class="p">,</span> <span class="p">[])</span>
+    <span class="c1"># Count tokens for this log</span>
+    <span class="n">parent_counts</span> <span class="o">=</span> <span class="n">temp_log</span><span class="o">.</span><span class="n">count_tokens</span><span class="p">()</span>
+    <span class="c1"># Create combined counts (starting with parent counts)</span>
+    <span class="n">combined_counts</span> <span class="o">=</span> <span class="p">{}</span>
+    <span class="n">combined_counts</span><span class="p">[</span><span class="s1">&#39;input_tokens_sequence&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">parent_counts</span><span class="p">[</span><span class="s1">&#39;input_tokens_sequence&#39;</span><span class="p">]</span>
+    <span class="n">combined_counts</span><span class="p">[</span><span class="s1">&#39;output_tokens_sequence&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">parent_counts</span><span class="p">[</span><span class="s1">&#39;output_tokens_sequence&#39;</span><span class="p">]</span>
+    <span class="n">combined_counts</span><span class="p">[</span><span class="s1">&#39;input_tokens_total&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">parent_counts</span><span class="p">[</span><span class="s1">&#39;input_tokens_total&#39;</span><span class="p">]</span>
+    <span class="c1"># Find delegated task log IDs</span>
+    <span class="n">delegated_log_ids</span> <span class="o">=</span> <span class="n">extract_delegate_task_log_ids</span><span class="p">(</span><span class="n">temp_log</span><span class="o">.</span><span class="n">messages</span><span class="p">)</span>
+    <span class="c1"># Recursively count tokens for delegated tasks</span>
+    <span class="k">for</span> <span class="n">delegated_id</span> <span class="ow">in</span> <span class="n">delegated_log_ids</span><span class="p">:</span>
+        <span class="n">delegated_counts</span> <span class="o">=</span> <span class="k">await</span> <span class="n">count_tokens_for_log_id</span><span class="p">(</span><span class="n">delegated_id</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">delegated_counts</span><span class="p">:</span>
+            <span class="n">combined_counts</span><span class="p">[</span><span class="s1">&#39;input_tokens_sequence&#39;</span><span class="p">]</span> <span class="o">+=</span> <span class="n">delegated_counts</span><span class="p">[</span><span class="s1">&#39;input_tokens_sequence&#39;</span><span class="p">]</span>
+            <span class="n">combined_counts</span><span class="p">[</span><span class="s1">&#39;output_tokens_sequence&#39;</span><span class="p">]</span> <span class="o">+=</span> <span class="n">delegated_counts</span><span class="p">[</span><span class="s1">&#39;output_tokens_sequence&#39;</span><span class="p">]</span>
+            <span class="n">combined_counts</span><span class="p">[</span><span class="s1">&#39;input_tokens_total&#39;</span><span class="p">]</span> <span class="o">+=</span> <span class="n">delegated_counts</span><span class="p">[</span><span class="s1">&#39;input_tokens_total&#39;</span><span class="p">]</span>
+    <span class="c1"># Create final result with both parent and combined counts</span>
+    <span class="n">token_counts</span> <span class="o">=</span> <span class="p">{}</span>
+    <span class="c1"># Parent session only counts</span>
+    <span class="n">token_counts</span><span class="p">[</span><span class="s1">&#39;input_tokens_sequence&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">parent_counts</span><span class="p">[</span><span class="s1">&#39;input_tokens_sequence&#39;</span><span class="p">]</span>
+    <span class="n">token_counts</span><span class="p">[</span><span class="s1">&#39;output_tokens_sequence&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">parent_counts</span><span class="p">[</span><span class="s1">&#39;output_tokens_sequence&#39;</span><span class="p">]</span>
+    <span class="n">token_counts</span><span class="p">[</span><span class="s1">&#39;input_tokens_total&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">parent_counts</span><span class="p">[</span><span class="s1">&#39;input_tokens_total&#39;</span><span class="p">]</span>
+    <span class="c1"># Combined counts (parent + all subtasks)</span>
+    <span class="n">token_counts</span><span class="p">[</span><span class="s1">&#39;combined_input_tokens_sequence&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">combined_counts</span><span class="p">[</span><span class="s1">&#39;input_tokens_sequence&#39;</span><span class="p">]</span>
+    <span class="n">token_counts</span><span class="p">[</span><span class="s1">&#39;combined_output_tokens_sequence&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">combined_counts</span><span class="p">[</span><span class="s1">&#39;output_tokens_sequence&#39;</span><span class="p">]</span>
+    <span class="n">token_counts</span><span class="p">[</span><span class="s1">&#39;combined_input_tokens_total&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">combined_counts</span><span class="p">[</span><span class="s1">&#39;input_tokens_total&#39;</span><span class="p">]</span>
+    <span class="c1"># Save to cache</span>
+    <span class="k">await</span> <span class="n">save_token_counts_to_cache</span><span class="p">(</span><span class="n">log_id</span><span class="p">,</span> <span class="n">token_counts</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">token_counts</span></div>
+</pre></div>
+           </div>
+          </div>
+          <footer>
+  <hr/>
+  <div role="contentinfo">
+    <p>&#169; Copyright 2025, Jason Livesay.</p>
+  </div>
+  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
+    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
+    provided by <a href="https://readthedocs.org">Read the Docs</a>.
+</footer>
+        </div>
+      </div>
+    </section>
+  </div>
+  <script>
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script>
+</body>
+</html>

mindroot/docs/_build/html/_modules/mindroot/migrate.html ADDED Viewed

@@ -0,0 +1,153 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" data-content_root="../../">
+<head>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>mindroot.migrate &mdash; MindRoot 9.10.0 documentation</title>
+      <link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
+      <link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
+      <script src="../../_static/jquery.js?v=5d32c60e"></script>
+      <script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
+      <script src="../../_static/documentation_options.js?v=3c16008f"></script>
+      <script src="../../_static/doctools.js?v=9bcbadda"></script>
+      <script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
+    <script src="../../_static/js/theme.js"></script>
+    <link rel="index" title="Index" href="../../genindex.html" />
+    <link rel="search" title="Search" href="../../search.html" />
+</head>
+<body class="wy-body-for-nav">
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search"  style="background: #343131" >
+          <a href="../../index.html" class="icon icon-home">
+            MindRoot
+          </a>
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <!-- Local TOC -->
+              <div class="local-toc"></div>
+        </div>
+      </div>
+    </nav>
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu"  style="background: #343131" >
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../index.html">MindRoot</a>
+      </nav>
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+  <ul class="wy-breadcrumbs">
+      <li><a href="../../index.html" class="icon icon-home" aria-label="Home"></a></li>
+          <li class="breadcrumb-item"><a href="../index.html">Module code</a></li>
+      <li class="breadcrumb-item active">mindroot.migrate</li>
+      <li class="wy-breadcrumbs-aside">
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+  <h1>Source code for mindroot.migrate</h1><div class="highlight"><pre>
+<span></span><span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">logging</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">.lib.plugins.manifest</span><span class="w"> </span><span class="kn">import</span> <span class="n">create_default_plugin_manifest</span><span class="p">,</span> <span class="n">_get_absolute_paths</span><span class="p">,</span> <span class="n">_validate_manifest</span>
+<span class="c1"># Setup logging</span>
+<span class="n">logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="vm">__name__</span><span class="p">)</span>
+<div class="viewcode-block" id="migrate_plugin_manifest">
+<a class="viewcode-back" href="../../source/mindroot.html#mindroot.migrate.migrate_plugin_manifest">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">migrate_plugin_manifest</span><span class="p">():</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Migrate plugin_manifest.json from root to data/ directory if needed.</span>
+<span class="sd">    </span>
+<span class="sd">    This function now delegates to the consolidated manifest handling logic.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">manifest_abs_path</span><span class="p">,</span> <span class="n">root_manifest_abs_path</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">_get_absolute_paths</span><span class="p">()</span>
+    <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Checking plugin manifest migration...&quot;</span><span class="p">)</span>
+    <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Target manifest path: </span><span class="si">{</span><span class="n">manifest_abs_path</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+    <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Source manifest path: </span><span class="si">{</span><span class="n">root_manifest_abs_path</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+    <span class="c1"># Check if target manifest already exists and is valid</span>
+    <span class="n">is_valid</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">_validate_manifest</span><span class="p">(</span><span class="n">manifest_abs_path</span><span class="p">)</span>
+    <span class="k">if</span> <span class="n">is_valid</span><span class="p">:</span>
+        <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Valid plugin manifest already exists at </span><span class="si">{</span><span class="n">manifest_abs_path</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">return</span>
+    <span class="c1"># Check if source manifest exists</span>
+    <span class="n">source_exists</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">root_manifest_abs_path</span><span class="p">)</span>
+    <span class="k">if</span> <span class="n">source_exists</span><span class="p">:</span>
+        <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Found manifest to migrate from </span><span class="si">{</span><span class="n">root_manifest_abs_path</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;No existing plugin manifest found to migrate&quot;</span><span class="p">)</span>
+    <span class="c1"># Use the consolidated manifest creation logic which handles migration</span>
+    <span class="k">try</span><span class="p">:</span>
+        <span class="n">create_default_plugin_manifest</span><span class="p">()</span>
+        <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Plugin manifest migration/creation completed successfully&quot;</span><span class="p">)</span>
+    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+        <span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Plugin manifest migration failed: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">raise</span></div>
+<div class="viewcode-block" id="run_migrations">
+<a class="viewcode-back" href="../../source/mindroot.html#mindroot.migrate.run_migrations">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">run_migrations</span><span class="p">():</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Run all necessary migrations.&quot;&quot;&quot;</span>
+    <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Running MindRoot migrations...&quot;</span><span class="p">)</span>
+    <span class="k">try</span><span class="p">:</span>
+        <span class="n">migrate_plugin_manifest</span><span class="p">()</span>
+        <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Migrations completed successfully&quot;</span><span class="p">)</span>
+    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+        <span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Migration failed: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="c1"># Don&#39;t raise here - let the system continue with default manifest</span>
+        <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;Continuing with default manifest due to migration failure&quot;</span><span class="p">)</span></div>
+</pre></div>
+           </div>
+          </div>
+          <footer>
+  <hr/>
+  <div role="contentinfo">
+    <p>&#169; Copyright 2025, Jason Livesay.</p>
+  </div>
+  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
+    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
+    provided by <a href="https://readthedocs.org">Read the Docs</a>.
+</footer>
+        </div>
+      </div>
+    </section>
+  </div>
+  <script>
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script>
+</body>
+</html>