cnhkmcp 2.1.9__py3-none-any.whl → 2.3.0__py3-none-any.whl

cnhkmcp/untracked/APP//321/210/342/224/220/320/240/321/210/320/261/320/234/321/206/320/231/320/243/321/205/342/225/235/320/220/321/206/320/230/320/241.py

@@ -115,12 +115,15 @@ if not check_and_install_dependencies():
 
 # Now import the packages
 try:
-    from flask import Flask, render_template, request, jsonify, session as flask_session, Response, stream_with_context, send_from_directory
+    from flask import Flask, render_template, request, jsonify, session as flask_session, Response, stream_with_context, send_from_directory, send_file, after_this_request
+    from werkzeug.utils import secure_filename
     from flask_cors import CORS
     import requests
     import json
     import time
     import os
+    import zipfile
+    import tempfile
     import threading
     import queue
     import uuid
@@ -660,6 +663,7 @@ def authenticate():
         brain_sessions[session_id] = {
             'session': brain_session,
             'username': username,
+            'password': password,
             'timestamp': time.time(),
             'options': valid_options
         }
@@ -748,6 +752,7 @@ def complete_biometric():
             brain_sessions[new_session_id] = {
                 'session': brain_session,
                 'username': session_info['username'],
+                'password': session_info.get('password'),
                 'timestamp': time.time()
             }
             
@@ -1754,6 +1759,12 @@ def get_usage_doc():
 # Global task manager for Transformer Web
 transformer_tasks = {}
 
+# Global task manager for Inspiration direct pipeline
+inspiration_pipeline_tasks = {}
+
+# Global task manager for template enhancement
+inspiration_enhance_tasks = {}
+
 @app.route('/transformer-web')
 def transformer_web():
     return render_template('transformer_web.html')
@@ -2385,6 +2396,16 @@ def inspiration_generate():
         
         datafields_df = get_datafields(s, region=region, delay=int(delay), universe=universe, dataset_id=dataset_id, data_type="ALL")
         
+        # count the datatype of the datafields_df, if most of them are VECTOR, then we keep the VECTOR category operators in the operators_df, otherwise we remove them
+        datatype_counts = datafields_df['type'].value_counts().to_dict()
+        vector_count = datatype_counts.get('VECTOR', 0)
+        total_fields = sum(datatype_counts.values())
+        if total_fields > 0 and vector_count > (total_fields / 2):
+            # keep VECTOR operators
+            pass
+            print("Keeping VECTOR operators because majority of datafields are VECTOR type")
+            operators_df = operators_df[operators_df['category'] != 'Vector']
+
         script_dir = os.path.dirname(os.path.abspath(__file__))
         prompt_path = os.path.join(script_dir, "give_me_idea", "what_is_Alpha_template.md")
         try:
@@ -2392,7 +2413,7 @@ def inspiration_generate():
                 system_prompt = f.read()
         except:
             system_prompt = "You are a helpful assistant for generating Alpha templates."
-            
+        
         client = openai.OpenAI(api_key=api_key, base_url=base_url)
         
         max_retries = 5
@@ -2405,9 +2426,35 @@ def inspiration_generate():
             ops_subset = operators_df.head(n_ops)
             fields_subset = datafields_df.head(n_fields)
             
-            operators_info = ops_subset[['name', 'category', 'description']].to_string()
-            datafields_info = fields_subset[['id', 'description', 'subcategory']].to_string()
-            
+            # Render subsets as Markdown tables (with robust fallbacks)
+            try:
+                operators_info = ops_subset[['name', 'category', 'description']].to_markdown(index=False)
+            except Exception:
+                try:
+                    from tabulate import tabulate
+                    operators_info = tabulate(
+                        ops_subset[['name', 'category', 'description']].fillna(''),
+                        headers='keys',
+                        tablefmt='github',
+                        showindex=False
+                    )
+                except Exception:
+                    operators_info = ops_subset[['name', 'category', 'description']].to_string(index=False)
+
+            try:
+                datafields_info = fields_subset[['id', 'description', 'subcategory']].to_markdown(index=False)
+            except Exception:
+                try:
+                    from tabulate import tabulate
+                    datafields_info = tabulate(
+                        fields_subset[['id', 'description', 'subcategory']].fillna(''),
+                        headers='keys',
+                        tablefmt='github',
+                        showindex=False
+                    )
+                except Exception:
+                    datafields_info = fields_subset[['id', 'description', 'subcategory']].to_string(index=False)
+
             user_prompt = f"""
 Here is the information about available operators (first {n_ops} rows):
 {operators_info}
@@ -2415,8 +2462,8 @@ Here is the information about available operators (first {n_ops} rows):
 Here is the information about the dataset '{dataset_id}' (first {n_fields} rows):
 {datafields_info}
 
-Please come up with several Alpha templates based on this information.
-Specify the AI answer in Chinese.
+Please come up with as much diverse Alpha templates as you can based on above information. And do remember to make some innovation of the templates.
+Answer in Chinese.
 """
             try:
                 completion = client.chat.completions.create(
@@ -2425,7 +2472,7 @@ Specify the AI answer in Chinese.
                         {"role": "system", "content": system_prompt},
                         {"role": "user", "content": user_prompt}
                     ],
-                    temperature=0.3,
+                    temperature=1,
                 )
                 return jsonify({'result': completion.choices[0].message.content})
                 
@@ -2445,6 +2492,339 @@ Specify the AI answer in Chinese.
     except Exception as e:
         return jsonify({'error': str(e)}), 500
 
+
+def _safe_dataset_id(dataset_id: str) -> str:
+    return "".join([c for c in str(dataset_id) if c.isalnum() or c in ("-", "_")])
+
+
+def _get_pipeline_paths(dataset_id: str, region: str, delay: int):
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    trail_dir = os.path.join(script_dir, 'trailSomeAlphas')
+    run_pipeline_path = os.path.join(trail_dir, 'run_pipeline.py')
+    data_dir = os.path.join(trail_dir, 'skills', 'brain-feature-implementation', 'data')
+    dataset_folder = f"{_safe_dataset_id(dataset_id)}_{region}_delay{delay}"
+    output_folder = os.path.join(data_dir, dataset_folder)
+    return run_pipeline_path, trail_dir, output_folder, dataset_folder
+
+
+@app.route('/api/inspiration/run-pipeline', methods=['POST'])
+def inspiration_run_pipeline():
+    try:
+        data = request.get_json() or {}
+        dataset_id = data.get('datasetId')
+        data_category = data.get('dataCategory')
+        region = data.get('region')
+        delay = data.get('delay')
+        universe = data.get('universe')
+        api_key = data.get('apiKey')
+        base_url = data.get('baseUrl')
+        model = data.get('model')
+
+        if not dataset_id or not data_category or not region or delay is None or not universe:
+            return jsonify({'success': False, 'error': 'Missing required parameters'}), 400
+
+        run_pipeline_path, trail_dir, output_folder, dataset_folder = _get_pipeline_paths(dataset_id, region, int(delay))
+        if not os.path.exists(run_pipeline_path):
+            return jsonify({'success': False, 'error': f'run_pipeline.py not found: {run_pipeline_path}'}), 404
+
+        task_id = str(uuid.uuid4())
+        log_queue = queue.Queue()
+        inspiration_pipeline_tasks[task_id] = {
+            'queue': log_queue,
+            'status': 'running',
+            'output_folder': output_folder,
+            'dataset_folder': dataset_folder
+        }
+
+        session_id = request.headers.get('Session-ID') or flask_session.get('brain_session_id')
+        session_info = brain_sessions.get(session_id) if session_id else None
+
+        def run_process():
+            try:
+                cmd = [
+                    sys.executable,
+                    run_pipeline_path,
+                    '--data-category', str(data_category),
+                    '--region', str(region),
+                    '--delay', str(delay),
+                    '--dataset-id', str(dataset_id),
+                    '--universe', str(universe)
+                ]
+
+                if api_key:
+                    cmd.extend(['--moonshot-api-key', str(api_key)])
+                if model:
+                    cmd.extend(['--moonshot-model', str(model)])
+
+                env = os.environ.copy()
+                if api_key:
+                    env['MOONSHOT_API_KEY'] = str(api_key)
+                if base_url:
+                    env['MOONSHOT_BASE_URL'] = str(base_url)
+                if model:
+                    env['MOONSHOT_MODEL'] = str(model)
+                if session_info and session_info.get('username') and session_info.get('password'):
+                    env['BRAIN_USERNAME'] = session_info['username']
+                    env['BRAIN_PASSWORD'] = session_info['password']
+
+                proc = subprocess.Popen(
+                    cmd,
+                    cwd=trail_dir,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.STDOUT,
+                    text=True,
+                    encoding='utf-8',
+                    errors='replace',
+                    bufsize=1,
+                    env=env
+                )
+
+                if proc.stdout:
+                    for line in proc.stdout:
+                        log_queue.put(line.rstrip('\n'))
+
+                exit_code = proc.wait()
+                success = exit_code == 0
+                inspiration_pipeline_tasks[task_id]['status'] = 'completed' if success else 'failed'
+                log_queue.put({
+                    '__event__': 'done',
+                    'success': success,
+                    'exit_code': exit_code,
+                    'dataset_folder': dataset_folder
+                })
+            except Exception as e:
+                inspiration_pipeline_tasks[task_id]['status'] = 'failed'
+                log_queue.put({
+                    '__event__': 'done',
+                    'success': False,
+                    'error': str(e),
+                    'dataset_folder': dataset_folder
+                })
+
+        thread = threading.Thread(target=run_process)
+        thread.daemon = True
+        thread.start()
+
+        return jsonify({'success': True, 'taskId': task_id})
+
+    except Exception as e:
+        return jsonify({'success': False, 'error': str(e)}), 500
+
+
+@app.route('/api/inspiration/stream-pipeline/<task_id>')
+def inspiration_stream_pipeline(task_id):
+    task = inspiration_pipeline_tasks.get(task_id)
+    if not task:
+        return jsonify({'success': False, 'error': 'Task not found'}), 404
+
+    def generate():
+        q = task['queue']
+        while True:
+            item = q.get()
+            if isinstance(item, dict) and item.get('__event__') == 'done':
+                yield f"event: done\ndata: {json.dumps(item, ensure_ascii=False)}\n\n"
+                break
+
+            payload = {'line': item}
+            yield f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
+
+    return Response(stream_with_context(generate()), mimetype='text/event-stream')
+
+
+@app.route('/api/inspiration/download-pipeline/<task_id>')
+def inspiration_download_pipeline(task_id):
+    task = inspiration_pipeline_tasks.get(task_id)
+    if not task:
+        return jsonify({'success': False, 'error': 'Task not found'}), 404
+
+    output_folder = task.get('output_folder')
+    if not output_folder or not os.path.isdir(output_folder):
+        return jsonify({'success': False, 'error': 'Output folder not found'}), 404
+
+    temp = tempfile.NamedTemporaryFile(delete=False, suffix='.zip')
+    temp.close()
+
+    with zipfile.ZipFile(temp.name, 'w', zipfile.ZIP_DEFLATED) as zf:
+        base_name = os.path.basename(output_folder.rstrip(os.sep))
+        for root, _, files in os.walk(output_folder):
+            for filename in files:
+                abs_path = os.path.join(root, filename)
+                rel_path = os.path.relpath(abs_path, output_folder)
+                arcname = os.path.join(base_name, rel_path)
+                zf.write(abs_path, arcname=arcname)
+
+    @after_this_request
+    def _cleanup_zip(response):
+        try:
+            os.remove(temp.name)
+        except Exception:
+            pass
+        return response
+
+    download_name = f"{os.path.basename(output_folder)}.zip"
+    return send_file(temp.name, as_attachment=True, download_name=download_name)
+
+
+@app.route('/api/inspiration/enhance-template', methods=['POST'])
+def inspiration_enhance_template():
+    try:
+        idea_files = request.files.getlist('ideaFiles')
+        api_key = request.form.get('apiKey')
+        base_url = request.form.get('baseUrl')
+        model = request.form.get('model')
+
+        if not idea_files or not api_key:
+            return jsonify({'success': False, 'error': 'Missing ideaFiles or apiKey'}), 400
+
+        script_dir = os.path.dirname(os.path.abspath(__file__))
+        trail_dir = os.path.join(script_dir, 'trailSomeAlphas')
+        enhance_script = os.path.join(trail_dir, 'enhance_template.py')
+        if not os.path.exists(enhance_script):
+            return jsonify({'success': False, 'error': f'enhance_template.py not found: {enhance_script}'}), 404
+
+        task_id = str(uuid.uuid4())
+        log_queue = queue.Queue()
+        task_root = tempfile.mkdtemp(prefix='enhance_batch_')
+
+        saved_files = []
+        for idx, idea_file in enumerate(idea_files, start=1):
+            name = secure_filename(idea_file.filename or f'idea_{idx}.json')
+            file_dir = os.path.join(task_root, f"{idx:02d}_{os.path.splitext(name)[0]}")
+            os.makedirs(file_dir, exist_ok=True)
+            idea_path = os.path.join(file_dir, name)
+            idea_file.save(idea_path)
+            saved_files.append((name, idea_path))
+
+        inspiration_enhance_tasks[task_id] = {
+            'queue': log_queue,
+            'status': 'running',
+            'task_root': task_root,
+            'saved_files': saved_files
+        }
+
+        def run_process():
+            try:
+                total = len(saved_files)
+                completed_ok = True
+
+                for idx, (name, idea_path) in enumerate(saved_files, start=1):
+                    env = os.environ.copy()
+                    env['IDEA_JSON'] = idea_path
+                    env['MOONSHOT_API_KEY'] = api_key
+                    if base_url:
+                        env['MOONSHOT_BASE_URL'] = base_url
+                    if model:
+                        env['MOONSHOT_MODEL'] = model
+                    env['PYTHONIOENCODING'] = 'utf-8'
+
+                    log_queue.put(f"=== 开始处理: {name} ({idx}/{total}) ===")
+                    proc = subprocess.Popen(
+                        [sys.executable, enhance_script],
+                        cwd=trail_dir,
+                        stdout=subprocess.PIPE,
+                        stderr=subprocess.STDOUT,
+                        text=True,
+                        encoding='utf-8',
+                        errors='replace',
+                        bufsize=1,
+                        env=env
+                    )
+
+                    if proc.stdout:
+                        for line in proc.stdout:
+                            log_queue.put({'line': line.rstrip('\n'), 'file': name})
+
+                    exit_code = proc.wait()
+                    success = exit_code == 0
+                    if not success:
+                        completed_ok = False
+                    log_queue.put({'__event__': 'file_done', 'type': 'file_done', 'file': name, 'success': success})
+
+                inspiration_enhance_tasks[task_id]['status'] = 'completed' if completed_ok else 'failed'
+                log_queue.put({
+                    '__event__': 'done',
+                    'success': completed_ok,
+                    'total': total
+                })
+            except Exception as e:
+                inspiration_enhance_tasks[task_id]['status'] = 'failed'
+                log_queue.put({
+                    '__event__': 'done',
+                    'success': False,
+                    'error': str(e)
+                })
+
+        thread = threading.Thread(target=run_process)
+        thread.daemon = True
+        thread.start()
+
+        return jsonify({'success': True, 'taskId': task_id})
+
+    except Exception as e:
+        return jsonify({'success': False, 'error': str(e)}), 500
+
+
+@app.route('/api/inspiration/stream-enhance/<task_id>')
+def inspiration_stream_enhance(task_id):
+    task = inspiration_enhance_tasks.get(task_id)
+    if not task:
+        return jsonify({'success': False, 'error': 'Task not found'}), 404
+
+    def generate():
+        q = task['queue']
+        while True:
+            item = q.get()
+            if isinstance(item, dict) and item.get('__event__') == 'done':
+                yield f"event: done\ndata: {json.dumps(item, ensure_ascii=False)}\n\n"
+                break
+
+            if isinstance(item, dict) and item.get('__event__') == 'file_done':
+                payload = {'type': 'file_done', 'file': item.get('file'), 'success': item.get('success')}
+                yield f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
+                continue
+
+            if isinstance(item, dict):
+                payload = item
+            else:
+                payload = {'line': item}
+            yield f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
+
+    return Response(stream_with_context(generate()), mimetype='text/event-stream')
+
+
+@app.route('/api/inspiration/download-enhance/<task_id>')
+def inspiration_download_enhance(task_id):
+    task = inspiration_enhance_tasks.get(task_id)
+    if not task:
+        return jsonify({'success': False, 'error': 'Task not found'}), 404
+
+    task_root = task.get('task_root')
+    if not task_root or not os.path.isdir(task_root):
+        return jsonify({'success': False, 'error': 'Task output not found'}), 404
+
+    temp = tempfile.NamedTemporaryFile(delete=False, suffix='.zip')
+    temp.close()
+
+    with zipfile.ZipFile(temp.name, 'w', zipfile.ZIP_DEFLATED) as zf:
+        base_name = os.path.basename(task_root.rstrip(os.sep))
+        for root, _, files in os.walk(task_root):
+            for filename in files:
+                abs_path = os.path.join(root, filename)
+                rel_path = os.path.relpath(abs_path, task_root)
+                arcname = os.path.join(base_name, rel_path)
+                zf.write(abs_path, arcname=arcname)
+
+    @after_this_request
+    def _cleanup_zip(response):
+        try:
+            os.remove(temp.name)
+        except Exception:
+            pass
+        return response
+
+    download_name = f"{os.path.basename(task_root)}.zip"
+    return send_file(temp.name, as_attachment=True, download_name=download_name)
+
 if __name__ == '__main__':
     print("Starting BRAIN Expression Template Decoder Web Application...")
     print("Starting in safe mode: binding only to localhost (127.0.0.1)")