ApiLogicServer 12.0.4__py3-none-any.whl → 12.1.26__py3-none-any.whl

api_logic_server_cli/genai.py

@@ -1,17 +1,62 @@
+import json
+import sys
+import time
+import traceback
 from typing import Dict, List
 from api_logic_server_cli.cli_args_project import Project
 import logging
 from pathlib import Path
 import importlib
 import requests
-import os
+import os,re
 import create_from_model.api_logic_server_utils as utils
 import shutil
+import openai
+from openai import OpenAI
+from typing import List, Dict
+from pydantic import BaseModel
+from dotmap import DotMap
+import importlib.util
 
 log = logging.getLogger(__name__)
 
-k_update_prior_response = ', by updating the prior response.'
-""" , by updating the prior response. """
+K_LogicBankOff = "LBX"
+''' Disable Logic (for demos) '''
+
+class Rule(BaseModel):
+    name: str
+    description: str
+    use_case: str
+    code: str # logicbank rule code
+    
+class Model(BaseModel):
+    classname: str
+    code: str # sqlalchemy model code
+    sqlite_create: str # sqlite create table statement
+    description: str
+    name: str
+
+class TestDataRow(BaseModel):
+    test_data_row_variable: str  # the Python test data row variable
+    code: str  # Python code to create a test data row instance
+
+class WGResult(BaseModel):  # must match system/genai/prompt_inserts/response_format.prompt
+    # response: str # result
+    models : List[Model] # list of sqlalchemy classes in the response
+    rules : List[Rule] # list rule declarations
+    test_data: str
+    test_data_rows: List[TestDataRow]  # list of test data rows
+    test_data_sqlite: str # test data as sqlite INSERT statements
+    name: str  # suggest a short name for the project
+
+
+def import_module_from_path(module_name, file_path):
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)
+    return module
+
 
 class GenAI(object):
     """ Create project from genai prompt(s).  
@@ -29,12 +74,11 @@ class GenAI(object):
         The key argument is `--using`
         * It can be a file, dir (conversation) or text argument.
         * It's "stem" denotes the project name to be created at cwd
-        * `self.project.genai_using`
+        * `self.project.genai_using` (not used by WebGenAI)
 
-        The (rarely used) `--repaired_response` 
+        The (rarely used) `--repaired_response` --> `self.project.genai_repaired_response`
         * is for retry from corrected response
         * `--using` is required to get the project name, to be created at cwd
-        * `self.project.genai_repaired_response`
 
         __init__() is the main driver (work directory is <manager>/system/genai/temp/)
         
@@ -54,28 +98,10 @@ class GenAI(object):
 
         see key_module_map() for key methods
 
-
-        ##### Explore interim copilot access:
-
-        VSCode/Copilot-chat can turn prompts into logic, so can we automate with API?
-
-        https://stackoverflow.com/questions/76741410/how-to-invoke-github-copilot-programmatically
-        https://docs.google.com/document/d/1o0TeNQtuT6moWU1bOq2K20IbSw4YhV1x_aFnKwo_XeU/edit#heading=h.3xmoi7pevsnp
-        https://code.visualstudio.com/api/extension-guides/chat
-        https://code.visualstudio.com/api/extension-guides/language-model
-        https://github.com/B00TK1D/copilot-api
-
-        ### Or use ChatGPT:
-
-        Not sure vscode/copilot is best approach, since we'd like to activate this during project creation
-        (eg on web/GenAI - not using vscode).
-
-        * Thomas suggests there are ways to "teach" ChatGPT about Logic Bank.  This is a *great* idea.
-
-        https://platform.openai.com/docs/guides/fine-tuning/create-a-fine-tuned-model
+        https://platform.openai.com/finetune/ftjob-2i1wkh4t4l855NKCovJeHExs?filter=all
         """        
 
-        self.project = project
+        self.project = project  # als project info (cli args etc)
         log.info(f'\nGenAI [{self.project.project_name}] creating microservice from: {self.project.genai_using}')
         if self.project.genai_repaired_response != '':
             log.info(f'..     retry from [repaired] response file: {self.project.genai_repaired_response}')
@@ -83,32 +109,57 @@ class GenAI(object):
         self.project.from_model = f'system/genai/temp/create_db_models.py' # we always write the model to this file
         self.ensure_system_dir_exists()  # ~ manager, so we can write to system/genai/temp
         self.delete_temp_files()
+        self.post_error = ""
+        """ eg, if response contains table defs, save_prompt_messages_to_system_genai_temp_project raises an exception to trigger retry """
         self.prompt = ""
         """ `--using` - can come from file or text argument """
-
+        self.logic_enabled = True
+        """ K_LogicBankOff is used for demos, where we don't want to create logic """
         self.messages = self.get_prompt_messages()  # compute self.messages, from file, dir or text argument
 
         if self.project.genai_repaired_response == '':  # normal path - get response from ChatGPT
-            log.debug(f'.. ChatGPT - saving response to: system/genai/temp/chatgpt_original.response')
-            self.headers = self.get_headers_with_openai_api_key()
-            url = "https://api.openai.com/v1/chat/completions"
-            api_version = f'{self.project.genai_version}'  # eg, "gpt-4o"
-            data = {"model": api_version, "messages": self.messages}
-            response = requests.post(url, headers=self.headers, json=data)
-            create_db_models = self.get_and_save_raw_response_data(response)
+            try:
+                api_version = f'{self.project.genai_version}'  # eg, "gpt-4o"
+                start_time = time.time()
+                db_key = os.getenv("APILOGICSERVER_CHATGPT_APIKEY")
+                client = OpenAI(api_key=os.getenv("APILOGICSERVER_CHATGPT_APIKEY"))
+                model = api_version
+                if model == "":  # default from CLI is '', meaning fall back to env variable or system default...
+                    model = os.getenv("APILOGICSERVER_CHATGPT_MODEL")
+                    if model is None or model == "*":  # system default chatgpt model
+                        model = "gpt-4o-2024-08-06"
+                self.resolved_model = model
+                completion = client.beta.chat.completions.parse(
+                    messages=self.messages, response_format=WGResult,
+                    # temperature=self.project.genai_temperature,  values .1 and .7 made students / charges fail
+                    model=model  # for own model, use "ft:gpt-4o-2024-08-06:personal:logicbank:ARY904vS" 
+                )
+                log.debug(f'ChatGPT ({str(int(time.time() - start_time))} secs) - response at: system/genai/temp/chatgpt_original.response')
+                
+                data = completion.choices[0].message.content
+                response_dict = json.loads(data)
+                self.get_and_save_raw_response_data(completion=completion, response_dict=response_dict)
+                # print(json.dumps(json.loads(data), indent=4))
+                pass
+            except Exception as inst:
+                log.error(f"\n\nError: ChatGPT call failed\n{inst}\n\n")
+                sys.exit('ChatGPT call failed - please see https://apilogicserver.github.io/Docs/WebGenAI-CLI/#configuration')
         else: # for retry from corrected response... eg system/genai/temp/chatgpt_retry.response
+            self.resolved_model = "(n/a: model not used for repaired response)"
             log.debug(f'\nUsing [corrected] response from: {self.project.genai_repaired_response}')
-            with open(self.project.genai_repaired_response, 'r') as file:
-                create_db_models = file.read()
-        self.create_db_models = create_db_models
+            with open(self.project.genai_repaired_response, 'r') as response_file:
+                response_dict = json.load(response_file)
+
+        self.response_dict = DotMap(response_dict)
         """ the raw response data from ChatGPT which will be fixed & saved create_db_models.py """
 
-        self.project.genai_logic = self.get_logic_from_prompt()
+        self.get_valid_project_name()
 
-        self.fix_and_write_model_file(create_db_models) # write create_db_models.py for db creation   
+        self.fix_and_write_model_file() # write create_db_models.py for db creation, & logic 
         self.save_prompt_messages_to_system_genai_temp_project()  # save prompts, response and models.py
         if project.project_name_last_node == 'genai_demo_conversation':
             debug_string = "good breakpoint - check create_db_models.py"
+        pass # if we've set self.post_error, we'll raise an exception to trigger retry
         pass # return to api_logic_server.ProjectRun to create db/project from create_db_models.py
 
     def delete_temp_files(self):
@@ -123,7 +174,10 @@ class GenAI(object):
         """ Create presets - you are a data modelling expert, and logicbank api etc """
         pass
 
-        starting_message = {"role": "system", "content": "You are a data modelling expert and python software architect who expands on user input ideas. You create data models with at least 4 tables"}
+        you_are = "You are a data modelling expert and python software architect who expands on user input ideas. You create data models with at least 4 tables"
+        if self.project.genai_tables > 0:
+            you_are = you_are.replace('4', str(self.project.genai_tables))
+        starting_message = {"role": "system", "content": you_are}
         prompt_messages.append( starting_message)
 
         learning_requests = self.get_learning_requests()
@@ -133,6 +187,27 @@ class GenAI(object):
         log.debug(f'.. conv[001] presets: {learning_requests[0]["content"][:30]}...')
         return len(learning_requests)
 
+    def chatgpt_excp(self):
+        # https://apilogicserver.github.io/Docs/WebGenAI-CLI/
+        pass
+
+    def get_valid_project_name(self):
+        """ Get a valid project name from the project name
+        Takes a string and returns a valid filename constructed from the string.
+        """
+
+        # Replace invalid characters with underscores
+        valid_name = re.sub(r'[ \\/*?:"<>|\t\n\r\x0b\x0c]', '_', self.response_dict.name)    
+        valid_name = valid_name.strip()     # Remove leading and trailing spaces
+        valid_name = valid_name[:255]       # Limit the filename length
+        if self.project.project_name == '_genai_default':
+            log.debug(f'.. project name: {valid_name} (from response: {self.response_dict.name})')
+            self.response_dict.name = valid_name
+            self.project.project_name = self.response_dict.name
+            self.project.project_name_last_node = self.response_dict.name 
+        else:
+            self.project.directory_setup()  # avoid names like "system/genai/temp/TBD"
+        return
     
     def get_prompt_messages(self) -> List[Dict[str, str]]:
         """ Get prompt from file, dir (conversation) or text argument
@@ -176,9 +251,9 @@ class GenAI(object):
                         response_count += 1
                     else:
                         request_count += 1      # rebuild response with *all* tables
-                        if request_count > 1:   # Run Config: genai AUTO DEALERSHIP CONVERSATION
+                        if request_count > 2:   # Run Config: genai AUTO DEALERSHIP CONVERSATION
                             if 'updating the prior response' not in prompt:
-                                prompt += k_update_prior_response                    
+                                prompt = self.get_prompt__with_inserts(raw_prompt=prompt, for_iteration=True)                  
                     prompt_messages.append( {"role": role, "content": prompt})
                 else:
                     log.debug(f'.. .. conv ignores: {os.path.basename(each_file)}')
@@ -218,7 +293,7 @@ class GenAI(object):
                         learning_requests.append( {"role": "user", "content": learning_request_lines})
         return learning_requests  # TODO - what if no learning requests?
     
-    def get_prompt__with_inserts(self, raw_prompt: str) -> str:
+    def get_prompt__with_inserts(self, raw_prompt: str, for_iteration: bool = False) -> str:
         """ prompt-engineering: insert db-specific logic into prompt 
             raw_prompt: the prompt from file or text argument
         """
@@ -235,13 +310,52 @@ class GenAI(object):
         elif 'mysql' in self.project.db_url:
             prompt_inserts = f'mysql_inserts.prompt'
 
-        if prompt_inserts != "*":
-            assert Path(f'system/genai/prompt_inserts/{prompt_inserts}').exists(), \
-                f"Missing prompt_inserts file: {prompt_inserts}"  # eg api_logic_server_cli/prototypes/manager/system/genai/prompt_inserts/sqlite_inserts.prompt
-            log.debug(f'get_prompt__with_inserts: {str(os.getcwd())} / {prompt_inserts}')
-            with open(f'system/genai/prompt_inserts/{prompt_inserts}', 'r') as file:
+        if prompt_inserts == "*":  
+            pass    # '*' means caller has computed their own prompt -- no inserts
+        else:       # do prompt engineering (inserts)
+            prompt_eng_file_name = f'system/genai/prompt_inserts/{prompt_inserts}'
+            assert Path(prompt_eng_file_name).exists(), \
+                f"Missing prompt_inserts file: {prompt_eng_file_name}"  # eg api_logic_server_cli/prototypes/manager/system/genai/prompt_inserts/sqlite_inserts.prompt
+            log.debug(f'get_prompt__with_inserts: {str(os.getcwd())} \n .. merged with: {prompt_eng_file_name}')
+            with open(prompt_eng_file_name, 'r') as file:
                 pre_post = file.read()  # eg, Use SQLAlchemy to create a sqlite database named system/genai/temp/create_db_models.sqlite, with
             prompt_result = pre_post.replace('{{prompt}}', raw_prompt)
+            if for_iteration:
+                # Update the prior response - be sure not to lose classes and test data already created.
+                prompt_result = 'Update the prior response - be sure not to lose classes and test data already created.' \
+                    + '\n\n' + prompt_result
+                log.debug(f'.. iteration inserted: Update the prior response')
+                log.debug(f'.... iteration prompt result: {prompt_result}')
+
+            prompt_lines = prompt_result.split('\n')
+            prompt_line_number = 0
+            do_logic = True
+            for each_line in prompt_lines:
+                if 'Create multiple rows of test data' in each_line:
+                    if self.project.genai_test_data_rows > 0:
+                        each_line = each_line.replace(
+                            f'Create multiple rows',  
+                            f'Create {self.project.genai_test_data_rows} rows')
+                        prompt_lines[prompt_line_number] = each_line
+                        log.debug(f'.. inserted explicit test data: {each_line}')
+                if K_LogicBankOff in each_line:
+                    self.logic_enabled = False  # for demos
+                if "LogicBank" in each_line and do_logic == True:
+                    log.debug(f'.. inserted: {each_line}')
+                    prompt_eng_logic_file_name = f'system/genai/prompt_inserts/logic_inserts.prompt'
+                    with open(prompt_eng_logic_file_name, 'r') as file:
+                        prompt_logic = file.read()  # eg, Use SQLAlchemy to...
+                    prompt_lines[prompt_line_number] = prompt_logic
+                    do_logic = False
+                prompt_line_number += 1
+            
+            response_format_file_name = f'system/genai/prompt_inserts/response_format.prompt'
+            with open(response_format_file_name, 'r') as file:
+                response_format = file.readlines()
+            prompt_lines.extend(response_format)
+
+            prompt_result = "\n".join(prompt_lines)  # back to a string
+            pass
         return prompt_result
     
     def ensure_system_dir_exists(self):
@@ -261,7 +375,7 @@ class GenAI(object):
             copied_path = shutil.copytree(src=from_dir, dst=to_dir, dirs_exist_ok=True)
 
     def get_logic_from_prompt(self) -> list[str]:
-        """ Get logic from ChatGPT prompt
+        """ Get logic from ChatGPT prompt (code after "Enforce")
 
         Args:
 
@@ -288,39 +402,6 @@ class GenAI(object):
                 logic_text += '    ' + each_line + '\n'
         return logic_text
 
-    @staticmethod
-    def remove_logic_halluncinations(each_line: str) -> str:
-        """remove hallucinations from logic
-
-        eg: Rule.setup()
-
-        Args:
-            each_line (str): _description_
-
-        Returns:
-            str: _description_
-        """        """ """
-        return_line = each_line
-        if each_line.startswith('    Rule.') or each_line.startswith('    DeclareRule.'):
-            if 'Rule.sum' in each_line:
-                pass
-            elif 'Rule.count' in each_line:
-                pass
-            elif 'Rule.formula' in each_line:
-                pass
-            elif 'Rule.copy' in each_line:
-                pass
-            elif 'Rule.constraint' in each_line:
-                pass
-            elif 'Rule.allocate' in each_line:
-                pass
-            elif 'Rule.calculate' in each_line:
-                return_line = each_line.replace('Rule.calculate', 'Rule.copy')
-            else:
-                return_line = each_line.replace('    ', '    # ')
-                log.debug(f'.. removed hallucination: {each_line}')
-        return return_line
-
     def insert_logic_into_created_project(self):  # TODO - redesign if conversation
         """Called *after project created* to insert prompt logic into 
         1. declare_logic.py (as comment)
@@ -329,20 +410,62 @@ class GenAI(object):
         Also creates the doc directory for record of prompt, response.
         """
 
+        def remove_logic_halluncinations(each_line: str) -> str:
+            """remove hallucinations from logic
+
+            eg: Rule.setup()
+
+            Args:
+                each_line (str): _description_
+
+            Returns:
+                str: _description_
+            """        """ """
+            return_line = each_line
+            if each_line.startswith('Rule.'):
+                # Sometimes indents left out (EmpDepts) - "code": "Rule.sum(derive=Department.salary_total, as_sum_of=Employee.salary)\nRule.constraint(validate=Department,\n                as_condition=lambda row: row.salary_total <= row.budget,\n                error_msg=\"Department salary total ({row.salary_total}) exceeds budget ({row.budget})\")"
+                each_line = "    " + each_line  # add missing indent
+                log.debug(f'.. fixed hallucination/indent: {each_line}')
+            if each_line.startswith('    Rule.') or each_line.startswith('    DeclareRule.'):
+                if 'Rule.sum' in each_line:
+                    pass
+                elif 'Rule.count' in each_line:
+                    pass
+                elif 'Rule.formula' in each_line:
+                    pass
+                elif 'Rule.copy' in each_line:
+                    pass
+                elif 'Rule.constraint' in each_line:
+                    pass
+                elif 'Rule.allocate' in each_line:
+                    pass
+                elif 'Rule.calculate' in each_line:
+                    return_line = each_line.replace('Rule.calculate', 'Rule.copy')
+                else:
+                    return_line = each_line.replace('    ', '    # ')
+                    log.debug(f'.. removed hallucination: {each_line}')
+            return return_line
+
+        logic_enabled = True
         logic_file = self.project.project_directory_path.joinpath('logic/declare_logic.py')
         in_logic = False
-        translated_logic = "\n    # Logic from GenAI:\n\n"
-        for each_line in self.create_db_models.split('\n'):
-            if in_logic:
-                if each_line.startswith('    '):    # indent => still in logic
-                    each_repaired_line = self.remove_logic_halluncinations(each_line=each_line)
-                    translated_logic += each_repaired_line + '\n'      
-                elif each_line.strip() == '':       # blank => still in logic
-                    pass
-                else:                               # no-indent => end of logic
-                    in_logic = False
-            if "declare_logic()" in each_line:
-                in_logic = True
+        translated_logic = "\n    # Logic from GenAI: (or, use your IDE w/ code completion)\n"
+        for each_rule in self.response_dict.rules:
+            comment_line = each_rule.description
+            translated_logic += f'\n    # {comment_line}\n'
+            code_lines = each_rule.code.split('\n')
+            if '\n' in each_rule.code:
+                debug_string = "good breakpoint - multi-line rule"
+            for each_line in code_lines:
+                if 'declare_logic.py' not in each_line:
+                    each_repaired_line = remove_logic_halluncinations(each_line=each_line)
+                    if not each_repaired_line.startswith('    '):  # sometimes in indents, sometimes not
+                        each_repaired_line = '    ' + each_repaired_line
+                    if 'def declare_logic' not in each_repaired_line:
+                        translated_logic += each_repaired_line + '\n'    
+        if self.logic_enabled == False:
+            translated_logic = "\n    # Logic from GenAI: (or, use your IDE w/ code completion)\n"
+            translated_logic += "\n    # LogicBank Disabled \n"  
         translated_logic += "\n    # End Logic from GenAI\n\n"
         utils.insert_lines_at(lines=translated_logic, 
                               file_name=logic_file, 
@@ -380,7 +503,7 @@ class GenAI(object):
             log.error(f"\n\nERROR creating genai project docs: {docs_dir}\n\n{traceback.format_exc()}")
         pass
 
-    def fix_and_write_model_file(self, response_data: str):
+    def fix_and_write_model_file(self):
         """
         1. break response data into lines
         2. throw away instructions
@@ -392,30 +515,24 @@ class GenAI(object):
             response_data (str): the chatgpt response
 
         """
-        model_class =  "# created from response - used to create database and project\n"
-        model_class += "#  should run without error\n"
-        model_class += "#  if not, check for decimal, indent, or import issues\n\n"
-        with open(f'system/genai/create_db_models_inserts/create_db_models_prefix.py', "r") as inserts:
-            model_lines = inserts.readlines()
-        for each_line in model_lines:
-            model_class += each_line + '\n'
-
-        response_array = response_data.split('\n')
-        line_num = 0
-        writing = False
-        indents_to_remove = 0
-        for each_line in response_array:
-            line_num += 1
-            if "```python" in each_line:
-                writing = True
-                # count spaces before "```"
-                # next_line = response_array[line_num+1]
-                position = each_line.find("```")
-                if position > 0:
-                    indents_to_remove = each_line[:position].count(' ')                
-            elif "```" in each_line:
-                writing = False
-            elif writing:  # ChatGPT work-arounds
+
+        def get_model_class_lines(model: DotMap) -> list[str]:
+            """Get the model class from the model
+
+            Args:
+                model (Model): the model
+
+            Returns:
+                stlist[str]: the model class lines, fixed up
+            """
+
+            create_db_model_lines =  list()
+            create_db_model_lines.append('\n\n')
+            class_lines = model.code.split('\n')
+            line_num = 0
+            indents_to_remove = 0
+            for each_line in class_lines:
+                line_num += 1
                 ''' decimal issues
 
                     1. bad import: see Run: tests/test_databases/ai-created/genai_demo/genai_demo_decimal
@@ -439,6 +556,9 @@ class GenAI(object):
                         got: or DECIMAL('
                         needed: or decimal.Decimal('0.00')
                 '''
+                if "= Table(" in each_line:  # tests/test_databases/ai-created/time_cards/time_card_kw_arg/genai.response
+                    log.debug(f'.. fix_and_write_model_file detects table - raise excp to trigger retry')
+                    self.post_error = "ChatGPT Response contains table (not class) definitions: " + each_line
                 if 'sqlite:///' in each_line:  # must be sqlite:///system/genai/temp/create_db_models.sqlite
                     current_url_rest = each_line.split('sqlite:///')[1]
                     quote_type = "'"
@@ -476,7 +596,7 @@ class GenAI(object):
                         each_line = '# ' + each_line
                 if 'sqlite:///system/genai/temp/model.sqlite':  # fix prior version
                     each_line = each_line.replace('sqlite:///system/genai/temp/model.sqlite', 
-                                                  'sqlite:///system/genai/temp/create_db_models.sqlite')
+                                                'sqlite:///system/genai/temp/create_db_models.sqlite')
 
                 # logicbank fixes
                 if 'from logic_bank' in each_line:  # we do our own imports
@@ -484,12 +604,135 @@ class GenAI(object):
                 if 'LogicBank.activate' in each_line:
                     each_line = each_line.replace('LogicBank.activate', '# LogicBank.activate')
                 
-                model_class += each_line + '\n'
-        with open(f'{self.project.from_model}', "w") as model_file:
-            model_file.write(model_class)
+                create_db_model_lines.append(each_line + '\n')
+            return create_db_model_lines
+    
+        def fix_model_lines(models, create_db_model_lines):
+            did_base = False
+            for each_model in models:
+                model_lines = get_model_class_lines(model=each_model)
+                for each_line in model_lines:
+                    each_fixed_line = each_line.replace('sa.', '')      # sometimes it puts sa. in front of Column
+                    if 'Base = declarative_base()' in each_fixed_line:  # sometimes created for each class
+                        if did_base:
+                            each_fixed_line = '# ' + each_fixed_line
+                        did_base = True 
+                    if 'datetime.datetime.utcnow' in each_fixed_line:
+                        each_fixed_line = each_fixed_line.replace('datetime.datetime.utcnow', 'datetime.now()') 
+                    if 'Column(date' in each_fixed_line:
+                        each_fixed_line = each_fixed_line.replace('Column(dat', 'column(Date') 
+                    create_db_model_lines.append(each_fixed_line)
+            return create_db_model_lines
+        
+        def insert_test_data_lines(test_data_lines : list[str]) -> list[str]:
+            """Insert test data lines into the model file
+
+            Args:
+                test_data_lines (list(str)):  
+                                       * initially header (engine =, sesssion =)
+                                       * this function appends CPT test data
+
+            Returns:
+                list[str]: variable names for the test data rows (for create_all)
+            """
+            
+            def fix_test_data_line(each_fixed_line: str) -> str:
+                """Fix the test data line
+
+                Args:
+                    each_fixed_line (str): the test data line
+
+                Returns:
+                    str: the fixed test data line
+                """
+
+                if '=datetime' in each_fixed_line:
+                    each_fixed_line = each_fixed_line.replace('=datetime.date', '=date') 
+                if 'datetime.datetime.utcnow' in each_fixed_line:
+                    each_fixed_line = each_fixed_line.replace('datetime.datetime.utcnow', 'datetime.now()') 
+                if 'engine = create_engine' in each_fixed_line:  # CBT sometimes has engine = create_engine, so do we!
+                    each_fixed_line = each_fixed_line.replace('engine = create_engine', '# engine = create_engine')
+                    check_for_row_name = False
+                if each_fixed_line.startswith('Base') or each_fixed_line.startswith('engine'):
+                    check_for_row_name = False
+                if 'Base.metadata.create_all(engine)' in each_fixed_line:
+                    each_fixed_line = each_fixed_line.replace('Base.metadata.create_all(engine)', '# Base.metadata.create_all(engine)')
+                return each_fixed_line
+
+            row_names = list()
+            use_test_data_rows = True # CPT test data, new format - test_data_rows (*way* less variable)
+            if use_test_data_rows & hasattr(self.response_dict, 'test_data_rows'):
+                test_data_rows = self.response_dict.test_data_rows
+                log.debug(f'.... test_data_rows: {len(test_data_rows)}')
+                for each_row in test_data_rows:
+                    each_fixed_line = fix_test_data_line(each_row.code)
+                    test_data_lines.append(each_fixed_line) 
+                    row_names.append(each_row.test_data_row_variable)
+                pass
+            else:  # CPT test data, old format - rows, plus session, engine etc (quite variable)
+                test_data_lines_ori = self.response_dict.test_data.split('\n') # gpt response
+                log.debug(f'.... test_data_lines...')
+                for each_line in test_data_lines_ori:
+                    each_fixed_line = fix_test_data_line(each_line)
+                    check_for_row_name = True
+                    test_data_lines.append(each_fixed_line)  # append the fixed test data line
+                    if check_for_row_name and ' = ' in each_line and '(' in each_line:  # CPT test data might have: tests = []
+                        assign = each_line.split(' = ')[0]
+                        # no tokens for: Session = sessionmaker(bind=engine) or session = Session()
+                        if '.' not in assign and 'Session' not in each_line and 'session.' not in each_line:
+                            row_names.append(assign)
+            return row_names
+        
+        create_db_model_lines =  list()
+        create_db_model_lines.append(f'# using resolved_model {self.resolved_model}')
+        create_db_model_lines.extend(  # imports for classes (comes from api_logic_server_cli/prototypes/manager/system/genai/create_db_models_inserts/create_db_models_imports.py)
+            self.get_lines_from_file(f'system/genai/create_db_models_inserts/create_db_models_imports.py'))
+        create_db_model_lines.append("\nfrom sqlalchemy.dialects.sqlite import *\n") # specific for genai 
+        
+        models = self.response_dict.models
+        
+        # Usage inside the class
+        create_db_model_lines = fix_model_lines(models, create_db_model_lines)
+
+        with open(f'{self.project.from_model}', "w") as create_db_model_file:
+            create_db_model_file.write("".join(create_db_model_lines))
+            create_db_model_file.write("\n\n# end of model classes\n\n")
+            
+        # classes done, create db and add test_data code
+        test_data_lines = self.get_lines_from_file(f'system/genai/create_db_models_inserts/create_db_models_create_db.py')
+        test_data_lines.append('session.commit()')
+        
+        row_names = insert_test_data_lines(test_data_lines)
+
+        test_data_lines.append('\n\n')
+        row_name_list = ', '.join(row_names)
+        add_rows = f'session.add_all([{row_name_list}])'
+        test_data_lines.append(add_rows )  
+        test_data_lines.append('session.commit()')
+        test_data_lines.append('# end of test data\n\n')
+        
+        with open(f'{self.project.from_model}', "a") as create_db_model_file:
+            create_db_model_file.write("try:\n    ")
+            create_db_model_file.write("\n    ".join(test_data_lines))
+            create_db_model_file.write("except Exception as exc:\n")
+            create_db_model_file.write("    print(f'Test Data Error: {exc}')\n")
         
-        log.debug(f'.. model file created: {self.project.from_model}')
+        log.debug(f'.. code for db creation and test data: {self.project.from_model}')
 
+    def get_lines_from_file(self, file_name: str) -> list[str]:
+        """Get lines from a file
+
+        Args:
+            file_name (str): the file name
+
+        Returns:
+            list[str]: the lines from the file
+        """
+
+        with open(file_name, "r") as file:
+            lines = file.readlines()
+        return lines
+    
     def save_prompt_messages_to_system_genai_temp_project(self):
         """
         Save prompts / responses to system/genai/temp/{project}/genai.response
@@ -509,90 +752,61 @@ class GenAI(object):
             if to_dir_save_dir.exists():
                 shutil.rmtree(to_dir_save_dir)
             os.makedirs(to_dir_save_dir, exist_ok=True)
-            log.debug(f'save_prompt_messages_to_system_genai_temp_project()')
+            log.debug(f'save_prompt_messages_to_system_genai_temp_project() - {str(to_dir_save_dir)}')
 
             if self.project.genai_repaired_response == '':  # normal path, from --using
                 if write_prompt := True:
                     pass
                     file_num = 0
+                    flat_project_name = Path(self.project.project_name).stem  # in case project is dir/project-name
                     for each_message in self.messages:
                         suffix = 'prompt'
                         if each_message['role'] == 'system':
                             suffix = 'response' 
-                        file_name = f'{self.project.project_name}_{str(file_num).zfill(3)}.{suffix}'
+                        file_name = f'{flat_project_name}_{str(file_num).zfill(3)}.{suffix}'
                         file_path = to_dir_save_dir.joinpath(file_name)
                         log.debug(f'.. saving[{file_name}]  - {each_message["content"][:30]}...')
                         with open(file_path, "w") as message_file:
                             message_file.write(each_message['content']) 
                         file_num += 1
                     suffix = 'response'  # now add the this response
-                    file_name = f'{self.project.project_name}_{str(file_num).zfill(3)}.{suffix}'
+                    file_name = f'{flat_project_name}_{str(file_num).zfill(3)}.{suffix}'  # FIXME 
                     file_path = to_dir_save_dir.joinpath(file_name)
                     log.debug(f'.. saving[{file_name}]  - {each_message["content"][:30]}...')
                     with open(file_path, "w") as message_file:
-                        message_file.write(self.create_db_models)
+                        json.dump(self.response_dict.toDict(), message_file, indent=4)
                 shutil.copyfile(self.project.from_model, to_dir_save_dir.joinpath('create_db_models.py'))
         except Exception as inst:
             # FileNotFoundError(2, 'No such file or directory')
-            log.error(f"\n\nError {inst} creating project diagnostic files: {str(gen_temp_dir)}\n\n")
+            log.error(f"\n\nError: {inst} \n..creating diagnostic files into dir: {str(gen_temp_dir)}\n\n")
             pass  # intentional try/catch/bury - it's just diagnostics, so don't fail
         debug_string = "good breakpoint - return to main driver, and execute create_db_models.py"
-
-    def get_headers_with_openai_api_key(self) -> dict:
-        """
-        Returns:
-            dict: api header with OpenAI key (exits if not provided)
-        """
-        
-        pass  # https://community.openai.com/t/how-do-i-call-chatgpt-api-with-python-code/554554
-        if os.getenv('APILOGICSERVER_CHATGPT_APIKEY'):
-            openai_api_key = os.getenv('APILOGICSERVER_CHATGPT_APIKEY')
-        else:
-            from dotenv import dotenv_values
-            secrets = dotenv_values("system/secrets.txt")
-            openai_api_key = secrets['APILOGICSERVER_CHATGPT_APIKEY']
-            if openai_api_key == 'your-api-key-here':
-                if os.getenv('APILOGICSERVER_CHATGPT_APIKEY'):
-                    openai_api_key = os.getenv('APILOGICSERVER_CHATGPT_APIKEY')
-                else:
-                    log.error("\n\nMissing env value: APILOGICSERVER_CHATGPT_APIKEY")
-                    log.error("... Check your system/secrets file...\n")
-                    exit(1)
-        headers = {
-            "Content-Type": "application/json",
-            "Authorization": f"Bearer {openai_api_key}"
-        }
-        return headers
     
-    def get_and_save_raw_response_data(self, response) -> str:
+    def get_and_save_raw_response_data(self, completion: object, response_dict: dict):
         """
-        Returns:
-            str: response_data
+        Write prompt --> system/genai/temp/chatgpt_original/retry.response
         """
         
-
+        '''  TODO - is exception used instead of return_code...
         # Check if the request was successful
-        if response.status_code == 400:
-            raise Exception("Bad ChatGPT Request: " + response.text)
+        if completion.status_code == 400:
+            raise Exception("Bad ChatGPT Request: " + completion.text)
         
-        if response.status_code != 200:
-            print("Error:", response.status_code, response.text)   # eg, You exceeded your current quota 
-
-        response_data = response.json()['choices'][0]['message']['content']
-        file_name = f'system/genai/temp/chatgpt_original.response'
-        with open(f'system/genai/temp/chatgpt_original.response', "w") as model_file:  # save for debug
-            model_file.write(response_data)
-            file_name = model_file.name
-        with open(f'system/genai/temp/chatgpt_retry.response', "w") as model_file:     # repair this & retry
-            model_file.write(response_data)
-        log.debug(f'.. stored raw response: {model_file.name}')
-        return response_data
+        if completion.status_code != 200:
+            print("Error:", completion.status_code, completion.text)   # eg, You exceeded your current quota 
+        '''
+        with open(f'system/genai/temp/chatgpt_original.response', "w") as response_file:  # save for debug
+            json.dump(response_dict, response_file, indent=4)
+        with open(f'system/genai/temp/chatgpt_retry.response', "w") as response_file:     # repair this & retry
+            json.dump(response_dict, response_file, indent=4)
+        return
 
 
-def genai(using, db_url, repaired_response: bool, genai_version: str, 
+def genai_cli_retry(using: str, db_url: str, repaired_response: str, genai_version: str, 
           retries: int, opt_locking: str, prompt_inserts: str, quote: bool,
-          use_relns: bool, project_name: str):
-    """ cli caller provides using, or repaired_response & using
+          use_relns: bool, project_name: str, tables: int, test_data_rows: int,
+          temperature: float) -> None:
+    """ CLI Caller: provides using, or repaired_response & using
     
         Called from cli commands: genai, genai-create, genai-iterate
         
@@ -603,45 +817,48 @@ def genai(using, db_url, repaired_response: bool, genai_version: str,
     import api_logic_server_cli.api_logic_server as PR
 
     resolved_project_name = project_name
-    if resolved_project_name == '' or resolved_project_name is None:
-        resolved_project_name = Path(using).stem  # default project name is the <cwd>/last node of using
+    if repaired_response != "":
+        if resolved_project_name == '' or resolved_project_name is None:
+            resolved_project_name = Path(using).stem  # project dir is the <cwd>/last node of using
     resolved_project_name  = resolved_project_name.replace(' ', '_')
+    start_time = time.time()
 
     try_number = 1
     genai_use_relns = use_relns
     """ if 'unable to determine join condition', we retry this with False """
     if repaired_response != "":
         try_number = retries  # if not calling GenAI, no need to retry:
-    # TODO or 0, right?
+    failed = False
+    pr = PR.ProjectRun(command="create", 
+                genai_version=genai_version, 
+                genai_temperature = temperature,
+                genai_using=using,                      # the prompt file, or dir of prompt/response
+                repaired_response=repaired_response,    # retry from [repaired] response file
+                opt_locking=opt_locking,
+                genai_prompt_inserts=prompt_inserts,
+                genai_use_relns=genai_use_relns,
+                quote=quote,
+                genai_tables=tables,
+                genai_test_data_rows=test_data_rows,
+                project_name=resolved_project_name, db_url=db_url,
+                execute=False)
     if retries < 0:  # for debug: catch exceptions at point of failure
-        PR.ProjectRun(command="create", genai_version=genai_version, 
-                    genai_using=using,                      # the prompt file, or conversation dir
-                    repaired_response=repaired_response,    # retry from [repaired] response file
-                    opt_locking=opt_locking,
-                    genai_prompt_inserts=prompt_inserts,
-                    genai_use_relns=genai_use_relns,
-                    quote=quote,
-                    project_name=resolved_project_name, db_url=db_url)
+        pr.create_project()  # calls GenAI() - the main driver
         log.info(f"GENAI successful")  
     else:
         while try_number <= retries:
             try:
                 failed = False
-                PR.ProjectRun(command="create", genai_version=genai_version, 
-                            genai_using=using,                      # the prompt file, or dir of prompt/response
-                            repaired_response=repaired_response,    # retry from [repaired] response file
-                            opt_locking=opt_locking,
-                            genai_prompt_inserts=prompt_inserts,
-                            genai_use_relns=genai_use_relns,
-                            quote=quote,
-                            project_name=resolved_project_name, db_url=db_url)
+                pr.create_project()  # calls GenAI() - the main driver
                 if do_force_failure := False:
                     if try_number < 3:
                         raise Exception("Forced Failure for Internal Testing")
                 break  # success - exit the loop
             except Exception as e:  # almost certaily in api_logic_server_cli/create_from_model/create_db_from_model.py
+                log.error(traceback.format_exc())
                 log.error(f"\n\nGenai failed With Error: {e}")
-
+                if resolved_project_name == '_genai_default':
+                    resolved_project_name = pr.project_name  # defaulted in genai from response
                 if Path(using).is_dir():
                     log.debug('conversation dir, check in-place iteration')
                     '''
@@ -687,6 +904,9 @@ def genai(using, db_url, repaired_response: bool, genai_version: str,
                 # to_dir_save_dir.rename(to_dir_save_dir_retry) 
                 assert to_dir_save_dir.is_dir(), f"\nInternal Error - missing save directory: {to_dir_save_dir}"
                 # assert to_dir_save_dir_retry.is_dir(), f"\nInternal Error - missing retry directory: {to_dir_save_dir_retry}"
+                log.debug(f'.. copying work files...')
+                log.debug(f'.... from: {to_dir_save_dir}')
+                log.debug(f'.... to:   {to_dir_save_dir_retry}')
                 shutil.copytree(to_dir_save_dir, to_dir_save_dir_retry, dirs_exist_ok=True) 
 
                 failed = True
@@ -696,11 +916,13 @@ def genai(using, db_url, repaired_response: bool, genai_version: str,
                     failed = False
                 else:
                     try_number += 1
+                    log.debug(f"\n\nRetry Genai #{try_number}\n")
             pass # retry (retries times)
-        if failed:
+        if failed == True:    # retries exhausted (if failed: threw "an integer is required" ??
+            pass                # https://github.com/microsoft/debugpy/issues/1708
             log.error(f"\n\nGenai Failed (Retries: {retries})") 
             exit(1) 
-        log.info(f"GENAI successful on try {try_number}")  
+        log.info(f"\nGENAI ({str(int(time.time() - start_time))} secs) successful on try {try_number}\n")  
 
 
 def key_module_map():