testeranto 0.200.1 → 0.203.0

package/src/utils/pitonoMetafile.ts

@@ -1,68 +1,340 @@
-import fs from 'fs';
-import path from 'path';
-import { execSync } from 'child_process';
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import fs from "fs";
+import path from "path";
 
 export interface PitonoMetafile {
-    testName: string;
-    entryPoints: string[];
-    timestamp: number;
+  errors: any[];
+  warnings: any[];
+  metafile: {
+    inputs: Record<
+      string,
+      {
+        bytes: number;
+        imports: any[];
+      }
+    >;
+    outputs: Record<
+      string,
+      {
+        imports: any[];
+        exports: any[];
+        entryPoint: string;
+        inputs: Record<
+          string,
+          {
+            bytesInOutput: number;
+          }
+        >;
+        bytes: number;
+        signature: string;
+      }
+    >;
+  };
 }
 
-export async function generatePitonoMetafile(testName: string, entryPoints: string[]): Promise<PitonoMetafile> {
-    return {
-        testName,
-        entryPoints,
-        timestamp: Date.now()
-    };
-}
-
-export function writePitonoMetafile(testName: string, metafile: PitonoMetafile): void {
-    const metafilePath = path.join(process.cwd(), 'testeranto', 'pitono', testName, 'metafile.json');
-    const metafileDir = path.dirname(metafilePath);
-    
-    // Ensure directory exists
-    if (!fs.existsSync(metafileDir)) {
-        fs.mkdirSync(metafileDir, { recursive: true });
+function resolvePythonImport(importPath: string, currentFile: string): string | null {
+    // Handle relative imports
+    if (importPath.startsWith('.')) {
+        const currentDir = path.dirname(currentFile);
+        // For relative imports, we need to handle the dots properly
+        // Count the number of dots to determine how many levels to go up
+        let dotCount = 0;
+        let remainingPath = importPath;
+        while (remainingPath.startsWith('.')) {
+            dotCount++;
+            remainingPath = remainingPath.substring(1);
+        }
+        
+        // Remove any leading slash
+        if (remainingPath.startsWith('/')) {
+            remainingPath = remainingPath.substring(1);
+        }
+        
+        // Build the base path by going up the appropriate number of directories
+        let baseDir = currentDir;
+        for (let i = 1; i < dotCount; i++) {
+            baseDir = path.dirname(baseDir);
+        }
+        
+        // Handle the case where there's no remaining path (just dots)
+        if (remainingPath.length === 0) {
+            const initPath = path.join(baseDir, '__init__.py');
+            if (fs.existsSync(initPath)) {
+                return initPath;
+            }
+            return null;
+        }
+        
+        // Resolve the full path
+        const resolvedPath = path.join(baseDir, remainingPath);
+        
+        // Try different extensions
+        const extensions = ['.py', '/__init__.py'];
+        for (const ext of extensions) {
+            const potentialPath = resolvedPath + ext;
+            if (fs.existsSync(potentialPath)) {
+                return potentialPath;
+            }
+        }
+        
+        // Check if it's a directory with __init__.py
+        if (fs.existsSync(resolvedPath) && fs.statSync(resolvedPath).isDirectory()) {
+            const initPath = path.join(resolvedPath, '__init__.py');
+            if (fs.existsSync(initPath)) {
+                return initPath;
+            }
+        }
+        return null;
     }
     
-    // Write the metafile
-    fs.writeFileSync(metafilePath, JSON.stringify(metafile, null, 2));
-    console.log(`Pitono metafile written to: ${metafilePath}`);
+    // Handle absolute imports by looking in the same directory and parent directories
+    // This is a simplification - Python's import system is more complex
+    const dirs = [
+        path.dirname(currentFile),
+        process.cwd(),
+        ...(process.env.PYTHONPATH ? process.env.PYTHONPATH.split(path.delimiter) : [])
+    ];
     
-    // Generate core.json using the Python script
+    for (const dir of dirs) {
+        const potentialPaths = [
+            path.join(dir, importPath + '.py'),
+            path.join(dir, importPath, '__init__.py'),
+            path.join(dir, importPath.replace(/\./g, '/') + '.py'),
+            path.join(dir, importPath.replace(/\./g, '/'), '__init__.py')
+        ];
+        
+        for (const potentialPath of potentialPaths) {
+            if (fs.existsSync(potentialPath)) {
+                return potentialPath;
+            }
+        }
+    }
+    
+    return null;
+}
+
+function parsePythonImports(filePath: string): {path: string, kind: string, external?: boolean, original?: string}[] {
     try {
-        // First try using the installed command
-        const command = `pitono-core-generator ${testName} ${metafile.entryPoints.join(' ')}`;
-        execSync(command, { stdio: 'inherit' });
-        console.log(`Pitono core.json generated successfully for ${testName}`);
+        const content = fs.readFileSync(filePath, 'utf-8');
+        const imports: {path: string, kind: string, external?: boolean, original?: string}[] = [];
+        
+        // Match import statements (including multiple imports)
+        const importRegex = /^import\s+([\w., ]+)/gm;
+        // Match from ... import statements
+        const fromImportRegex = /^from\s+([\w.]+)\s+import/gm;
+        
+        let match;
+        while ((match = importRegex.exec(content)) !== null) {
+            // Handle multiple imports in one line
+            const importPaths = match[1].split(',').map(p => p.trim());
+            for (const importPath of importPaths) {
+                // Try to resolve the import to see if it's external
+                const resolvedPath = resolvePythonImport(importPath, filePath);
+                imports.push({
+                    path: importPath,
+                    kind: 'import-statement',
+                    external: resolvedPath === null,
+                    original: importPath
+                });
+            }
+        }
+        
+        while ((match = fromImportRegex.exec(content)) !== null) {
+            const importPath = match[1];
+            // Try to resolve the import to see if it's external
+            const resolvedPath = resolvePythonImport(importPath, filePath);
+            imports.push({
+                path: importPath,
+                kind: 'import-statement',
+                external: resolvedPath === null,
+                original: importPath
+            });
+        }
+        
+        return imports;
     } catch (error) {
-        console.error(`Failed to generate Pitono core.json with installed command: ${error}`);
-        // Fallback to direct Python execution
-        try {
-            const pythonCommand = `python ${process.cwd()}/pitono/core_generator.py ${testName} ${metafile.entryPoints.join(' ')}`;
-            execSync(pythonCommand, { stdio: 'inherit' });
-            console.log(`Pitono core.json generated successfully using direct Python execution`);
-        } catch (fallbackError) {
-            console.error(`Direct Python execution also failed: ${fallbackError}`);
-            // Last resort: create the core.json manually
-            try {
-                const coreData = {
-                    testName: testName,
-                    entryPoints: metafile.entryPoints,
-                    outputs: {},
-                    metafile: {
-                        inputs: {},
-                        outputs: {}
-                    },
-                    timestamp: Date.now(),
-                    runtime: "pitono"
+        console.warn(`Could not parse imports for ${filePath}:`, error);
+        return [];
+    }
+}
+
+function collectDependencies(filePath: string, visited: Set<string> = new Set()): string[] {
+    if (visited.has(filePath)) return [];
+    visited.add(filePath);
+    
+    const dependencies: string[] = [filePath];
+    const imports = parsePythonImports(filePath);
+    
+    for (const imp of imports) {
+        if (!imp.external && imp.path) {
+            const resolvedPath = resolvePythonImport(imp.path, filePath);
+            if (resolvedPath && fs.existsSync(resolvedPath)) {
+                dependencies.push(...collectDependencies(resolvedPath, visited));
+            }
+        }
+    }
+    
+    return [...new Set(dependencies)];
+}
+
+export async function generatePitonoMetafile(
+    testName: string,
+    entryPoints: string[]
+): Promise<PitonoMetafile> {
+    const inputs: Record<string, any> = {};
+    const outputs: Record<string, any> = {};
+    const signature = Date.now().toString(36);
+
+    // Process each entry point
+    for (const entryPoint of entryPoints) {
+        if (!fs.existsSync(entryPoint)) {
+            console.warn(`Entry point ${entryPoint} does not exist`);
+            continue;
+        }
+
+        // Collect all dependencies recursively
+        const allDependencies = collectDependencies(entryPoint);
+        
+        // Process each dependency to add to inputs
+        for (const dep of allDependencies) {
+            if (!inputs[dep]) {
+                const bytes = fs.statSync(dep).size;
+                const imports = parsePythonImports(dep);
+                inputs[dep] = {
+                    bytes,
+                    imports,
                 };
-                const coreFilePath = path.join(process.cwd(), 'testeranto', 'pitono', testName, 'core.json');
-                fs.writeFileSync(coreFilePath, JSON.stringify(coreData, null, 2));
-                console.log(`Pitono core.json created manually as fallback`);
-            } catch (manualError) {
-                console.error(`Even manual creation failed: ${manualError}`);
             }
         }
+
+        // Generate output path
+        const entryPointName = path.basename(entryPoint, ".py");
+        const outputKey = `python/core/${entryPointName}.py`;
+        
+        // Calculate total bytes from all inputs
+        const inputBytes: Record<string, { bytesInOutput: number }> = {};
+        let totalBytes = 0;
+        for (const dep of allDependencies) {
+            const bytes = fs.statSync(dep).size;
+            inputBytes[dep] = { bytesInOutput: bytes };
+            totalBytes += bytes;
+        }
+        
+        outputs[outputKey] = {
+            imports: [],
+            exports: [],
+            entryPoint,
+            inputs: inputBytes,
+            bytes: totalBytes,
+            signature,
+        };
+    }
+
+    // If no valid entry points were found, add a placeholder
+    if (Object.keys(inputs).length === 0) {
+        inputs["placeholder.py"] = {
+            bytes: 0,
+            imports: [],
+        };
+        outputs["testeranto/bundles/python/core/placeholder.py"] = {
+            imports: [],
+            exports: [],
+            entryPoint: "placeholder.py",
+            inputs: {
+                "placeholder.py": {
+                    bytesInOutput: 0,
+                },
+            },
+            bytes: 0,
+            signature: "placeholder",
+        };
+    }
+
+    return {
+        errors: [],
+        warnings: [],
+        metafile: {
+            inputs,
+            outputs,
+        },
+    };
+}
+
+export function writePitonoMetafile(
+  testName: string,
+  metafile: PitonoMetafile
+): void {
+  // Always use the project root for testeranto directories
+  const projectRoot = process.cwd();
+  const metafilePath = path.join(
+    projectRoot,
+    "testeranto",
+    "metafiles",
+    "python",
+    "core.json"
+  );
+  const metafileDir = path.dirname(metafilePath);
+
+  // Ensure directory exists
+  if (!fs.existsSync(metafileDir)) {
+    fs.mkdirSync(metafileDir, { recursive: true });
+  }
+
+  // Write the metafile
+  fs.writeFileSync(metafilePath, JSON.stringify(metafile, null, 2));
+
+  // Generate the output Python files in the core directory
+  const outputDir = path.join(
+    projectRoot,
+    "testeranto",
+    "bundles",
+    "python",
+    "core"
+  );
+  if (!fs.existsSync(outputDir)) {
+    fs.mkdirSync(outputDir, { recursive: true });
+  }
+
+  // Create each output file
+  for (const [outputPath, outputInfo] of Object.entries(
+    metafile.metafile.outputs
+  )) {
+    // Use just the filename part of the output path
+    const fileName = path.basename(outputPath);
+    const fullOutputPath = path.join(outputDir, fileName);
+    const outputDirPath = path.dirname(fullOutputPath);
+
+    if (!fs.existsSync(outputDirPath)) {
+      fs.mkdirSync(outputDirPath, { recursive: true });
     }
+
+    const entryPoint = (outputInfo as any).entryPoint;
+    const signature = (outputInfo as any).signature;
+    const content = `# This file is auto-generated by testeranto
+# Signature: ${signature}
+# It runs tests from: ${entryPoint}
+
+import os
+import sys
+
+# Add the original entry point to the path
+sys.path.insert(0, os.path.dirname(os.path.abspath("${entryPoint}")))
+
+# Import and run the tests
+try:
+    # Import the module
+    module_name = os.path.basename("${entryPoint}").replace('.py', '')
+    module = __import__(module_name)
+    
+    # Run the tests if there's a main block
+    if hasattr(module, 'main'):
+        module.main()
+    else:
+        print(f"No main function found in {module_name}")
+except Exception as e:
+    print(f"Error running tests from ${entryPoint}: {e}")
+    sys.exit(1)
+`;
+    fs.writeFileSync(fullOutputPath, content);
+  }
 }

package/src/utils/pitonoWatcher.ts

@@ -0,0 +1,157 @@
+/* eslint-disable @typescript-eslint/no-unused-vars */
+import chokidar from "chokidar";
+import type { FSWatcher } from "chokidar";
+import {
+  generatePitonoMetafile,
+  writePitonoMetafile,
+} from "./pitonoMetafile";
+import path from "path";
+import fs from "fs";
+
+export class PitonoWatcher {
+  private watcher: FSWatcher | null = null;
+  private testName: string;
+  private entryPoints: string[];
+  private onChangeCallback: (() => void) | null = null;
+
+  constructor(testName: string, entryPoints: string[]) {
+    this.testName = testName;
+    this.entryPoints = entryPoints;
+  }
+
+  async start() {
+    // Watch source Python files to trigger metafile regeneration
+    const pythonFilesPattern = "**/*.py";
+
+    this.watcher = chokidar.watch(pythonFilesPattern, {
+      persistent: true,
+      ignoreInitial: true,
+      cwd: process.cwd(),
+      ignored: [
+        "**/node_modules/**",
+        "**/.git/**",
+        "**/testeranto/bundles/**",
+        "**/testeranto/reports/**",
+      ],
+      usePolling: true,
+      interval: 1000,
+      binaryInterval: 1000,
+      depth: 99,
+      followSymlinks: false,
+      atomic: false,
+    });
+
+    // Add event listeners for source file changes
+    this.watcher
+      .on("add", (filePath) => {
+        this.handleFileChange("add", filePath);
+      })
+      .on("change", (filePath) => {
+        this.handleFileChange("change", filePath);
+      })
+      .on("unlink", (filePath) => {
+        this.handleFileChange("unlink", filePath);
+      })
+      .on("error", (error) => {
+        console.error(`Source watcher error: ${error}`);
+      })
+      .on("ready", () => {
+        console.log(
+          "Initial python source file scan complete. Ready for changes."
+        );
+      });
+
+    // Second watcher: watches bundle files to schedule tests when they change
+    const outputDir = path.join(
+      process.cwd(),
+      `testeranto/bundles/python/${this.testName}`
+    );
+    // Ensure the output directory exists
+    if (!fs.existsSync(outputDir)) {
+      fs.mkdirSync(outputDir, { recursive: true });
+    }
+
+    // Track the last seen signatures to detect changes
+    const lastSignatures = new Map<string, string>();
+
+    // Create a separate watcher for bundle files
+    const bundleWatcher = chokidar.watch(path.join(outputDir, "*.py"), {
+      persistent: true,
+      ignoreInitial: false,
+      awaitWriteFinish: {
+        stabilityThreshold: 100,
+        pollInterval: 50,
+      },
+    });
+
+    bundleWatcher
+      .on("add", (filePath) => {
+        this.readAndCheckSignature(filePath, lastSignatures);
+      })
+      .on("change", (filePath) => {
+        this.readAndCheckSignature(filePath, lastSignatures);
+      })
+      .on("error", (error) => console.error(`Bundle watcher error: ${error}`));
+
+    // Initial metafile generation
+    await this.regenerateMetafile();
+  }
+
+  private async handleFileChange(event: string, filePath: string) {
+    // Add a small delay to ensure the file is fully written
+    await new Promise((resolve) => setTimeout(resolve, 100));
+    await this.regenerateMetafile();
+    if (this.onChangeCallback) {
+      this.onChangeCallback();
+    }
+  }
+
+  private readAndCheckSignature(
+    filePath: string,
+    lastSignatures: Map<string, string>
+  ) {
+    try {
+      const content = fs.readFileSync(filePath, "utf-8");
+      // Extract the signature from the comment
+      const signatureMatch = content.match(/# Signature: (\w+)/);
+      if (signatureMatch && signatureMatch[1]) {
+        const currentSignature = signatureMatch[1];
+        const lastSignature = lastSignatures.get(filePath);
+
+        if (lastSignature === undefined) {
+          lastSignatures.set(filePath, currentSignature);
+        } else if (lastSignature !== currentSignature) {
+          lastSignatures.set(filePath, currentSignature);
+          if (this.onChangeCallback) {
+            this.onChangeCallback();
+          }
+        }
+      }
+    } catch (error) {
+      console.error(`Error reading bundle file ${filePath}:`, error);
+    }
+  }
+
+  private async regenerateMetafile() {
+    try {
+      const metafile = await generatePitonoMetafile(
+        this.testName,
+        this.entryPoints
+      );
+      writePitonoMetafile(this.testName, metafile);
+    } catch (error) {
+      console.error("Error regenerating pitono metafile:", error);
+    }
+  }
+
+  onMetafileChange(callback: () => void) {
+    this.onChangeCallback = callback;
+  }
+
+  stop() {
+    if (this.watcher) {
+      this.watcher.close();
+      this.watcher = null;
+    }
+  }
+}

package/src/utils.ts

@@ -4,6 +4,17 @@ import path from "path";
 import { IBuiltConfig, IRunTime, ITestTypes } from "./Types";
 import { IRunnables } from "./lib";
 
+export const webEvaluator = (d, webArgz) => {
+  return `
+import('${d}').then(async (x) => {
+  try {
+    return await (await x.default).receiveTestResourceConfig(${webArgz})
+  } catch (e) {
+    console.log("web run failure", e.toString())
+  }
+})
+`;
+};
 export const destinationOfRuntime = (
   f: string,
   r: IRunTime,
@@ -80,7 +91,6 @@ export const getRunnables = (
   tests: ITestTypes[],
   projectName: string,
   payload = {
-    pythonEntryPoints: {},
     nodeEntryPoints: {},
     nodeEntryPointSidecars: {},
     webEntryPoints: {},
@@ -89,13 +99,12 @@ export const getRunnables = (
     pureEntryPointSidecars: {},
     golangEntryPoints: {},
     golangEntryPointSidecars: {},
-    pitonoEntryPoints: {},
-    pitonoEntryPointSidecars: {},
+    pythonEntryPoints: {},
+    pythonEntryPointSidecars: {},
   }
 ): IRunnables => {
   // Ensure all properties are properly initialized
   const initializedPayload = {
-    pythonEntryPoints: payload.pythonEntryPoints || {},
     nodeEntryPoints: payload.nodeEntryPoints || {},
     nodeEntryPointSidecars: payload.nodeEntryPointSidecars || {},
     webEntryPoints: payload.webEntryPoints || {},
@@ -104,8 +113,8 @@ export const getRunnables = (
     pureEntryPointSidecars: payload.pureEntryPointSidecars || {},
     golangEntryPoints: payload.golangEntryPoints || {},
     golangEntryPointSidecars: payload.golangEntryPointSidecars || {},
-    pitonoEntryPoints: payload.pitonoEntryPoints || {},
-    pitonoEntryPointSidecars: payload.pitonoEntryPointSidecars || {},
+    pythonEntryPoints: payload.pythonEntryPoints || {},
+    pythonEntryPointSidecars: payload.pythonEntryPointSidecars || {},
   };
   return tests.reduce((pt, cv, cndx, cry) => {
     if (cv[1] === "node") {
@@ -135,9 +144,9 @@ export const getRunnables = (
     } else if (cv[1] === "golang") {
       // For Go files, we'll use the original path since they're not compiled to JS
       pt.golangEntryPoints[cv[0]] = path.resolve(cv[0]);
-    } else if (cv[1] === "pitono") {
-      // For pitono files, use the original Python file path
-      pt.pitonoEntryPoints[cv[0]] = path.resolve(cv[0]);
+    } else if (cv[1] === "python") {
+      // For python files, use the original Python file path
+      pt.pythonEntryPoints[cv[0]] = path.resolve(cv[0]);
     }
 
     //////////////////////////////////////////////////////////
@@ -182,10 +191,10 @@ export const getRunnables = (
         pt.golangEntryPointSidecars[`${t[0]}`] = path.resolve(t[0]);
       });
     cv[3]
-      .filter((t) => t[1] === "pitono")
+      .filter((t) => t[1] === "python")
       .forEach((t) => {
-        // For pitono sidecars, use the original Python file path
-        pt.pitonoEntryPointSidecars[`${t[0]}`] = path.resolve(t[0]);
+        // For python sidecars, use the original Python file path
+        pt.pythonEntryPointSidecars[`${t[0]}`] = path.resolve(t[0]);
       });
 
     return pt;