pynect 0.0.0-stage → 0.2.0

package/src/native/python_bridge.h

@@ -0,0 +1,263 @@
+/**
+ * pynect - Python Bridge Header
+ * High-level operations on Python objects: import, call, attribute access.
+ * Uses TypeConverter for serialization and ThreadPool for async execution.
+ */
+
+#pragma once
+
+#include <Python.h>
+#include <napi.h>
+#include <string>
+#include <vector>
+#include <memory>
+#include <functional>
+#include <map>
+
+#include "type_converter.h"
+#include "thread_pool.h"
+
+namespace pynect {
+
+/**
+ * Options for function calls.
+ */
+struct CallOptions {
+  std::vector<SerializedValue> args;
+  std::vector<std::pair<std::string, SerializedValue>> kwargs;
+  uint32_t timeoutMs = 0;
+  uint32_t executionTimeoutMs = 0;
+  std::string requestId;
+};
+
+/**
+ * Attribute access result.
+ */
+struct AttributeResult {
+  bool success = false;
+  SerializedValue value;
+  bool isCallable = false;
+  bool isObject = false;
+  uint32_t objectId = 0;
+  std::string errorType;
+  std::string errorMessage;
+  std::string errorTraceback;
+};
+
+/**
+ * PythonBridge - orchestrates Python operations.
+ *
+ * This class provides the business logic on top of raw Python C API calls.
+ * All operations are designed to be executed in worker threads (GIL held).
+ */
+class PythonBridge {
+public:
+  static PythonBridge& getInstance();
+
+  // Non-copyable singleton
+  PythonBridge(const PythonBridge&) = delete;
+  PythonBridge& operator=(const PythonBridge&) = delete;
+
+  // Python Lifecycle
+
+  /**
+   * Initialize the Python interpreter.
+   * Must be called once from the main thread before any Python operations.
+   */
+  void initializePython(
+    const std::string& pythonHome,
+    const std::string& pythonExecutable,
+    const std::string& pythonLibrary,
+    const std::vector<std::string>& extraSysPaths,
+    const std::vector<std::pair<std::string, std::string>>& envVars
+  );
+
+  /**
+   * Finalize the Python interpreter.
+   * Must be called once from the main thread at shutdown.
+   */
+  void finalizePython();
+
+  bool isInitialized() const noexcept { return _initialized; }
+
+  // Module Import
+
+  /**
+   * Import a Python module. Returns objectId.
+   * Sync version (GIL must be held by caller).
+   */
+  TaskResult importModuleSync(const std::string& moduleName);
+
+  /**
+   * Reload a module by objectId.
+   */
+  TaskResult reloadModuleSync(uint32_t objectId);
+
+  // Attribute Access
+
+  /**
+   * Get attribute from a Python object by name.
+   */
+  TaskResult getAttributeSync(uint32_t objectId, const std::string& attrName);
+
+  /**
+   * Get attribute path (chain of attribute accesses).
+   * e.g. path = ["groupby", "agg"] means obj.groupby.agg
+   */
+  TaskResult getAttributePathSync(
+    uint32_t objectId,
+    const std::vector<std::string>& path
+  );
+
+  /**
+   * Set attribute on a Python object.
+   */
+  TaskResult setAttributeSync(
+    uint32_t objectId,
+    const std::string& attrName,
+    const SerializedValue& value
+  );
+
+  /**
+   * Check if an attribute exists.
+   */
+  bool hasAttributeSync(uint32_t objectId, const std::string& attrName);
+
+  /**
+   * Delete an attribute.
+   */
+  TaskResult deleteAttributeSync(uint32_t objectId, const std::string& attrName);
+
+  // Function Calls
+
+  /**
+   * Call a Python callable (resolving attribute path first).
+   */
+  TaskResult callFunctionSync(
+    uint32_t objectId,
+    const std::vector<std::string>& path,
+    const CallOptions& options
+  );
+
+  /**
+   * Call a method by name on a Python object.
+   */
+  TaskResult callMethodSync(
+    uint32_t objectId,
+    const std::string& methodName,
+    const CallOptions& options
+  );
+
+  /** Cooperatively cancel a running asynchronous call. */
+  TaskResult cancelCallSync(const std::string& requestId);
+
+  // Direct Evaluation
+
+  /**
+   * Run arbitrary Python code (statements).
+   */
+  TaskResult runCodeSync(const std::string& code);
+
+  /**
+   * Evaluate a Python expression and return the result.
+   */
+  TaskResult evalExpressionSync(const std::string& expression);
+
+  /**
+   * Run a Python file.
+   */
+  TaskResult runFileSync(const std::string& filePath);
+
+  // Object Operations
+
+  /**
+   * Get the serialized value of an object (resolve it).
+   */
+  TaskResult getObjectValueSync(uint32_t objectId);
+
+  /**
+   * Get repr() of an object.
+   */
+  std::string getObjectReprSync(uint32_t objectId);
+
+  /**
+   * Get len() of an object.
+   */
+  TaskResult getObjectLengthSync(uint32_t objectId);
+
+  /**
+   * Get list of attribute names (dir()).
+   */
+  TaskResult getObjectKeysSync(uint32_t objectId);
+
+  // Iterator Support
+
+  /**
+   * Create an iterator for an object.
+   */
+  TaskResult createIteratorSync(uint32_t objectId, const std::vector<std::string>& path);
+
+  /**
+   * Get next value from an iterator.
+   */
+  TaskResult iteratorNextSync(uint32_t iteratorId);
+
+  /**
+   * Destroy an iterator.
+   */
+  void destroyIteratorSync(uint32_t iteratorId);
+
+  // Memory
+
+  void collectGarbage();
+  size_t getTrackedObjectCount() const;
+
+private:
+  PythonBridge() = default;
+  ~PythonBridge() = default;
+
+  /**
+   * Resolve attribute path starting from an objectId.
+   * Returns new Python reference, or nullptr on error.
+   * GIL must be held.
+   */
+  PyObject* _resolveAttributePath(
+    PyObject* obj,
+    const std::vector<std::string>& path
+  );
+
+  /**
+   * Convert a SerializedValue back to a Python object.
+   * Returns new reference.
+   */
+  PyObject* _deserialize(const SerializedValue& value);
+
+  /**
+   * Build Python args tuple and kwargs dict from CallOptions.
+   */
+  bool _buildCallArgs(
+    const CallOptions& options,
+    PyObject*& argsOut,
+    PyObject*& kwargsOut
+  );
+
+  /**
+   * Build a TaskResult from Python exception state.
+   */
+  TaskResult _makeErrorResult();
+
+  /**
+   * Build a successful TaskResult from a Python object.
+   */
+  TaskResult _makeSuccessResult(PyObject* obj);
+
+  bool _initialized = false;
+  PyThreadState* _mainThreadState = nullptr;
+
+  // Iterator registry: iteratorId -> Python iterator object
+  std::map<uint32_t, PyObject*> _iterators;
+  std::mutex _iteratorsMutex;
+  std::atomic<uint32_t> _nextIteratorId{1};
+};
+
+} // namespace pynect

package/src/native/thread_pool.cpp

@@ -0,0 +1,375 @@
+/**
+ * pynect - Thread Pool Implementation
+ * Worker thread management for executing Python code off the event loop.
+ */
+
+#include "thread_pool.h"
+#include "gil_guard.h"
+
+#include <Python.h>
+#include <algorithm>
+#include <cassert>
+#include <stdexcept>
+#include <iostream>
+
+namespace pynect {
+
+// Constructor / Destructor
+
+PythonThreadPool::PythonThreadPool(uint32_t numThreads, uint32_t maxQueueSize)
+  : _numThreads(numThreads > 0 ? numThreads : 1)
+  , _maxQueueSize(maxQueueSize > 0 ? maxQueueSize : 1000)
+{
+  // Validate parameters
+  if (_numThreads > 64) {
+    std::cerr << "[pynect] Warning: Thread pool size " << _numThreads
+              << " is very large. Consider using <= 16 threads.\n";
+  }
+}
+
+PythonThreadPool::~PythonThreadPool() {
+  if (_running.load()) {
+    forceShutdown();
+  }
+}
+
+// Initialization
+
+void PythonThreadPool::initialize(Napi::Env env) {
+  if (_running.load()) {
+    throw std::runtime_error("PythonThreadPool already initialized");
+  }
+
+  // Create a ThreadSafeFunction for delivering results back to the JS event loop.
+  // This is the safe way to call JS functions from non-JS threads.
+  _tsfn = Napi::ThreadSafeFunction::New(
+    env,
+    Napi::Function::New(env, [](const Napi::CallbackInfo&) {}),  // dummy JS function
+    "PythonThreadPoolCallback",
+    0,   // unlimited queue size
+    1    // initial thread count (we manage threads ourselves)
+  );
+  _env = env;
+  _tsfn.Unref(env);
+
+  _running.store(true);
+  _draining.store(false);
+
+  // Spawn worker threads
+  _threads.reserve(_numThreads);
+  for (uint32_t i = 0; i < _numThreads; ++i) {
+    _threads.emplace_back(&PythonThreadPool::_workerThread, this, i);
+  }
+}
+
+// Task Submission
+
+uint64_t PythonThreadPool::submit(
+  std::function<TaskResult()> work,
+  std::function<void(TaskResult)> callback,
+  uint32_t timeoutMs,
+  std::string cancelKey
+) {
+  if (!_running.load()) {
+    TaskResult errorResult;
+    errorResult.success = false;
+    errorResult.errorType = "RuntimeError";
+    errorResult.errorMessage = "Thread pool is shut down";
+    errorResult.errorTraceback = "";
+    callback(std::move(errorResult));
+    return 0;
+  }
+
+  // Check queue capacity
+  {
+    std::lock_guard<std::mutex> lock(_queueMutex);
+    if (_queue.size() >= _maxQueueSize) {
+      // Backpressure: queue is full
+      TaskResult errorResult;
+      errorResult.success = false;
+      errorResult.errorType = "RuntimeError";
+      errorResult.errorMessage = "Python thread pool queue is full (max: " +
+                                  std::to_string(_maxQueueSize) + "). " +
+                                  "Consider increasing threadPoolSize or maxQueueSize.";
+      errorResult.errorTraceback = "";
+      callback(std::move(errorResult));
+      return 0;
+    }
+  }
+
+  uint64_t taskId = _nextTaskId.fetch_add(1, std::memory_order_relaxed);
+
+  PythonTask task;
+  task.id = taskId;
+  task.work = std::move(work);
+  task.callback = std::move(callback);
+  task.submittedAt = std::chrono::steady_clock::now();
+  task.timeoutMs = timeoutMs;
+  task.cancelKey = std::move(cancelKey);
+
+  // Keep Node alive only while a submitted task still owes JavaScript a result.
+  if (_pendingResults.fetch_add(1, std::memory_order_acq_rel) == 0) {
+    _tsfn.Ref(_env);
+  }
+
+  {
+    std::lock_guard<std::mutex> lock(_queueMutex);
+    _queue.push_back(std::move(task));
+    _stats.totalTasksSubmitted.fetch_add(1, std::memory_order_relaxed);
+    _stats.currentQueueSize.fetch_add(1, std::memory_order_relaxed);
+  }
+
+  _queueCv.notify_one();
+  return taskId;
+}
+
+bool PythonThreadPool::cancel(const std::string& cancelKey) {
+  if (cancelKey.empty()) return false;
+  PythonTask task;
+  {
+    std::lock_guard<std::mutex> lock(_queueMutex);
+    auto found = std::find_if(_queue.begin(), _queue.end(), [&cancelKey](const PythonTask& candidate) {
+      return candidate.cancelKey == cancelKey;
+    });
+    if (found == _queue.end()) return false;
+    task = std::move(*found);
+    _queue.erase(found);
+    _stats.currentQueueSize.fetch_sub(1, std::memory_order_relaxed);
+    _stats.totalTasksFailed.fetch_add(1, std::memory_order_relaxed);
+  }
+  TaskResult cancelled;
+  cancelled.success = false;
+  cancelled.errorType = "CancelledError";
+  cancelled.errorMessage = "Python call was cancelled before execution";
+  _postResult(task.id, std::move(cancelled), std::move(task.callback));
+  if (_draining.load()) _drainCv.notify_all();
+  return true;
+}
+
+// Worker Thread
+
+void PythonThreadPool::_workerThread(uint32_t threadId) {
+  // Give the thread a name for debugging
+  // Note: thread naming is platform-specific
+#ifdef __linux__
+  pthread_setname_np(pthread_self(), ("pynect-worker-" + std::to_string(threadId)).c_str());
+#endif
+
+  while (_running.load()) {
+    PythonTask task;
+
+    // Wait for a task
+    {
+      std::unique_lock<std::mutex> lock(_queueMutex);
+      _queueCv.wait(lock, [this] {
+        return !_queue.empty() || !_running.load();
+      });
+
+      if (!_running.load() && _queue.empty()) {
+        break;
+      }
+
+      if (_queue.empty()) {
+        continue;
+      }
+
+      task = std::move(_queue.front());
+      _queue.pop_front();
+      _stats.currentQueueSize.fetch_sub(1, std::memory_order_relaxed);
+    }
+
+    // Check timeout before executing
+    if (_isTimedOut(task)) {
+      TaskResult timeoutResult;
+      timeoutResult.success = false;
+      timeoutResult.errorType = "TimeoutError";
+      timeoutResult.errorMessage = "Python task timed out after " +
+                                    std::to_string(task.timeoutMs) + "ms in queue";
+      timeoutResult.errorTraceback = "";
+      _stats.totalTasksTimedOut.fetch_add(1, std::memory_order_relaxed);
+      _postResult(task.id, std::move(timeoutResult), std::move(task.callback));
+      continue;
+    }
+
+    // Execute the task with GIL held
+    _stats.activeThreads.fetch_add(1, std::memory_order_relaxed);
+
+    auto startTime = std::chrono::steady_clock::now();
+    TaskResult result;
+
+    try {
+      // Acquire GIL - critical: must be released when we're done
+      GILAcquire gil;
+
+      // Check timeout again (task might have waited in queue)
+      if (task.timeoutMs > 0 && _isTimedOut(task)) {
+        result.success = false;
+        result.errorType = "TimeoutError";
+        result.errorMessage = "Python task timed out";
+        result.errorTraceback = "";
+      } else {
+        // Execute the Python work
+        result = task.work();
+      }
+    } catch (const std::exception& e) {
+      result.success = false;
+      result.errorType = "RuntimeError";
+      result.errorMessage = std::string("C++ exception in worker thread: ") + e.what();
+      result.errorTraceback = "";
+    } catch (...) {
+      result.success = false;
+      result.errorType = "RuntimeError";
+      result.errorMessage = "Unknown C++ exception in worker thread";
+      result.errorTraceback = "";
+    }
+
+    auto endTime = std::chrono::steady_clock::now();
+    auto durationUs = std::chrono::duration_cast<std::chrono::microseconds>(endTime - startTime).count();
+    _stats.totalExecutionTimeUs.fetch_add(static_cast<uint64_t>(durationUs), std::memory_order_relaxed);
+
+    if (result.success) {
+      _stats.totalTasksCompleted.fetch_add(1, std::memory_order_relaxed);
+    } else {
+      _stats.totalTasksFailed.fetch_add(1, std::memory_order_relaxed);
+    }
+
+    _stats.activeThreads.fetch_sub(1, std::memory_order_relaxed);
+
+    // Post result back to Node.js event loop
+    _postResult(task.id, std::move(result), std::move(task.callback));
+
+    // Signal drain waiters
+    if (_draining.load()) {
+      std::lock_guard<std::mutex> lock(_queueMutex);
+      _drainCv.notify_all();
+    }
+  }
+}
+
+// Result Delivery
+
+void PythonThreadPool::_postResult(
+  uint64_t taskId,
+  TaskResult result,
+  std::function<void(TaskResult)> callback
+) {
+  // Package the callback + result into a heap allocation
+  // that will be freed in the JS thread
+  auto* payload = new std::pair<std::function<void(TaskResult)>, TaskResult>(
+    std::move(callback),
+    std::move(result)
+  );
+
+  // Use ThreadSafeFunction to invoke the callback in the JS event loop
+  napi_status status = _tsfn.NonBlockingCall(
+    payload,
+    [this](Napi::Env env, Napi::Function jsCallback, void* data) {
+      auto* p = static_cast<std::pair<std::function<void(TaskResult)>, TaskResult>*>(data);
+      try {
+        p->first(std::move(p->second));
+      } catch (...) {
+        // Don't let JS callback exceptions crash the worker
+      }
+      delete p;
+      _resultDelivered(env);
+    }
+  );
+
+  if (status != napi_ok) {
+    // This can happen during shutdown; just clean up
+    auto* p = static_cast<std::pair<std::function<void(TaskResult)>, TaskResult>*>(payload);
+    try {
+      // Try to call with error result
+      TaskResult shutdownResult;
+      shutdownResult.success = false;
+      shutdownResult.errorType = "RuntimeError";
+      shutdownResult.errorMessage = "Thread pool shut down while task was running";
+      p->first(std::move(shutdownResult));
+    } catch (...) {}
+    delete p;
+    _pendingResults.fetch_sub(1, std::memory_order_acq_rel);
+  }
+}
+
+void PythonThreadPool::_resultDelivered(Napi::Env env) {
+  if (_pendingResults.fetch_sub(1, std::memory_order_acq_rel) == 1) {
+    _tsfn.Unref(env);
+  }
+}
+
+// Timeout Check
+
+bool PythonThreadPool::_isTimedOut(const PythonTask& task) const {
+  if (task.timeoutMs == 0) return false;
+
+  auto now = std::chrono::steady_clock::now();
+  auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
+    now - task.submittedAt
+  ).count();
+
+  return elapsed >= static_cast<long long>(task.timeoutMs);
+}
+
+// Resize
+
+void PythonThreadPool::resize(uint32_t newNumThreads) {
+  if (!_running.load()) return;
+  if (newNumThreads == 0) newNumThreads = 1;
+
+  // For simplicity: add threads if growing, remove by setting flag if shrinking
+  // A production implementation would use a more sophisticated approach
+  uint32_t current = static_cast<uint32_t>(_threads.size());
+
+  if (newNumThreads > current) {
+    for (uint32_t i = current; i < newNumThreads; ++i) {
+      _threads.emplace_back(&PythonThreadPool::_workerThread, this, i);
+    }
+  }
+  // Shrinking is complex and rarely needed; skip for now
+  _numThreads = newNumThreads;
+}
+
+// Shutdown
+
+void PythonThreadPool::shutdown() {
+  if (!_running.load()) return;
+
+  // Drain the queue first
+  _draining.store(true);
+  {
+    std::unique_lock<std::mutex> lock(_queueMutex);
+    _drainCv.wait_for(lock, std::chrono::seconds(30), [this] {
+      return _queue.empty() && _stats.activeThreads.load() == 0;
+    });
+  }
+  _draining.store(false);
+
+  forceShutdown();
+}
+
+void PythonThreadPool::forceShutdown() {
+  if (!_running.load()) return;
+
+  _running.store(false);
+  _queueCv.notify_all();
+
+  // Wait for all threads to finish
+  for (auto& thread : _threads) {
+    if (thread.joinable()) {
+      thread.join();
+    }
+  }
+  _threads.clear();
+
+  // Release ThreadSafeFunction
+  _tsfn.Release();
+}
+
+// Queue Size
+
+size_t PythonThreadPool::getQueueSize() const {
+  std::lock_guard<std::mutex> lock(_queueMutex);
+  return _queue.size();
+}
+
+} // namespace pynect