dc-python-sdk 1.5.23__tar.gz → 1.5.24__tar.gz

{dc_python_sdk-1.5.23/src/dc_python_sdk.egg-info → dc_python_sdk-1.5.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dc-python-sdk
-Version: 1.5.23
+Version: 1.5.24
 Summary: Data Connector Python SDK
 Home-page: https://github.com/data-connector/dc-python-sdk
 Author: DataConnector

{dc_python_sdk-1.5.23 → dc_python_sdk-1.5.24}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dc-python-sdk"
-version = "1.5.23"
+version = "1.5.24"
 description = "Data Connector Python SDK"
 readme = "README.md"
 requires-python = ">=3.6"

{dc_python_sdk-1.5.23 → dc_python_sdk-1.5.24}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = dc-python-sdk
-version = 1.5.23
+version = 1.5.24
 author = DataConnector
 author_email = josh@dataconnector.com
 description = A small example package

{dc_python_sdk-1.5.23 → dc_python_sdk-1.5.24/src/dc_python_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dc-python-sdk
-Version: 1.5.23
+Version: 1.5.24
 Summary: Data Connector Python SDK
 Home-page: https://github.com/data-connector/dc-python-sdk
 Author: DataConnector

{dc_python_sdk-1.5.23 → dc_python_sdk-1.5.24}/src/dc_sdk/cli.py

@@ -1,5 +1,6 @@
 import sys
 from importlib.metadata import version
+import os
 
 def get_arg(name, default=None):
     if name in sys.argv:
@@ -15,15 +16,13 @@ def main():
 
     command = sys.argv[1]
 
-    port = int(get_arg("--port", 5000))
-
     print("version: ", version("dc-python-sdk"))
 
     print(f"[DC SDK] Command: {command}")
 
     if command == "http":
         from dc_sdk.src.server import start_server
-        start_server(port)
+        start_server()
 
     elif command == "ai":
         from dc_sdk.src.ai import start_ai
@@ -31,7 +30,7 @@ def main():
 
     elif command == "ai-http":
         from dc_sdk.src.ai_http import start_ai_http
-        start_ai_http(connector_port=port)
+        start_ai_http()
     
     else:
         print(f"Unknown command: {command}")

{dc_python_sdk-1.5.23 → dc_python_sdk-1.5.24}/src/dc_sdk/src/ai_http.py

@@ -20,15 +20,23 @@ app = FastAPI()
 
 _DEFAULT_PROCESS_TTL_SECONDS = 600.0
 
+# Default ports: connector HTTP (dc-sdk http) 5000, AI FastAPI 5001, code-server 5002.
+CONNECTOR_PORT = os.getenv("CONNECTOR_PORT", 5000)
+AI_PORT = os.getenv("AI_PORT", 5001)
+CODE_SERVER_PORT = os.getenv("CODE_SERVER_PORT", 5002)
+
 client = None
 workspace = os.getenv("WORKSPACE", "/workspace")
-port_for_connector = 5000
 
 # Process-wide TTL for ai-http (see DC_SDK_TTL_SECONDS); exposed in /session-info.
 process_ttl_seconds: Optional[float] = None
 process_ttl_deadline_unix: Optional[float] = None
 _process_ttl_timer: Optional[threading.Timer] = None
 
+# Long-lived `dc-sdk http` subprocess (uvicorn with CONNECTOR_HTTP_RELOAD reloads connector.py).
+_connector_proc: Optional[subprocess.Popen] = None
+_connector_lock = threading.Lock()
+
 # -----------------------------
 # MODELS
 # -----------------------------
@@ -91,40 +99,68 @@ def _parse_process_ttl_seconds() -> float:
         return 0.0
     return v
 
-def start_connector():
-    global workspace
-    global port_for_connector
+def _wait_connector_healthy() -> None:
+    for _ in range(50):
+        try:
+            res = requests.get(f"http://localhost:{CONNECTOR_PORT}/health", timeout=2)
+            if res.status_code == 200:
+                logger.info("Connector ready on port %s", CONNECTOR_PORT)
+                return
+        except Exception:
+            logger.debug("Health check not ready yet on port %s", CONNECTOR_PORT)
+        time.sleep(0.1)
+    raise RuntimeError(
+        f"Connector failed to become healthy on port {CONNECTOR_PORT} within timeout"
+    )
+
+
+def _start_connector_subprocess_unlocked() -> None:
+    global _connector_proc, workspace
+    env = os.environ.copy()
+    env["CONNECTOR_HTTP_RELOAD"] = "1"
     proc = subprocess.Popen(
-        ["dc-sdk", "http", "--port", str(port_for_connector)],
+        ["dc-sdk", "http"],
         cwd=workspace,
+        env=env,
         stdout=subprocess.DEVNULL,
-        stderr=subprocess.DEVNULL
+        stderr=subprocess.DEVNULL,
     )
-
-    # wait for health
-    for _ in range(50):
+    try:
+        _wait_connector_healthy()
+    except Exception:
+        proc.terminate()
         try:
-            res = requests.get(f"http://localhost:{port_for_connector}/health")
-            if res.status_code == 200:
-                logger.info("Connector ready on port %s", port_for_connector)
-                return {"process": proc, "port": port_for_connector}
+            proc.wait(timeout=5)
         except Exception:
-            logger.debug("Health check not ready yet on port %s", port_for_connector)
-        time.sleep(0.1)
+            pass
+        raise
+    _connector_proc = proc
 
-    logger.error("Connector failed to become healthy on port %s within timeout", port_for_connector)
-    proc.terminate()
-    raise Exception("Failed to start connector")
 
-def stop_connector(runtime):
+def _stop_connector_subprocess_unlocked() -> None:
+    global _connector_proc
+    proc = _connector_proc
+    _connector_proc = None
+    if proc is None:
+        return
     try:
-        runtime["process"].terminate()
-        runtime["process"].wait()
+        proc.terminate()
+        proc.wait(timeout=15)
         logger.debug("Connector subprocess stopped")
     except Exception:
         logger.debug("Error stopping connector subprocess", exc_info=True)
 
-def invoke(port, method, session_id=None, credentials=None, params=None):
+
+def start_connector_http_server() -> None:
+    """Start the connector HTTP child once; restart if the previous child exited."""
+    with _connector_lock:
+        if _connector_proc is not None and _connector_proc.poll() is None:
+            return
+        _stop_connector_subprocess_unlocked()
+        _start_connector_subprocess_unlocked()
+
+
+def invoke(method, session_id=None, credentials=None, params=None):
     payload = {
         "method": method,
         "session_id": session_id,
@@ -132,8 +168,8 @@ def invoke(port, method, session_id=None, credentials=None, params=None):
         "params": params or {}
     }
 
-    logger.debug("invoke port=%s method=%s session_id=%s", port, method, session_id)
-    res = requests.post(f"http://localhost:{port}/invoke", json=payload)
+    logger.debug("invoke port=%s method=%s session_id=%s", CONNECTOR_PORT, method, session_id)
+    res = requests.post(f"http://localhost:{CONNECTOR_PORT}/invoke", json=payload)
     if res.status_code != 200:
         logger.warning(
             "invoke returned HTTP %s for method=%s", res.status_code, method
@@ -208,166 +244,159 @@ def run_test(
     connector_context: ConnectorContext,
     test_object_ids: Optional[List[str]] = None,
 ):
-    runtime = start_connector()
     requested_ids = [_canon_object_id(x) for x in (test_object_ids or []) if _canon_object_id(x)]
     print(requested_ids)
 
-    try:
-        session_id = None
+    session_id = None
 
-        # authenticate
-        res = invoke(runtime["port"], "authenticate", None, credentials)
-        if res.get("results") is not True:
-            logger.warning("run_test failed at stage=authenticate")
-            return False, "authenticate", res
+    # authenticate
+    res = invoke("authenticate", None, credentials)
+    if res.get("results") is not True:
+        logger.warning("run_test failed at stage=authenticate")
+        return False, "authenticate", res
 
-        session_id = res.get("session_id")
+    session_id = res.get("session_id")
 
-        # get_objects
-        res = invoke(runtime["port"], "get_objects", session_id)
-        if not isinstance(res.get("results"), list):
-            logger.warning("run_test failed at stage=get_objects (invalid results type)")
-            return False, "get_objects", res
-
-        objects = res["results"]
-        api_ids = _object_ids_from_get_objects_results(objects)
-        dynamic = connector_context.objects_dynamic
-
-        if dynamic:
-            if api_ids:
-                if requested_ids:
-                    api_set = set(api_ids)
-                    wanted = [x for x in requested_ids if x in api_set]
-                    if not wanted:
-                        logger.warning(
-                            "run_test failed: test_object_ids disjoint from get_objects"
-                        )
-                        return False, "get_objects", {
-                            "message": (
-                                f"objects_dynamic is true: none of test_object_ids {requested_ids} "
-                                f"appear in get_objects results {api_ids}."
-                            ),
-                        }
-                    ids_for_fields = wanted
-                else:
-                    ids_for_fields = list(api_ids)
-            else:
-                if not requested_ids:
+    # get_objects
+    res = invoke("get_objects", session_id)
+    if not isinstance(res.get("results"), list):
+        logger.warning("run_test failed at stage=get_objects (invalid results type)")
+        return False, "get_objects", res
+
+    objects = res["results"]
+    api_ids = _object_ids_from_get_objects_results(objects)
+    dynamic = connector_context.objects_dynamic
+
+    if dynamic:
+        if api_ids:
+            if requested_ids:
+                api_set = set(api_ids)
+                wanted = [x for x in requested_ids if x in api_set]
+                if not wanted:
                     logger.warning(
-                        "run_test failed: objects_dynamic with empty get_objects and no test_object_ids"
+                        "run_test failed: test_object_ids disjoint from get_objects"
                     )
                     return False, "get_objects", {
                         "message": (
-                            "objects_dynamic is true: get_objects returned no objects. "
-                            "Provide test_object_ids in the request to probe get_fields and get_data."
+                            f"objects_dynamic is true: none of test_object_ids {requested_ids} "
+                            f"appear in get_objects results {api_ids}."
                         ),
                     }
-                ids_for_fields = list(requested_ids)
+                ids_for_fields = wanted
+            else:
+                ids_for_fields = list(api_ids)
         else:
-            print(requested_ids)
             if not requested_ids:
-                logger.warning("run_test failed: static objects require test_object_ids")
+                logger.warning(
+                    "run_test failed: objects_dynamic with empty get_objects and no test_object_ids"
+                )
                 return False, "get_objects", {
                     "message": (
-                        "objects_dynamic is false: test_object_ids is required and must list "
-                        "object ids you expect get_objects to return."
+                        "objects_dynamic is true: get_objects returned no objects. "
+                        "Provide test_object_ids in the request to probe get_fields and get_data."
                     ),
                 }
-            if not objects:
-                logger.warning("run_test failed at stage=get_objects (empty list)")
-                return False, "get_objects", {"message": "No objects returned from get_objects"}
-
             ids_for_fields = list(requested_ids)
-
-        if not ids_for_fields:
-            logger.warning("run_test failed: no object ids to probe")
+    else:
+        print(requested_ids)
+        if not requested_ids:
+            logger.warning("run_test failed: static objects require test_object_ids")
             return False, "get_objects", {
-                "message": "No object ids available to test get_fields (empty candidate list).",
-            }
-
-        objects_with_fields: List[Tuple[str, List[Any]]] = []
-        last_fields_payload: Any = None
-
-        print(ids_for_fields)
-
-        for object_id in ids_for_fields:
-            res = invoke(
-                runtime["port"],
-                "get_fields",
-                session_id,
-                params={"object_id": object_id},
-            )
-            if not isinstance(res.get("results"), list):
-                logger.warning(
-                    "run_test failed at stage=get_fields (invalid results type) object_id=%s",
-                    object_id,
-                )
-                return False, "get_fields", res
-
-            fields = res["results"]
-            last_fields_payload = res
-            logger.info("get_fields results for object_id=%s: %s", object_id, len(fields))
-            if fields:
-                fids = [
-                    f.get("field_id") or f.get("id")
-                    for f in fields[:5]
-                    if (f.get("field_id") or f.get("id")) is not None
-                ]
-                if fids:
-                    objects_with_fields.append((object_id, fids))
-
-        if not objects_with_fields:
-            logger.warning(
-                "run_test failed: no fields on any object tried ids=%s",
-                ids_for_fields,
-            )
-            return False, "get_fields", {
                 "message": (
-                    f"get_fields returned no usable fields for any tested object_id(s): {ids_for_fields}. "
-                    "Verify object ids and connector field discovery."
+                    "objects_dynamic is false: test_object_ids is required and must list "
+                    "object ids you expect get_objects to return."
                 ),
-                "last_invoke": last_fields_payload,
             }
+        if not objects:
+            logger.warning("run_test failed at stage=get_objects (empty list)")
+            return False, "get_objects", {"message": "No objects returned from get_objects"}
 
-        last_data_res: Any = None
-        for object_id, field_ids in objects_with_fields:
-            res = invoke(
-                runtime["port"],
-                "get_data",
-                session_id,
-                params={
-                    "object_id": object_id,
-                    "field_ids": field_ids,
-                    "n_rows": 5,
-                },
-            )
-            last_data_res = res
-            if not isinstance(res.get("results"), dict):
-                logger.warning(
-                    "run_test failed at stage=get_data (invalid results type) object_id=%s",
-                    object_id,
-                )
-                return False, "get_data", res
+        ids_for_fields = list(requested_ids)
+
+    if not ids_for_fields:
+        logger.warning("run_test failed: no object ids to probe")
+        return False, "get_objects", {
+            "message": "No object ids available to test get_fields (empty candidate list).",
+        }
+
+    objects_with_fields: List[Tuple[str, List[Any]]] = []
+    last_fields_payload: Any = None
 
-            payload = res["results"]
-            rows = payload.get("data")
-            if isinstance(rows, list) and len(rows) > 0:
-                logger.info("run_test completed successfully object_id=%s, rows retrieved=%s", object_id, len(rows))
-                return True, None, None
+    print(ids_for_fields)
 
+    for object_id in ids_for_fields:
+        res = invoke(
+            "get_fields",
+            session_id,
+            params={"object_id": object_id},
+        )
+        if not isinstance(res.get("results"), list):
+            logger.warning(
+                "run_test failed at stage=get_fields (invalid results type) object_id=%s",
+                object_id,
+            )
+            return False, "get_fields", res
+
+        fields = res["results"]
+        last_fields_payload = res
+        logger.info("get_fields results for object_id=%s: %s", object_id, len(fields))
+        if fields:
+            fids = [
+                f.get("field_id") or f.get("id")
+                for f in fields[:5]
+                if (f.get("field_id") or f.get("id")) is not None
+            ]
+            if fids:
+                objects_with_fields.append((object_id, fids))
+
+    if not objects_with_fields:
         logger.warning(
-            "run_test failed: get_data returned no rows for any object with fields"
+            "run_test failed: no fields on any object tried ids=%s",
+            ids_for_fields,
         )
-        return False, "get_data", {
+        return False, "get_fields", {
             "message": (
-                f"get_data returned no rows for any object that had fields "
-                f"(tried {len(objects_with_fields)} object(s))."
+                f"get_fields returned no usable fields for any tested object_id(s): {ids_for_fields}. "
+                "Verify object ids and connector field discovery."
             ),
-            "last_invoke": last_data_res,
+            "last_invoke": last_fields_payload,
         }
 
-    finally:
-        stop_connector(runtime)
+    last_data_res: Any = None
+    for object_id, field_ids in objects_with_fields:
+        res = invoke(
+            "get_data",
+            session_id,
+            params={
+                "object_id": object_id,
+                "field_ids": field_ids,
+                "n_rows": 5,
+            },
+        )
+        last_data_res = res
+        if not isinstance(res.get("results"), dict):
+            logger.warning(
+                "run_test failed at stage=get_data (invalid results type) object_id=%s",
+                object_id,
+            )
+            return False, "get_data", res
+
+        payload = res["results"]
+        rows = payload.get("data")
+        if isinstance(rows, list) and len(rows) > 0:
+            logger.info("run_test completed successfully object_id=%s, rows retrieved=%s", object_id, len(rows))
+            return True, None, None
+
+    logger.warning(
+        "run_test failed: get_data returned no rows for any object with fields"
+    )
+    return False, "get_data", {
+        "message": (
+            f"get_data returned no rows for any object that had fields "
+            f"(tried {len(objects_with_fields)} object(s))."
+        ),
+        "last_invoke": last_data_res,
+    }
 
 
 def _validate_static_requires_object_ids(
@@ -710,8 +739,7 @@ def clone_repo():
     subprocess.run(["git", "-C", workspace, "checkout", branch], check=True)
     subprocess.run(["git", "-C", workspace, "pull"], check=True)
 
-def run_code_server(port):
-    global port_for_connector
+def run_code_server():
     logger.info("code-server path: %s", shutil.which("code-server"))
 
     # ✅ Ensure config dir exists
@@ -720,7 +748,7 @@ def run_code_server(port):
     # ✅ Write code-server config
     with open("/root/.config/code-server/config.yaml", "w") as f:
         f.write(
-            f"bind-addr: 0.0.0.0:{port}\n"
+            f"bind-addr: 0.0.0.0:{CODE_SERVER_PORT}\n"
             "auth: none\n"
         )
 
@@ -755,15 +783,12 @@ def run_code_server(port):
 
     return code_server_proc
 
-def start_ai_http(connector_port=5000, start_code_server=True):
+def start_ai_http(start_code_server=True):
     global client
     global workspace
     global process_ttl_seconds
     global process_ttl_deadline_unix
     global _process_ttl_timer
-    global port_for_connector
-
-    port_for_connector = connector_port
 
     process_ttl_seconds = None
     process_ttl_deadline_unix = None
@@ -803,13 +828,13 @@ def start_ai_http(connector_port=5000, start_code_server=True):
         )
 
     api_key = os.getenv("OPENAI_API_KEY")
-    ai_port = int(os.getenv("AI_PORT", 5050))
-    connector_server_port = int(os.getenv("CONNECTOR_SERVER_PORT", 5001))
 
     clone_repo()
+    start_connector_http_server()
 
+    code_server_proc = None
     if start_code_server:
-        code_server_proc = run_code_server(connector_server_port)
+        code_server_proc = run_code_server()
 
     def shutdown():
         global _process_ttl_timer
@@ -818,8 +843,11 @@ def start_ai_http(connector_port=5000, start_code_server=True):
         if tm is not None:
             tm.cancel()
         logger.info("Shutting down code-server...")
-        if code_server_proc:
+        if code_server_proc is not None:
             code_server_proc.terminate()
+        logger.info("Shutting down connector HTTP...")
+        with _connector_lock:
+            _stop_connector_subprocess_unlocked()
 
     ttl_shutdown_holder["fn"] = shutdown
 
@@ -837,18 +865,19 @@ def start_ai_http(connector_port=5000, start_code_server=True):
 
     client = OpenAI(api_key=api_key)
 
-    logger.info("Starting AI HTTP controller on 0.0.0.0:%s", ai_port)
+    logger.info("Starting AI HTTP controller on 0.0.0.0:%s", AI_PORT)
 
     import uvicorn
-    uvicorn.run(app, host="0.0.0.0", port=ai_port)
+    uvicorn.run(app, host="0.0.0.0", port=AI_PORT)
 
 @app.get("/session-info")
 def session_info():
     out: Dict[str, Any] = {
         "workspace": workspace,
-        "ai_port": int(os.getenv("AI_PORT", 5050)),
-        "code_server_port": int(os.getenv("CODE_SERVER_PORT", 5002)),
-        "api_base": f"http://localhost:{os.getenv('AI_PORT', 5050)}",
+        "connector_port": CONNECTOR_PORT,
+        "ai_port": AI_PORT,
+        "code_server_port": CODE_SERVER_PORT,
+        "api_base": f"http://localhost:{AI_PORT}",
     }
     if process_ttl_seconds is not None and process_ttl_deadline_unix is not None:
         out["ttl_seconds"] = process_ttl_seconds

{dc_python_sdk-1.5.23 → dc_python_sdk-1.5.24}/src/dc_sdk/src/server.py

@@ -88,11 +88,21 @@ def invoke(req: InvokeRequest):
     }
 
 
-def start_server(port: Optional[int] = None):
+def start_server():
     import uvicorn
-    listen = int(os.getenv("PORT", 5000)) if port is None else int(port)
-    uvicorn.run(
-        "dc_sdk.src.server:app",
-        host="0.0.0.0",
-        port=listen,
-    )
+
+    listen = int(os.getenv("CONNECTOR_PORT", 5000))
+    workspace = os.getenv("WORKSPACE", os.getcwd())
+    connector_src = os.path.join(workspace, "src")
+    use_reload = os.getenv("CONNECTOR_HTTP_RELOAD", "0").lower() in ("1", "true", "yes")
+
+    kw: Dict[str, Any] = {
+        "host": "0.0.0.0",
+        "port": listen,
+    }
+    if use_reload:
+        kw["reload"] = True
+        watch = connector_src if os.path.isdir(connector_src) else workspace
+        kw["reload_dirs"] = [os.path.abspath(watch)]
+
+    uvicorn.run("dc_sdk.src.server:app", **kw)