nebu 0.1.91__py3-none-any.whl → 0.1.93__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nebu/cache.py +15 -11
- nebu/containers/container.py +13 -10
- nebu/data.py +112 -92
- nebu/logging.py +33 -0
- nebu/namespaces/namespace.py +7 -4
- nebu/processors/consumer.py +184 -152
- nebu/processors/consumer_process_worker.py +179 -96
- nebu/processors/decorate.py +226 -223
- nebu/processors/processor.py +38 -28
- {nebu-0.1.91.dist-info → nebu-0.1.93.dist-info}/METADATA +2 -1
- nebu-0.1.93.dist-info/RECORD +28 -0
- nebu/containers/decorator.py +0 -93
- nebu/containers/server.py +0 -70
- nebu/processors/remote.py +0 -47
- nebu-0.1.91.dist-info/RECORD +0 -30
- {nebu-0.1.91.dist-info → nebu-0.1.93.dist-info}/WHEEL +0 -0
- {nebu-0.1.91.dist-info → nebu-0.1.93.dist-info}/licenses/LICENSE +0 -0
- {nebu-0.1.91.dist-info → nebu-0.1.93.dist-info}/top_level.txt +0 -0
nebu/processors/decorate.py
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
import ast # For parsing notebook code
|
2
2
|
import inspect
|
3
3
|
import os
|
4
|
-
import re
|
4
|
+
import re
|
5
5
|
import textwrap
|
6
6
|
from typing import (
|
7
7
|
Any,
|
@@ -36,6 +36,7 @@ from nebu.containers.models import (
|
|
36
36
|
V1VolumePath,
|
37
37
|
)
|
38
38
|
from nebu.data import Bucket
|
39
|
+
from nebu.logging import logger
|
39
40
|
from nebu.meta import V1ResourceMetaRequest
|
40
41
|
from nebu.processors.models import (
|
41
42
|
Message,
|
@@ -78,19 +79,19 @@ def is_jupyter_notebook():
|
|
78
79
|
|
79
80
|
ip = get_ipython() # Use the imported function
|
80
81
|
if ip is None: # type: ignore
|
81
|
-
#
|
82
|
+
# logger.debug("is_jupyter_notebook: No IPython instance found.")
|
82
83
|
return False
|
83
84
|
class_name = str(ip.__class__)
|
84
|
-
#
|
85
|
+
# logger.debug(f"is_jupyter_notebook: IPython class name: {class_name}")
|
85
86
|
if "ZMQInteractiveShell" in class_name:
|
86
|
-
#
|
87
|
+
# logger.debug("is_jupyter_notebook: Jupyter detected (ZMQInteractiveShell).")
|
87
88
|
return True
|
88
|
-
#
|
89
|
+
# logger.debug("is_jupyter_notebook: Not Jupyter (IPython instance found, but not ZMQInteractiveShell).")
|
89
90
|
return False
|
90
91
|
except Exception as e:
|
91
|
-
|
92
|
-
f"
|
93
|
-
) #
|
92
|
+
logger.debug(
|
93
|
+
f"is_jupyter_notebook: Exception occurred: {e}"
|
94
|
+
) # Keep as debug for less noise
|
94
95
|
return False
|
95
96
|
|
96
97
|
|
@@ -99,35 +100,35 @@ def get_notebook_executed_code():
|
|
99
100
|
Returns all executed code from the current notebook session.
|
100
101
|
Returns str or None: All executed code as a string, or None if not possible.
|
101
102
|
"""
|
102
|
-
|
103
|
+
logger.debug("Attempting to get notebook execution history...")
|
103
104
|
try:
|
104
105
|
# Fix: Import get_ipython directly
|
105
106
|
from IPython.core.getipython import get_ipython
|
106
107
|
|
107
108
|
ip = get_ipython() # Use the imported function
|
108
109
|
if ip is None or not hasattr(ip, "history_manager"):
|
109
|
-
|
110
|
-
"
|
110
|
+
logger.debug(
|
111
|
+
"get_notebook_executed_code: No IPython instance or history_manager."
|
111
112
|
)
|
112
113
|
return None
|
113
114
|
history_manager = ip.history_manager
|
114
115
|
# Limiting history range for debugging? Maybe get_tail(N)? For now, get all.
|
115
116
|
# history = history_manager.get_range(start=1) # type: ignore
|
116
117
|
history = list(history_manager.get_range(start=1)) # type: ignore # Convert to list to get length
|
117
|
-
|
118
|
-
f"
|
118
|
+
logger.debug(
|
119
|
+
f"get_notebook_executed_code: Retrieved {len(history)} history entries."
|
119
120
|
)
|
120
121
|
source_code = ""
|
121
122
|
separator = "\n#<NEBU_CELL_SEP>#\n"
|
122
123
|
for _, _, content in history: # Use _ for unused session, lineno
|
123
124
|
if isinstance(content, str) and content.strip():
|
124
125
|
source_code += content + separator
|
125
|
-
|
126
|
-
f"
|
126
|
+
logger.debug(
|
127
|
+
f"get_notebook_executed_code: Total history source length: {len(source_code)}"
|
127
128
|
)
|
128
129
|
return source_code
|
129
130
|
except Exception as e:
|
130
|
-
|
131
|
+
logger.error(f"get_notebook_executed_code: Error getting history: {e}")
|
131
132
|
return None
|
132
133
|
|
133
134
|
|
@@ -140,15 +141,15 @@ def extract_definition_source_from_string(
|
|
140
141
|
Uses AST parsing for robustness.
|
141
142
|
def_type can be ast.FunctionDef or ast.ClassDef.
|
142
143
|
"""
|
143
|
-
|
144
|
-
f"
|
144
|
+
logger.debug(
|
145
|
+
f"Extracting '{def_name}' ({def_type.__name__}) from history string (len: {len(source_string)})..."
|
145
146
|
)
|
146
147
|
if not source_string or not def_name:
|
147
|
-
|
148
|
+
logger.debug("extract: Empty source string or def_name.")
|
148
149
|
return None
|
149
150
|
|
150
151
|
cells = source_string.split("#<NEBU_CELL_SEP>#")
|
151
|
-
|
152
|
+
logger.debug(f"extract: Split history into {len(cells)} potential cells.")
|
152
153
|
last_found_source = None
|
153
154
|
|
154
155
|
for i, cell in enumerate(reversed(cells)):
|
@@ -156,7 +157,7 @@ def extract_definition_source_from_string(
|
|
156
157
|
cell = cell.strip()
|
157
158
|
if not cell:
|
158
159
|
continue
|
159
|
-
#
|
160
|
+
# logger.debug(f"extract: Analyzing cell #{cell_num}...") # Can be very verbose
|
160
161
|
try:
|
161
162
|
tree = ast.parse(cell)
|
162
163
|
found_in_cell = False
|
@@ -167,22 +168,22 @@ def extract_definition_source_from_string(
|
|
167
168
|
) # Check if it's the right type (FuncDef or ClassDef)
|
168
169
|
and getattr(node, "name", None) == def_name # Safely check name
|
169
170
|
):
|
170
|
-
|
171
|
-
f"
|
171
|
+
logger.debug(
|
172
|
+
f"extract: Found node for '{def_name}' in cell #{cell_num}."
|
172
173
|
)
|
173
174
|
try:
|
174
175
|
# Use ast.get_source_segment for accurate extraction (Python 3.8+)
|
175
176
|
func_source = ast.get_source_segment(cell, node)
|
176
177
|
if func_source:
|
177
|
-
|
178
|
-
f"
|
178
|
+
logger.debug(
|
179
|
+
f"extract: Successfully extracted source using get_source_segment for '{def_name}'."
|
179
180
|
)
|
180
181
|
last_found_source = func_source
|
181
182
|
found_in_cell = True
|
182
183
|
break # Stop searching this cell
|
183
184
|
except AttributeError: # Fallback for Python < 3.8
|
184
|
-
|
185
|
-
f"
|
185
|
+
logger.debug(
|
186
|
+
f"extract: get_source_segment failed (likely Py < 3.8), using fallback for '{def_name}'."
|
186
187
|
)
|
187
188
|
start_lineno = getattr(node, "lineno", 1) - 1
|
188
189
|
end_lineno = getattr(node, "end_lineno", start_lineno + 1)
|
@@ -210,34 +211,34 @@ def extract_definition_source_from_string(
|
|
210
211
|
.startswith(("def ", "class "))
|
211
212
|
):
|
212
213
|
last_found_source = "\n".join(extracted_lines)
|
213
|
-
|
214
|
-
f"
|
214
|
+
logger.debug(
|
215
|
+
f"extract: Extracted source via fallback for '{def_name}'."
|
215
216
|
)
|
216
217
|
found_in_cell = True
|
217
218
|
break
|
218
219
|
else:
|
219
|
-
|
220
|
-
f"
|
220
|
+
logger.warning(
|
221
|
+
f"extract: Line numbers out of bounds for {def_name} in cell (fallback)."
|
221
222
|
)
|
222
223
|
|
223
224
|
if found_in_cell:
|
224
|
-
|
225
|
-
f"
|
225
|
+
logger.debug(
|
226
|
+
f"extract: Found and returning source for '{def_name}' from cell #{cell_num}."
|
226
227
|
)
|
227
228
|
return last_found_source # Found last definition, return immediately
|
228
229
|
|
229
230
|
except (SyntaxError, ValueError) as e:
|
230
|
-
#
|
231
|
+
# logger.debug(f"extract: Skipping cell #{cell_num} due to parse error: {e}") # Can be verbose
|
231
232
|
continue
|
232
233
|
except Exception as e:
|
233
|
-
|
234
|
-
f"
|
234
|
+
logger.warning(
|
235
|
+
f"extract: AST processing error for {def_name} in cell #{cell_num}: {e}"
|
235
236
|
)
|
236
237
|
continue
|
237
238
|
|
238
239
|
if not last_found_source:
|
239
|
-
|
240
|
-
f"
|
240
|
+
logger.debug(
|
241
|
+
f"extract: Definition '{def_name}' of type {def_type.__name__} not found in history search."
|
241
242
|
)
|
242
243
|
return last_found_source
|
243
244
|
|
@@ -257,13 +258,13 @@ def include(obj: Any) -> Any:
|
|
257
258
|
source = dill.source.getsource(obj)
|
258
259
|
dedented_source = textwrap.dedent(source)
|
259
260
|
setattr(obj, _NEBU_EXPLICIT_SOURCE_ATTR, dedented_source)
|
260
|
-
|
261
|
-
f"
|
261
|
+
logger.debug(
|
262
|
+
f"@include: Successfully captured source for: {getattr(obj, '__name__', str(obj))}"
|
262
263
|
)
|
263
264
|
except Exception as e:
|
264
265
|
# Don't fail the definition, just warn
|
265
|
-
|
266
|
-
f"
|
266
|
+
logger.warning(
|
267
|
+
f"@include could not capture source for {getattr(obj, '__name__', str(obj))}: {e}. Automatic source retrieval will be attempted later."
|
267
268
|
)
|
268
269
|
return obj
|
269
270
|
|
@@ -276,44 +277,44 @@ def get_model_source(
|
|
276
277
|
Checks explicit source, then notebook history (if provided), then dill.
|
277
278
|
"""
|
278
279
|
model_name_str = getattr(model_class, "__name__", str(model_class))
|
279
|
-
|
280
|
+
logger.debug(f"get_model_source: Getting source for: {model_name_str}")
|
280
281
|
# 1. Check explicit source
|
281
282
|
explicit_source = getattr(model_class, _NEBU_EXPLICIT_SOURCE_ATTR, None)
|
282
283
|
if explicit_source:
|
283
|
-
|
284
|
-
f"
|
284
|
+
logger.debug(
|
285
|
+
f"get_model_source: Using explicit source (@include) for: {model_name_str}"
|
285
286
|
)
|
286
287
|
return explicit_source
|
287
288
|
|
288
289
|
# 2. Check notebook history
|
289
290
|
if notebook_code and hasattr(model_class, "__name__"):
|
290
|
-
|
291
|
-
f"
|
291
|
+
logger.debug(
|
292
|
+
f"get_model_source: Attempting notebook history extraction for: {model_class.__name__}"
|
292
293
|
)
|
293
294
|
extracted_source = extract_definition_source_from_string(
|
294
295
|
notebook_code, model_class.__name__, ast.ClassDef
|
295
296
|
)
|
296
297
|
if extracted_source:
|
297
|
-
|
298
|
-
f"
|
298
|
+
logger.debug(
|
299
|
+
f"get_model_source: Using notebook history source for: {model_class.__name__}"
|
299
300
|
)
|
300
301
|
return extracted_source
|
301
302
|
else:
|
302
|
-
|
303
|
-
f"
|
303
|
+
logger.debug(
|
304
|
+
f"get_model_source: Notebook history extraction failed for: {model_class.__name__}. Proceeding to dill."
|
304
305
|
)
|
305
306
|
|
306
307
|
# 3. Fallback to dill
|
307
308
|
try:
|
308
|
-
|
309
|
-
f"
|
309
|
+
logger.debug(
|
310
|
+
f"get_model_source: Attempting dill fallback for: {model_name_str}"
|
310
311
|
)
|
311
312
|
source = dill.source.getsource(model_class)
|
312
|
-
|
313
|
+
logger.debug(f"get_model_source: Using dill source for: {model_name_str}")
|
313
314
|
return textwrap.dedent(source)
|
314
315
|
except (IOError, TypeError, OSError) as e:
|
315
|
-
|
316
|
-
f"
|
316
|
+
logger.debug(
|
317
|
+
f"get_model_source: Failed dill fallback for: {model_name_str}: {e}"
|
317
318
|
)
|
318
319
|
return None
|
319
320
|
|
@@ -324,22 +325,22 @@ def get_type_source(
|
|
324
325
|
) -> Optional[Any]:
|
325
326
|
"""Get the source code for a type, including generic parameters."""
|
326
327
|
type_obj_str = str(type_obj)
|
327
|
-
|
328
|
+
logger.debug(f"get_type_source: Getting source for type: {type_obj_str}")
|
328
329
|
origin = get_origin(type_obj)
|
329
330
|
args = get_args(type_obj)
|
330
331
|
|
331
332
|
if origin is not None:
|
332
333
|
# Use updated get_model_source for origin
|
333
|
-
|
334
|
-
f"
|
334
|
+
logger.debug(
|
335
|
+
f"get_type_source: Detected generic type. Origin: {origin}, Args: {args}"
|
335
336
|
)
|
336
337
|
origin_source = get_model_source(origin, notebook_code)
|
337
338
|
args_sources = []
|
338
339
|
|
339
340
|
# Recursively get sources for all type arguments
|
340
341
|
for arg in args:
|
341
|
-
|
342
|
-
f"
|
342
|
+
logger.debug(
|
343
|
+
f"get_type_source: Recursively getting source for generic arg #{arg}"
|
343
344
|
)
|
344
345
|
arg_source = get_type_source(arg, notebook_code)
|
345
346
|
if arg_source:
|
@@ -347,8 +348,8 @@ def get_type_source(
|
|
347
348
|
|
348
349
|
# Return tuple only if origin source or some arg sources were found
|
349
350
|
if origin_source or args_sources:
|
350
|
-
|
351
|
-
f"
|
351
|
+
logger.debug(
|
352
|
+
f"get_type_source: Returning tuple source for generic: {type_obj_str}"
|
352
353
|
)
|
353
354
|
return (origin_source, args_sources)
|
354
355
|
|
@@ -356,12 +357,12 @@ def get_type_source(
|
|
356
357
|
# Try get_model_source as a last resort for unknown types
|
357
358
|
fallback_source = get_model_source(type_obj, notebook_code)
|
358
359
|
if fallback_source:
|
359
|
-
|
360
|
-
f"
|
360
|
+
logger.debug(
|
361
|
+
f"get_type_source: Using fallback get_model_source for: {type_obj_str}"
|
361
362
|
)
|
362
363
|
return fallback_source
|
363
364
|
|
364
|
-
|
365
|
+
logger.debug(f"get_type_source: Failed to get source for: {type_obj_str}")
|
365
366
|
return None
|
366
367
|
|
367
368
|
|
@@ -393,20 +394,21 @@ def processor(
|
|
393
394
|
execution_mode: str = "inline",
|
394
395
|
config: Optional[GlobalConfig] = None,
|
395
396
|
hot_reload: bool = True,
|
397
|
+
debug: bool = False,
|
396
398
|
):
|
397
399
|
def decorator(
|
398
400
|
func: Callable[[Any], Any],
|
399
401
|
) -> Processor:
|
400
402
|
# --- Prevent Recursion Guard ---
|
401
403
|
if os.environ.get(_NEBU_INSIDE_CONSUMER_ENV_VAR) == "1":
|
402
|
-
|
403
|
-
f"
|
404
|
+
logger.debug(
|
405
|
+
f"Decorator Guard triggered for '{func.__name__}'. Returning original function."
|
404
406
|
)
|
405
407
|
return func # type: ignore
|
406
408
|
# --- End Guard ---
|
407
409
|
|
408
|
-
|
409
|
-
f"
|
410
|
+
logger.debug(
|
411
|
+
f"Decorator Init: @processor decorating function '{func.__name__}'"
|
410
412
|
)
|
411
413
|
all_env = env or []
|
412
414
|
processor_name = func.__name__
|
@@ -416,7 +418,7 @@ def processor(
|
|
416
418
|
effective_config = config
|
417
419
|
|
418
420
|
# --- Get Decorated Function File Path and Directory ---
|
419
|
-
|
421
|
+
logger.debug("Decorator: Getting source file path for decorated function...")
|
420
422
|
func_file_path: Optional[str] = None
|
421
423
|
func_dir: Optional[str] = None
|
422
424
|
rel_func_path: Optional[str] = None # Relative path within func_dir
|
@@ -427,9 +429,9 @@ def processor(
|
|
427
429
|
func_dir = os.path.dirname(func_file_path)
|
428
430
|
# Calculate relative path based on the resolved directory
|
429
431
|
rel_func_path = os.path.relpath(func_file_path, func_dir)
|
430
|
-
|
431
|
-
|
432
|
-
|
432
|
+
logger.debug(f"Decorator: Found real file path: {func_file_path}")
|
433
|
+
logger.debug(f"Decorator: Found function directory: {func_dir}")
|
434
|
+
logger.debug(f"Decorator: Relative function path: {rel_func_path}")
|
433
435
|
except (TypeError, OSError) as e:
|
434
436
|
# TypeError can happen if func is not a module, class, method, function, traceback, frame, or code object
|
435
437
|
raise ValueError(
|
@@ -448,7 +450,7 @@ def processor(
|
|
448
450
|
"Could not determine function directory or relative path for S3 upload."
|
449
451
|
)
|
450
452
|
# --- Get API Key ---
|
451
|
-
|
453
|
+
logger.debug("Decorator: Loading Nebu configuration...")
|
452
454
|
try:
|
453
455
|
if not effective_config:
|
454
456
|
effective_config = GlobalConfig.read()
|
@@ -456,7 +458,7 @@ def processor(
|
|
456
458
|
if not current_server or not current_server.api_key:
|
457
459
|
raise ValueError("Nebu server configuration or API key not found.")
|
458
460
|
api_key = current_server.api_key
|
459
|
-
|
461
|
+
logger.debug("Decorator: Nebu API key loaded successfully.")
|
460
462
|
|
461
463
|
# # Add additional environment variables from current configuration
|
462
464
|
# all_env.append(V1EnvVar(key="AGENTSEA_API_KEY", value=api_key))
|
@@ -481,10 +483,10 @@ def processor(
|
|
481
483
|
# if orign_server:
|
482
484
|
# all_env.append(V1EnvVar(key="ORIGN_SERVER", value=orign_server))
|
483
485
|
# else:
|
484
|
-
#
|
486
|
+
# logger.debug("Decorator: No Orign server found. Not setting...")
|
485
487
|
|
486
488
|
except Exception as e:
|
487
|
-
|
489
|
+
logger.error(f"Failed to load Nebu configuration or API key: {e}")
|
488
490
|
raise RuntimeError(
|
489
491
|
f"Failed to load Nebu configuration or API key: {e}"
|
490
492
|
) from e
|
@@ -493,19 +495,19 @@ def processor(
|
|
493
495
|
# --- Determine Namespace ---
|
494
496
|
effective_namespace = namespace # Start with the provided namespace
|
495
497
|
if effective_namespace is None:
|
496
|
-
|
498
|
+
logger.debug("Decorator: Namespace not provided, fetching user profile...")
|
497
499
|
try:
|
498
500
|
user_profile = get_user_profile(api_key)
|
499
501
|
if user_profile.handle:
|
500
502
|
effective_namespace = user_profile.handle
|
501
|
-
|
502
|
-
f"
|
503
|
+
logger.debug(
|
504
|
+
f"Decorator: Using user handle '{effective_namespace}' as namespace."
|
503
505
|
)
|
504
506
|
else:
|
505
507
|
raise ValueError("User profile does not contain a handle.")
|
506
508
|
except Exception as e:
|
507
|
-
|
508
|
-
f"
|
509
|
+
logger.error(
|
510
|
+
f"Failed to get user profile or handle for default namespace: {e}"
|
509
511
|
)
|
510
512
|
raise RuntimeError(
|
511
513
|
f"Failed to get user profile or handle for default namespace: {e}"
|
@@ -514,7 +516,7 @@ def processor(
|
|
514
516
|
|
515
517
|
# Use processor_name instead of name
|
516
518
|
S3_TOKEN_ENDPOINT = f"{NEBU_API_BASE_URL}/v1/auth/temp-s3-tokens/{effective_namespace}/{processor_name}"
|
517
|
-
|
519
|
+
logger.debug(f"Decorator: Fetching S3 token from: {S3_TOKEN_ENDPOINT}")
|
518
520
|
try:
|
519
521
|
headers = {"Authorization": f"Bearer {api_key}"} # Add headers here
|
520
522
|
|
@@ -523,7 +525,7 @@ def processor(
|
|
523
525
|
response.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx)
|
524
526
|
s3_token_data = response.json()
|
525
527
|
|
526
|
-
|
528
|
+
logger.debug(f"Decorator: S3 token data: {s3_token_data}")
|
527
529
|
|
528
530
|
aws_access_key_id = s3_token_data.get("access_key_id")
|
529
531
|
aws_secret_access_key = s3_token_data.get("secret_access_key")
|
@@ -559,8 +561,8 @@ def processor(
|
|
559
561
|
) # Ensure trailing slash for prefix
|
560
562
|
s3_destination_uri = f"s3://{parsed_base.netloc}/{s3_destination_key}"
|
561
563
|
|
562
|
-
|
563
|
-
f"
|
564
|
+
logger.debug(
|
565
|
+
f"Decorator: Uploading code from '{func_dir}' to '{s3_destination_uri}'"
|
564
566
|
)
|
565
567
|
|
566
568
|
# Instantiate Bucket with temporary credentials
|
@@ -584,48 +586,46 @@ def processor(
|
|
584
586
|
delete=True,
|
585
587
|
dry_run=False,
|
586
588
|
)
|
587
|
-
|
589
|
+
logger.debug("Decorator: S3 code upload completed.")
|
588
590
|
|
589
591
|
except requests.exceptions.RequestException as e:
|
590
|
-
|
592
|
+
logger.error(f"Failed to fetch S3 token from {S3_TOKEN_ENDPOINT}: {e}")
|
591
593
|
raise RuntimeError(
|
592
594
|
f"Failed to fetch S3 token from {S3_TOKEN_ENDPOINT}: {e}"
|
593
595
|
) from e
|
594
596
|
except ClientError as e:
|
595
|
-
|
597
|
+
logger.error(f"Failed to upload code to S3 {s3_destination_uri}: {e}")
|
596
598
|
# Attempt to provide more context from the error if possible
|
597
599
|
error_code = e.response.get("Error", {}).get("Code")
|
598
600
|
error_msg = e.response.get("Error", {}).get("Message")
|
599
|
-
|
601
|
+
logger.error(f" S3 Error Code: {error_code}, Message: {error_msg}")
|
600
602
|
raise RuntimeError(
|
601
603
|
f"Failed to upload code to {s3_destination_uri}: {e}"
|
602
604
|
) from e
|
603
605
|
except ValueError as e: # Catch ValueErrors from validation
|
604
|
-
|
606
|
+
logger.error(f"Configuration or response data error: {e}")
|
605
607
|
raise RuntimeError(f"Configuration or response data error: {e}") from e
|
606
608
|
except Exception as e:
|
607
|
-
|
609
|
+
logger.exception(f"Unexpected error during S3 token fetch or upload: {e}")
|
608
610
|
# Consider logging traceback here for better debugging
|
609
611
|
import traceback
|
610
612
|
|
611
|
-
traceback.print_exc()
|
613
|
+
traceback.print_exc() # Keep this explicit traceback for now in case logging isn't configured yet
|
612
614
|
raise RuntimeError(f"Unexpected error during S3 setup: {e}") from e
|
613
615
|
|
614
616
|
# --- Process Manually Included Objects (Keep for now, add source via env) ---
|
615
|
-
# This part remains unchanged for now, using @include and environment variables.
|
616
|
-
# Future: Could potentially upload these to S3 as well if they become large.
|
617
617
|
included_sources: Dict[Any, Any] = {}
|
618
618
|
notebook_code_for_include = None # Get notebook code only if needed for include
|
619
619
|
if include:
|
620
620
|
# Determine if we are in Jupyter only if needed for include fallback
|
621
|
-
#
|
621
|
+
# logger.debug("Decorator: Processing manually included objects...")
|
622
622
|
is_jupyter_env = is_jupyter_notebook()
|
623
623
|
if is_jupyter_env:
|
624
624
|
notebook_code_for_include = get_notebook_executed_code()
|
625
625
|
|
626
626
|
for i, obj in enumerate(include):
|
627
627
|
obj_name_str = getattr(obj, "__name__", str(obj))
|
628
|
-
#
|
628
|
+
# logger.debug(f"Decorator: Getting source for manually included object: {obj_name_str}")
|
629
629
|
# Pass notebook code only if available and needed by get_model_source
|
630
630
|
obj_source = get_model_source(
|
631
631
|
obj, notebook_code_for_include if is_jupyter_env else None
|
@@ -634,27 +634,29 @@ def processor(
|
|
634
634
|
included_sources[obj] = obj_source
|
635
635
|
# Decide how to pass included source - keep using Env Vars for now
|
636
636
|
env_key_base = f"INCLUDED_OBJECT_{i}"
|
637
|
-
|
637
|
+
# Correct check for string type - Linter might complain but it's safer
|
638
|
+
if isinstance(obj_source, str): # type: ignore
|
638
639
|
all_env.append(
|
639
640
|
V1EnvVar(key=f"{env_key_base}_SOURCE", value=obj_source)
|
640
641
|
)
|
641
|
-
#
|
642
|
+
# logger.debug(f"Decorator: Added string source to env for included obj: {obj_name_str}")
|
642
643
|
elif isinstance(obj_source, tuple):
|
643
644
|
# Handle tuple source (origin, args) - assumes get_model_source/get_type_source logic
|
644
645
|
# Ensure obj_source is indeed a tuple before unpacking
|
645
646
|
if len(obj_source) == 2:
|
646
647
|
# Now safe to unpack
|
647
|
-
origin_src, arg_srcs = obj_source # type: ignore
|
648
|
-
# type: ignore[misc] # Suppress persistent tuple unpacking error
|
648
|
+
origin_src, arg_srcs = obj_source # type: ignore
|
649
649
|
if origin_src and isinstance(origin_src, str):
|
650
650
|
all_env.append(
|
651
651
|
V1EnvVar(
|
652
652
|
key=f"{env_key_base}_SOURCE", value=origin_src
|
653
653
|
)
|
654
654
|
)
|
655
|
-
# Handle arg_srcs
|
655
|
+
# Handle arg_srcs - ensure it's iterable (list)
|
656
|
+
# Linter complains about "Never" not iterable, check type explicitly
|
656
657
|
if isinstance(arg_srcs, list):
|
657
658
|
for j, arg_src in enumerate(arg_srcs):
|
659
|
+
# Ensure arg_src is string before adding
|
658
660
|
if isinstance(arg_src, str):
|
659
661
|
all_env.append(
|
660
662
|
V1EnvVar(
|
@@ -663,29 +665,27 @@ def processor(
|
|
663
665
|
)
|
664
666
|
)
|
665
667
|
else:
|
666
|
-
|
667
|
-
f"
|
668
|
+
logger.warning(
|
669
|
+
f"Decorator: Expected arg_srcs to be a list, got {type(arg_srcs)}"
|
668
670
|
)
|
669
671
|
else:
|
670
672
|
# Handle unexpected type or structure for obj_source if necessary
|
671
|
-
|
672
|
-
|
673
|
-
print(
|
674
|
-
f"[DEBUG Decorator] Warning: Unexpected obj_source structure: {obj_source}"
|
673
|
+
logger.warning(
|
674
|
+
f"Decorator: Unexpected obj_source structure: {obj_source}"
|
675
675
|
)
|
676
676
|
else:
|
677
|
-
|
678
|
-
f"
|
677
|
+
logger.warning(
|
678
|
+
f"Unknown source type for included object {obj_name_str}: {type(obj_source)}"
|
679
679
|
)
|
680
680
|
else:
|
681
|
-
|
682
|
-
f"
|
681
|
+
logger.warning(
|
682
|
+
f"Could not retrieve source for manually included object: {obj_name_str}. It might not be available in the consumer."
|
683
683
|
)
|
684
684
|
# --- End Manually Included Objects ---
|
685
685
|
|
686
686
|
# --- Validate Function Signature and Types (Keep as is) ---
|
687
|
-
|
688
|
-
f"
|
687
|
+
logger.debug(
|
688
|
+
f"Decorator: Validating signature and type hints for {processor_name}..."
|
689
689
|
)
|
690
690
|
sig = inspect.signature(func)
|
691
691
|
params = list(sig.parameters.values())
|
@@ -697,17 +697,17 @@ def processor(
|
|
697
697
|
try:
|
698
698
|
# Attempt to resolve type hints
|
699
699
|
type_hints = get_type_hints(func, globalns=func.__globals__, localns=None)
|
700
|
-
|
700
|
+
logger.debug(f"Decorator: Resolved type hints: {type_hints}")
|
701
701
|
except NameError as e:
|
702
702
|
# Specific handling for NameError (common in notebooks/dynamic environments)
|
703
|
-
|
704
|
-
f"
|
703
|
+
logger.warning(
|
704
|
+
f"Could not fully resolve type hints for {processor_name} due to NameError: {e}. Type validation might be incomplete."
|
705
705
|
)
|
706
706
|
# Try to get raw annotations as fallback?
|
707
707
|
type_hints = getattr(func, "__annotations__", {})
|
708
|
-
|
708
|
+
logger.debug(f"Decorator: Using raw annotations as fallback: {type_hints}")
|
709
709
|
except Exception as e:
|
710
|
-
|
710
|
+
logger.error(f"Decorator: Error getting type hints: {e}")
|
711
711
|
# Potentially re-raise or handle based on severity
|
712
712
|
raise TypeError(
|
713
713
|
f"Could not evaluate type hints for {processor_name}: {e}. Ensure all type dependencies are defined or imported."
|
@@ -723,22 +723,22 @@ def processor(
|
|
723
723
|
param_name
|
724
724
|
) # Use .get for safety with raw annotations fallback
|
725
725
|
param_type_str_repr = str(param_type) # Use string representation
|
726
|
-
|
727
|
-
f"
|
726
|
+
logger.debug(
|
727
|
+
f"Decorator: Parameter '{param_name}' type hint: {param_type_str_repr}"
|
728
728
|
)
|
729
729
|
|
730
730
|
return_type = type_hints.get("return")
|
731
731
|
return_type_str_repr = str(return_type)
|
732
|
-
|
732
|
+
logger.debug(f"Decorator: Return type hint: {return_type_str_repr}")
|
733
733
|
|
734
734
|
# --- Determine Input Type (StreamMessage, ContentType) ---
|
735
735
|
# This logic remains mostly the same, using the resolved types
|
736
|
-
|
737
|
-
f"
|
736
|
+
logger.debug(
|
737
|
+
f"Decorator: Determining input type structure for param type hint: {param_type_str_repr}"
|
738
738
|
)
|
739
739
|
origin = get_origin(param_type) if param_type else None
|
740
740
|
args = get_args(param_type) if param_type else tuple()
|
741
|
-
|
741
|
+
logger.debug(f"Decorator: get_origin result: {origin}, get_args result: {args}")
|
742
742
|
is_stream_message = False
|
743
743
|
content_type = None
|
744
744
|
content_type_name_from_regex = None # Store regex result here
|
@@ -750,96 +750,96 @@ def processor(
|
|
750
750
|
if origin is message_cls or (
|
751
751
|
isinstance(origin, type) and origin is message_cls
|
752
752
|
):
|
753
|
-
|
754
|
-
"
|
753
|
+
logger.debug(
|
754
|
+
"Decorator: Input type identified as Message via get_origin/isinstance."
|
755
755
|
)
|
756
756
|
is_stream_message = True
|
757
757
|
if args:
|
758
758
|
content_type = args[0]
|
759
|
-
|
760
|
-
f"
|
759
|
+
logger.debug(
|
760
|
+
f"Decorator: Content type extracted via get_args: {content_type}"
|
761
761
|
)
|
762
762
|
else:
|
763
|
-
|
764
|
-
"
|
763
|
+
logger.debug(
|
764
|
+
"Decorator: Message detected, but no generic arguments found via get_args. Attempting regex fallback on string repr."
|
765
765
|
)
|
766
766
|
# --- Regex Fallback Start ---
|
767
767
|
match = re.search(r"Message\[([\w\.]+)\]", param_type_str_repr)
|
768
768
|
if match:
|
769
769
|
content_type_name_from_regex = match.group(1)
|
770
|
-
|
771
|
-
f"
|
770
|
+
logger.debug(
|
771
|
+
f"Decorator: Extracted content type name via regex: {content_type_name_from_regex}"
|
772
772
|
)
|
773
773
|
else:
|
774
|
-
|
775
|
-
"
|
774
|
+
logger.debug(
|
775
|
+
"Decorator: Regex fallback failed to extract content type name."
|
776
776
|
)
|
777
777
|
# --- Regex Fallback End ---
|
778
778
|
# Check 2a: Regex fallback if get_origin failed but string matches pattern
|
779
779
|
elif origin is None and param_type is not None:
|
780
|
-
|
781
|
-
"
|
780
|
+
logger.debug(
|
781
|
+
"Decorator: get_origin failed. Attempting regex fallback on string representation."
|
782
782
|
)
|
783
783
|
match = re.search(r"Message\[([\w\.]+)\]", param_type_str_repr)
|
784
784
|
if match:
|
785
|
-
|
786
|
-
"
|
785
|
+
logger.debug(
|
786
|
+
"Decorator: Regex fallback successful after get_origin failed."
|
787
787
|
)
|
788
788
|
is_stream_message = True
|
789
789
|
content_type_name_from_regex = match.group(1)
|
790
790
|
# We don't have the actual content_type object here, only the name
|
791
791
|
content_type = None
|
792
|
-
|
793
|
-
f"
|
792
|
+
logger.debug(
|
793
|
+
f"Decorator: Extracted content type name via regex: {content_type_name_from_regex}"
|
794
794
|
)
|
795
795
|
else:
|
796
|
-
|
797
|
-
"
|
796
|
+
logger.debug(
|
797
|
+
"Decorator: Regex fallback also failed. Treating as non-Message type."
|
798
798
|
)
|
799
799
|
is_stream_message = False
|
800
800
|
content_type = None
|
801
801
|
# Check 2: Direct type check (Handles cases where get_origin might fail but type is correct)
|
802
802
|
elif isinstance(param_type, type) and param_type is message_cls:
|
803
803
|
# This case likely won't have generic args accessible easily if get_origin failed
|
804
|
-
|
805
|
-
"
|
804
|
+
logger.debug(
|
805
|
+
"Decorator: Input type identified as direct Message type. Attempting regex fallback."
|
806
806
|
)
|
807
807
|
is_stream_message = True
|
808
808
|
# --- Regex Fallback Start ---
|
809
809
|
match = re.search(r"Message\[([\w\.]+)\]", param_type_str_repr)
|
810
810
|
if match:
|
811
811
|
content_type_name_from_regex = match.group(1)
|
812
|
-
|
813
|
-
f"
|
812
|
+
logger.debug(
|
813
|
+
f"Decorator: Extracted content type name via regex: {content_type_name_from_regex}"
|
814
814
|
)
|
815
815
|
else:
|
816
|
-
|
817
|
-
"
|
816
|
+
logger.debug(
|
817
|
+
"Decorator: Regex fallback failed to extract content type name."
|
818
818
|
)
|
819
819
|
# --- Regex Fallback End ---
|
820
820
|
# Check 3: Removed old placeholder elif branch
|
821
821
|
|
822
822
|
else: # Handle cases where param_type might be None or origin is something else
|
823
|
-
|
824
|
-
f"
|
823
|
+
logger.debug(
|
824
|
+
f"Decorator: Input parameter '{param_name}' type ({param_type_str_repr}) identified as non-Message type."
|
825
825
|
)
|
826
826
|
|
827
|
-
|
828
|
-
f"
|
827
|
+
logger.debug(
|
828
|
+
f"Decorator: Final Input Type Determination: is_stream_message={is_stream_message}, content_type={content_type}"
|
829
829
|
)
|
830
830
|
# --- End Input Type Determination ---
|
831
831
|
|
832
832
|
# --- Validate Types are BaseModel ---
|
833
|
-
|
834
|
-
"
|
833
|
+
logger.debug(
|
834
|
+
"Decorator: Validating parameter and return types are BaseModel subclasses..."
|
835
835
|
)
|
836
836
|
|
837
837
|
# Define check_basemodel locally or ensure it's available
|
838
838
|
def check_basemodel(type_to_check: Optional[Any], desc: str):
|
839
|
-
#
|
839
|
+
# logger.debug(f"Decorator check_basemodel: Checking {desc} - Type: {type_to_check}") # Verbose
|
840
840
|
if type_to_check is None or type_to_check is Any:
|
841
|
-
|
842
|
-
f"
|
841
|
+
logger.debug(
|
842
|
+
f"Decorator check_basemodel: Skipping check for {desc} (type is None or Any)."
|
843
843
|
)
|
844
844
|
return
|
845
845
|
# Handle Optional[T] by getting the inner type
|
@@ -853,11 +853,11 @@ def processor(
|
|
853
853
|
non_none_args = [arg for arg in type_args if arg is not type(None)]
|
854
854
|
if len(non_none_args) == 1:
|
855
855
|
actual_type = non_none_args[0]
|
856
|
-
#
|
856
|
+
# logger.debug(f"Decorator check_basemodel: Unwrapped Optional/Union to {actual_type} for {desc}")
|
857
857
|
else:
|
858
858
|
# Handle complex Unions later if needed, skip check for now
|
859
|
-
|
860
|
-
f"
|
859
|
+
logger.debug(
|
860
|
+
f"Decorator check_basemodel: Skipping check for complex Union {desc}: {type_to_check}"
|
861
861
|
)
|
862
862
|
return
|
863
863
|
|
@@ -865,7 +865,7 @@ def processor(
|
|
865
865
|
effective_type = (
|
866
866
|
get_origin(actual_type) or actual_type
|
867
867
|
) # Handle generics like List[Model]
|
868
|
-
#
|
868
|
+
# logger.debug(f"Decorator check_basemodel: Effective type for {desc}: {effective_type}") # Verbose
|
869
869
|
if isinstance(effective_type, type) and not issubclass(
|
870
870
|
effective_type, BaseModel
|
871
871
|
):
|
@@ -880,26 +880,26 @@ def processor(
|
|
880
880
|
type(None),
|
881
881
|
)
|
882
882
|
if effective_type not in allowed_non_model_types:
|
883
|
-
|
884
|
-
f"
|
883
|
+
logger.error(
|
884
|
+
f"Decorator check_basemodel: Error - {desc} effective type ({effective_type.__name__}) is not BaseModel or standard type."
|
885
885
|
)
|
886
886
|
raise TypeError(
|
887
887
|
f"{desc} effective type ({effective_type.__name__}) must be BaseModel subclass or standard type (str, int, etc.)"
|
888
888
|
)
|
889
889
|
else:
|
890
|
-
|
891
|
-
f"
|
890
|
+
logger.debug(
|
891
|
+
f"Decorator check_basemodel: OK - {desc} is standard type {effective_type.__name__}."
|
892
892
|
)
|
893
893
|
|
894
894
|
elif not isinstance(effective_type, type):
|
895
895
|
# Allow TypeVars or other constructs for now? Or enforce BaseModel? Enforce for now.
|
896
|
-
|
897
|
-
f"
|
896
|
+
logger.warning(
|
897
|
+
f"Decorator check_basemodel: Warning - {desc} effective type '{effective_type}' is not a class. Cannot verify BaseModel subclass."
|
898
898
|
)
|
899
899
|
# Revisit this if TypeVars bound to BaseModel are needed.
|
900
900
|
else:
|
901
|
-
|
902
|
-
f"
|
901
|
+
logger.debug(
|
902
|
+
f"Decorator check_basemodel: OK - {desc} effective type ({effective_type.__name__}) is a BaseModel subclass."
|
903
903
|
)
|
904
904
|
|
905
905
|
effective_param_type = (
|
@@ -913,7 +913,7 @@ def processor(
|
|
913
913
|
if effective_param_type is not message_cls:
|
914
914
|
check_basemodel(effective_param_type, f"Parameter '{param_name}'")
|
915
915
|
check_basemodel(return_type, "Return value")
|
916
|
-
|
916
|
+
logger.debug("Decorator: Type validation complete.")
|
917
917
|
# --- End Type Validation ---
|
918
918
|
|
919
919
|
# --- Validate Execution Mode ---
|
@@ -921,11 +921,11 @@ def processor(
|
|
921
921
|
raise ValueError(
|
922
922
|
f"Invalid execution_mode: '{execution_mode}'. Must be 'inline' or 'subprocess'."
|
923
923
|
)
|
924
|
-
|
924
|
+
logger.debug(f"Decorator: Using execution mode: {execution_mode}")
|
925
925
|
# --- End Execution Mode Validation ---
|
926
926
|
|
927
927
|
# --- Populate Environment Variables ---
|
928
|
-
|
928
|
+
logger.debug("Decorator: Populating environment variables...")
|
929
929
|
# Keep: FUNCTION_NAME, PARAM_TYPE_STR, RETURN_TYPE_STR, IS_STREAM_MESSAGE, CONTENT_TYPE_NAME, MODULE_NAME
|
930
930
|
# Add: NEBU_ENTRYPOINT_MODULE_PATH
|
931
931
|
# Add: Included object sources (if any)
|
@@ -945,15 +945,15 @@ def processor(
|
|
945
945
|
calculated_module_path = ".".join(module_path_parts)
|
946
946
|
else:
|
947
947
|
# Not a python file? Should not happen based on inspect.getfile
|
948
|
-
|
949
|
-
f"
|
948
|
+
logger.warning(
|
949
|
+
f"Decorator: Function source file is not a .py file: {rel_func_path}"
|
950
950
|
)
|
951
951
|
# Set calculated_module_path to None explicitly to trigger fallback later
|
952
952
|
calculated_module_path = None
|
953
953
|
else:
|
954
954
|
# Should have errored earlier if rel_func_path is None
|
955
|
-
|
956
|
-
"
|
955
|
+
logger.warning(
|
956
|
+
"Decorator: Could not determine relative function path. Falling back to func.__module__."
|
957
957
|
)
|
958
958
|
# Set calculated_module_path to None explicitly to trigger fallback later
|
959
959
|
calculated_module_path = None
|
@@ -961,10 +961,10 @@ def processor(
|
|
961
961
|
# Assign final module_path using fallback if calculation failed or wasn't applicable
|
962
962
|
if calculated_module_path is not None:
|
963
963
|
module_path = calculated_module_path
|
964
|
-
|
964
|
+
logger.debug(f"Decorator: Using calculated module path: {module_path}")
|
965
965
|
else:
|
966
966
|
module_path = func.__module__ # Fallback
|
967
|
-
|
967
|
+
logger.debug(f"Decorator: Falling back to func.__module__: {module_path}")
|
968
968
|
|
969
969
|
# Basic info needed by consumer to find and run the function
|
970
970
|
all_env.append(V1EnvVar(key="FUNCTION_NAME", value=processor_name))
|
@@ -973,8 +973,8 @@ def processor(
|
|
973
973
|
all_env.append(
|
974
974
|
V1EnvVar(key="NEBU_ENTRYPOINT_MODULE_PATH", value=rel_func_path)
|
975
975
|
)
|
976
|
-
|
977
|
-
f"
|
976
|
+
logger.debug(
|
977
|
+
f"Decorator: Set NEBU_ENTRYPOINT_MODULE_PATH to: {rel_func_path}"
|
978
978
|
)
|
979
979
|
# No else needed, handled by fallback calculation above
|
980
980
|
|
@@ -987,7 +987,7 @@ def processor(
|
|
987
987
|
f"init_func '{init_func_name}' must take zero parameters"
|
988
988
|
)
|
989
989
|
all_env.append(V1EnvVar(key="INIT_FUNC_NAME", value=init_func_name))
|
990
|
-
|
990
|
+
logger.debug(f"Decorator: Set INIT_FUNC_NAME to: {init_func_name}")
|
991
991
|
|
992
992
|
# Type info (still useful for deserialization/validation in consumer)
|
993
993
|
# Adjust type strings to replace '__main__' with the calculated module path
|
@@ -999,8 +999,8 @@ def processor(
|
|
999
999
|
param_type_str_repr = param_type_str_repr.replace(
|
1000
1000
|
"__main__.", f"{module_path}."
|
1001
1001
|
)
|
1002
|
-
|
1003
|
-
f"
|
1002
|
+
logger.debug(
|
1003
|
+
f"Decorator: Adjusted param type string: {param_type_str_repr}"
|
1004
1004
|
)
|
1005
1005
|
|
1006
1006
|
all_env.append(V1EnvVar(key="PARAM_TYPE_STR", value=param_type_str_repr))
|
@@ -1010,8 +1010,8 @@ def processor(
|
|
1010
1010
|
return_type_str_repr = return_type_str_repr.replace(
|
1011
1011
|
"__main__.", f"{module_path}."
|
1012
1012
|
)
|
1013
|
-
|
1014
|
-
f"
|
1013
|
+
logger.debug(
|
1014
|
+
f"Decorator: Adjusted return type string: {return_type_str_repr}"
|
1015
1015
|
)
|
1016
1016
|
|
1017
1017
|
all_env.append(V1EnvVar(key="RETURN_TYPE_STR", value=return_type_str_repr))
|
@@ -1021,19 +1021,19 @@ def processor(
|
|
1021
1021
|
content_type_name_to_set = None
|
1022
1022
|
if content_type and isinstance(content_type, type):
|
1023
1023
|
content_type_name_to_set = content_type.__name__
|
1024
|
-
|
1025
|
-
f"
|
1024
|
+
logger.debug(
|
1025
|
+
f"Decorator: Using content type name from resolved type object: {content_type_name_to_set}"
|
1026
1026
|
)
|
1027
1027
|
elif content_type_name_from_regex:
|
1028
1028
|
content_type_name_to_set = content_type_name_from_regex
|
1029
|
-
|
1030
|
-
f"
|
1029
|
+
logger.debug(
|
1030
|
+
f"Decorator: Using content type name from regex fallback: {content_type_name_to_set}"
|
1031
1031
|
)
|
1032
1032
|
else:
|
1033
1033
|
# Only warn if it was supposed to be a Message type
|
1034
1034
|
if is_stream_message:
|
1035
|
-
|
1036
|
-
f"
|
1035
|
+
logger.warning(
|
1036
|
+
f"Could not determine CONTENT_TYPE_NAME for Message parameter {param_name} ({param_type_str_repr}). Consumer might use raw content."
|
1037
1037
|
)
|
1038
1038
|
|
1039
1039
|
if content_type_name_to_set:
|
@@ -1047,17 +1047,17 @@ def processor(
|
|
1047
1047
|
key="MODULE_NAME", value=module_path
|
1048
1048
|
) # module_path is guaranteed to be a string here (calculated or fallback)
|
1049
1049
|
)
|
1050
|
-
|
1050
|
+
logger.debug(f"Decorator: Set MODULE_NAME to: {module_path}")
|
1051
1051
|
|
1052
1052
|
# Add Execution Mode
|
1053
1053
|
all_env.append(V1EnvVar(key="NEBU_EXECUTION_MODE", value=execution_mode))
|
1054
|
-
|
1054
|
+
logger.debug(f"Decorator: Set NEBU_EXECUTION_MODE to: {execution_mode}")
|
1055
1055
|
|
1056
1056
|
# Add Hot Reload Configuration
|
1057
1057
|
if not hot_reload:
|
1058
1058
|
all_env.append(V1EnvVar(key="NEBU_DISABLE_HOT_RELOAD", value="1"))
|
1059
|
-
|
1060
|
-
"
|
1059
|
+
logger.debug(
|
1060
|
+
"Decorator: Set NEBU_DISABLE_HOT_RELOAD to: 1 (Hot reload disabled)"
|
1061
1061
|
)
|
1062
1062
|
else:
|
1063
1063
|
# Ensure it's explicitly '0' or unset if enabled (consumer defaults to enabled if var missing)
|
@@ -1071,8 +1071,8 @@ def processor(
|
|
1071
1071
|
else:
|
1072
1072
|
# Not strictly needed as consumer defaults to enabled, but explicit is good.
|
1073
1073
|
all_env.append(V1EnvVar(key="NEBU_DISABLE_HOT_RELOAD", value="0"))
|
1074
|
-
|
1075
|
-
"
|
1074
|
+
logger.debug(
|
1075
|
+
"Decorator: Hot reload enabled (NEBU_DISABLE_HOT_RELOAD=0 or unset)"
|
1076
1076
|
)
|
1077
1077
|
|
1078
1078
|
# Add PYTHONPATH
|
@@ -1091,15 +1091,15 @@ def processor(
|
|
1091
1091
|
existing_pythonpath.value = pythonpath_value
|
1092
1092
|
else:
|
1093
1093
|
all_env.append(V1EnvVar(key="PYTHONPATH", value=pythonpath_value))
|
1094
|
-
|
1094
|
+
logger.debug(f"Decorator: Ensured PYTHONPATH includes: {pythonpath_value}")
|
1095
1095
|
|
1096
|
-
|
1096
|
+
logger.debug("Decorator: Finished populating environment variables.")
|
1097
1097
|
# --- End Environment Variables ---
|
1098
1098
|
|
1099
1099
|
# --- Add S3 Sync Volume ---
|
1100
1100
|
if s3_destination_uri:
|
1101
|
-
|
1102
|
-
f"
|
1101
|
+
logger.debug(
|
1102
|
+
f"Decorator: Adding volume to sync S3 code from {s3_destination_uri} to {CONTAINER_CODE_DIR}"
|
1103
1103
|
)
|
1104
1104
|
s3_sync_volume = V1VolumePath(
|
1105
1105
|
source=s3_destination_uri,
|
@@ -1114,8 +1114,8 @@ def processor(
|
|
1114
1114
|
):
|
1115
1115
|
all_volumes.append(s3_sync_volume)
|
1116
1116
|
else:
|
1117
|
-
|
1118
|
-
f"
|
1117
|
+
logger.debug(
|
1118
|
+
f"Decorator: Volume for {s3_destination_uri} to {CONTAINER_CODE_DIR} already exists."
|
1119
1119
|
)
|
1120
1120
|
else:
|
1121
1121
|
# Should have errored earlier if S3 upload failed
|
@@ -1125,7 +1125,7 @@ def processor(
|
|
1125
1125
|
# --- End S3 Sync Volume ---
|
1126
1126
|
|
1127
1127
|
# --- Final Setup ---
|
1128
|
-
|
1128
|
+
logger.debug("Decorator: Preparing final Processor object...")
|
1129
1129
|
metadata = V1ResourceMetaRequest(
|
1130
1130
|
name=processor_name, namespace=effective_namespace, labels=labels
|
1131
1131
|
)
|
@@ -1145,16 +1145,19 @@ def processor(
|
|
1145
1145
|
setup_commands_list = [base_deps_install]
|
1146
1146
|
|
1147
1147
|
if setup_script:
|
1148
|
-
|
1148
|
+
logger.debug("Decorator: Adding user setup script to setup commands.")
|
1149
1149
|
setup_commands_list.append(setup_script.strip())
|
1150
1150
|
|
1151
|
+
if debug:
|
1152
|
+
all_env.append(V1EnvVar(key="PYTHON_LOG", value="DEBUG"))
|
1153
|
+
|
1151
1154
|
# Combine setup commands and the final execution command
|
1152
1155
|
all_commands = setup_commands_list + [consumer_execution_command]
|
1153
1156
|
# Use newline separator for clarity in logs and script execution
|
1154
1157
|
final_command = "\n".join(all_commands)
|
1155
1158
|
|
1156
|
-
|
1157
|
-
f"
|
1159
|
+
logger.debug(
|
1160
|
+
f"Decorator: Final container command:\n-------\n{final_command}\n-------"
|
1158
1161
|
)
|
1159
1162
|
|
1160
1163
|
container_request = V1ContainerRequest(
|
@@ -1177,16 +1180,16 @@ def processor(
|
|
1177
1180
|
proxy_port=proxy_port,
|
1178
1181
|
health_check=health_check,
|
1179
1182
|
)
|
1180
|
-
|
1183
|
+
logger.debug("Decorator: Final Container Request Env Vars (Summary):")
|
1181
1184
|
for env_var in all_env:
|
1182
1185
|
# Avoid printing potentially large included source code
|
1183
1186
|
value_str = env_var.value or ""
|
1184
1187
|
if "SOURCE" in env_var.key and len(value_str) > 100:
|
1185
|
-
|
1186
|
-
f"
|
1188
|
+
logger.debug(
|
1189
|
+
f" {env_var.key}: <source code present, length={len(value_str)}>"
|
1187
1190
|
)
|
1188
1191
|
else:
|
1189
|
-
|
1192
|
+
logger.debug(f" {env_var.key}: {value_str}")
|
1190
1193
|
|
1191
1194
|
processor_instance = Processor(
|
1192
1195
|
name=processor_name,
|
@@ -1200,19 +1203,19 @@ def processor(
|
|
1200
1203
|
scale_config=scale,
|
1201
1204
|
no_delete=no_delete,
|
1202
1205
|
)
|
1203
|
-
|
1204
|
-
f"
|
1206
|
+
logger.debug(
|
1207
|
+
f"Decorator: Processor instance '{processor_name}' created successfully."
|
1205
1208
|
)
|
1206
1209
|
# Store original func for potential local invocation/testing? Keep for now.
|
1207
1210
|
# TODO: Add original_func to Processor model definition if this is desired
|
1208
|
-
#
|
1209
|
-
try:
|
1210
|
-
|
1211
|
-
|
1212
|
-
except AttributeError:
|
1213
|
-
|
1214
|
-
|
1215
|
-
|
1211
|
+
# Commenting out as Processor model does not have this field
|
1212
|
+
# try:
|
1213
|
+
# # This will fail if Processor hasn't been updated to include this field
|
1214
|
+
# processor_instance.original_func = func # type: ignore
|
1215
|
+
# except AttributeError:
|
1216
|
+
# logger.warning(
|
1217
|
+
# "Could not assign original_func to Processor instance. Update Processor model or remove assignment."
|
1218
|
+
# )
|
1216
1219
|
|
1217
1220
|
return processor_instance
|
1218
1221
|
|