pylantir 0.2.1__py3-none-any.whl → 0.2.3__py3-none-any.whl

pylantir/redcap_to_db.py

@@ -55,14 +55,23 @@ def fetch_redcap_entries(redcap_fields: list, interval: float) -> list:
     datetime_interval = datetime_now - timedelta(seconds=interval)
     records = project.export_records(fields=redcap_fields, date_begin=datetime_interval, date_end=datetime_now, format_type="df")
 
+    # Clean up PyCap Project immediately after export to free API client cache
+    del project
+    gc.collect()
+
     if records.empty:
         lgr.warning("No records retrieved from REDCap.")
+        # Explicitly clean up the empty DataFrame to release any allocated buffers
+        del records
+        gc.collect()
         return []
 
     filtered_records = []
 
     # Group by 'record_id' (index level 0)
-    for record_id, group in records.groupby(level=0):
+    # Convert to list to avoid holding groupby iterator reference
+    record_groups = list(records.groupby(level=0))
+    for record_id, group in record_groups:
 
         # Try to get baseline (non-repeated instrument) values
         baseline_rows = group[group['redcap_repeat_instrument'].isna()]
@@ -90,6 +99,11 @@ def fetch_redcap_entries(redcap_fields: list, interval: float) -> list:
 
             filtered_records.append(record)
 
+    # Explicitly clean up DataFrame and groupby list to free memory
+    del record_groups
+    del records
+    gc.collect()
+    
     return filtered_records
 
 # TODO: Implement age binning for paricipants
@@ -164,11 +178,11 @@ def cleanup_memory_and_connections():
             lgr.debug("Disposing database connection pool")
             engine.pool.dispose()
         
-        # 3. Force Python garbage collection
-        # Run multiple times to catch circular references
-        collected = 0
-        for _ in range(3):
-            collected += gc.collect()
+        # 3. Force Python garbage collection targeting all generations
+        # Target generation 2 (oldest) first to catch long-lived objects
+        collected = gc.collect(generation=2)  # Oldest generation
+        collected += gc.collect(generation=1)  # Middle generation
+        collected += gc.collect(generation=0)  # Youngest generation
         
         # 4. Clear any cached SQLAlchemy metadata
         if hasattr(engine, 'pool'):
@@ -178,20 +192,17 @@ def cleanup_memory_and_connections():
         # Get memory usage after cleanup
         memory_after = get_memory_usage()
         
-        # Log cleanup results
-        if memory_before and memory_after:
-            if 'rss_mb' in memory_before:
-                memory_freed = memory_before['rss_mb'] - memory_after['rss_mb']
-                lgr.info(f"Memory cleanup completed. "
-                        f"Before: {memory_before['rss_mb']}MB, "
-                        f"After: {memory_after['rss_mb']}MB, "
-                        f"Freed: {memory_freed:.2f}MB, "
-                        f"Collected {collected} objects")
-            else:
-                lgr.info(f"Memory cleanup completed. Collected {collected} objects. "
-                        f"Memory stats: {memory_after}")
+        # Log cleanup results with simplified, focused metrics
+        if memory_before and memory_after and 'rss_mb' in memory_before:
+            freed = memory_before['rss_mb'] - memory_after['rss_mb']
+            lgr.info(
+                f"Memory cleanup: Before={memory_before['rss_mb']:.1f}MB, "
+                f"After={memory_after['rss_mb']:.1f}MB, "
+                f"Freed={freed:.1f}MB, "
+                f"Objects={collected}"
+            )
         else:
-            lgr.info(f"Memory cleanup completed. Collected {collected} objects")
+            lgr.info(f"Memory cleanup: Collected {collected} objects")
             
     except Exception as e:
         lgr.error(f"Error during cleanup: {e}")
@@ -328,6 +339,8 @@ def sync_redcap_to_db(
     finally:
         # Always ensure session is properly closed
         if session:
+            # Detach all ORM objects from session to clear identity map
+            session.expunge_all()
             session.close()
         
         # Perform cleanup after sync

{pylantir-0.2.1.dist-info → pylantir-0.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pylantir
-Version: 0.2.1
+Version: 0.2.3
 Summary: Python - DICOM Modality WorkList with Optional API
 Author-email: Milton Camacho <miltoncamachoicc@gmail.com>
 Requires-Python: >=3.11.1
@@ -86,13 +86,15 @@ pip install pylantir[api]
 ```
 Includes: FastAPI, Uvicorn, JWT authentication, password hashing
 
-#### Memory Monitoring
+#### Memory Monitoring (Recommended for Production)
 For enhanced memory usage monitoring and cleanup during REDCap synchronization:
 ```bash
 pip install pylantir[monitoring]
 ```
 Includes: psutil for system resource monitoring
 
+**Note**: While memory cleanup functions work without psutil, you need it installed to see cleanup effectiveness in logs. Without psutil, logs will show high-water mark memory values that don't decrease, even though cleanup is working. For production deployments, installing `[monitoring]` is **highly recommended** to validate memory stability.
+
 #### Big Data Processing
 For Spark-based data processing capabilities:
 ```bash

{pylantir-0.2.1.dist-info → pylantir-0.2.3.dist-info}/RECORD

@@ -8,13 +8,13 @@ pylantir/db_setup.py,sha256=ljL3u6RTj1W7JqAsi_iYvV0rRUtwRut66X4IDAMQmCQ,3572
 pylantir/models.py,sha256=bKgI0EN1VSYanPTOvEhEY2Zzqa0gDYLpVnE_KNQ6PEc,1780
 pylantir/mwl_server.py,sha256=GMJDcK0u_KM3oa6UqQ87NxMVye2pvG2cdkcI9k_iExg,10338
 pylantir/populate_db.py,sha256=KIbkVA-EAuTlDArXMFOHkjMmVfjlsTApj7S1wpUu1bM,2207
-pylantir/redcap_to_db.py,sha256=fozeCwELxiQS1grGlbxLiK3r9myKtNmIC_p1CBlVcAI,18952
+pylantir/redcap_to_db.py,sha256=rzA9VZl2qG67Ff82lxKgpXFVVkrmxBlr7Fd5OZORrVU,19476
 pylantir/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pylantir/cli/run.py,sha256=4JI9cRwoKYGG53DDP32ZBfXlZp_5hl0C3A7FiM3JyJw,21885
 pylantir/config/config_example_with_cors.json,sha256=4F6YCOYMIPOZVnEdrWw3txIPWUyA3hRBIRs-zS5h2Vk,2885
 pylantir/config/mwl_config.json,sha256=I1nPzXofmw2JviS7hEOw01ExdL6pGdgH46rcKYu6ofo,1437
-pylantir-0.2.1.dist-info/entry_points.txt,sha256=vxaxvfGppLqRt9_4sqNDdP6b2jlgpcHIwP7UQfrM1T0,50
-pylantir-0.2.1.dist-info/licenses/LICENSE,sha256=ws_MuBL-SCEBqPBFl9_FqZkaaydIJmxHrJG2parhU4M,1141
-pylantir-0.2.1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-pylantir-0.2.1.dist-info/METADATA,sha256=B1Fmmu5hEz28mPlDXhNDlYyyaVEZzKTq-5LTvv_SlR8,19512
-pylantir-0.2.1.dist-info/RECORD,,
+pylantir-0.2.3.dist-info/entry_points.txt,sha256=vxaxvfGppLqRt9_4sqNDdP6b2jlgpcHIwP7UQfrM1T0,50
+pylantir-0.2.3.dist-info/licenses/LICENSE,sha256=ws_MuBL-SCEBqPBFl9_FqZkaaydIJmxHrJG2parhU4M,1141
+pylantir-0.2.3.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+pylantir-0.2.3.dist-info/METADATA,sha256=_QLhbm0N61fQGwKtlZHPONSlchoJGCm1Dgh4yIi0nVU,19888
+pylantir-0.2.3.dist-info/RECORD,,