cometx 3.2.2__tar.gz → 3.3.0__tar.gz

{cometx-3.2.2/cometx.egg-info → cometx-3.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cometx
-Version: 3.2.2
+Version: 3.3.0
 Summary: Python tools for Comet
 Home-page: https://github.com/comet-ml/cometx/
 Author: cometx development team

{cometx-3.2.2 → cometx-3.3.0}/cometx/_version.py

@@ -12,5 +12,5 @@
 #  the express permission of Comet ML Inc.
 # *******************************************************
 
-version_info = (3, 2, 2)
+version_info = (3, 3, 0)
 __version__ = ".".join(map(str, version_info))

{cometx-3.2.2 → cometx-3.3.0}/cometx/cli/__init__.py

@@ -48,6 +48,7 @@ from . import (
     reproduce,
     smoke_test,
     update,
+    upload_optimizer_experiments,
 )
 
 
@@ -103,6 +104,9 @@ def main(raw_args=sys.argv[1:]):
     add_subparser(subparsers, reproduce, "reproduce")
     add_subparser(subparsers, config, "config")
     add_subparser(subparsers, smoke_test, "smoke-test")
+    add_subparser(
+        subparsers, upload_optimizer_experiments, "upload-optimizer-experiments"
+    )
 
     # Parse arguments
     args, rest = parser.parse_known_args(raw_args)

{cometx-3.2.2 → cometx-3.3.0}/cometx/cli/admin.py

@@ -90,6 +90,33 @@ Commands:
         Output:
             Returns a dictionary of metrics keyed by experiment key.
 
+    optimizer-report
+        Generate a report for an optimizer instance.
+
+        Usage:
+            cometx admin optimizer-report OPTIMIZER_ID
+            cometx admin optimizer-report OPTIMIZER_ID --app
+
+        Arguments:
+            OPTIMIZER_ID (required)
+                The optimizer instance ID to generate a report for.
+
+        Options:
+            --app
+                Launch interactive Streamlit web app instead of generating JSON file.
+
+        Output:
+            Without --app:
+                Generates a JSON file containing all dashboard/data items:
+                - optimizer-report-{OPTIMIZER_ID}.json
+
+            With --app:
+                Launches an interactive web interface where you can:
+                - View optimizer dashboard with statistics
+                - Filter by status
+                - Navigate through paginated job assignments
+                - View summary statistics
+
 Global Options (available for all commands):
     --api-key KEY
         Set the COMET_API_KEY for authentication.
@@ -113,17 +140,21 @@ Examples:
     cometx admin gpu-report my-workspace --start-date 2024-01-01
     cometx admin gpu-report my-workspace --start-date 2024-01-01 --end-date 2024-12-31
     cometx admin gpu-report workspace1/project1 workspace2 --start-date 2024-01-01 --metrics sys.gpu.0.gpu_utilization
+    cometx admin optimizer-report abc123def456
+    cometx admin optimizer-report abc123def456 --app
 
 """
 
 import argparse
 import json
+import os
 import sys
 from urllib.parse import urlparse
 
 from comet_ml import API
 
 from .admin_gpu_report import main as gpu_report_main
+from .admin_optimizer_report import generate_json_report
 from .admin_usage_report import generate_usage_report
 
 ADDITIONAL_ARGS = False
@@ -275,7 +306,7 @@ Arguments:
 
 Options:
     --start-date DATE
-        Start date for the report in YYYY-MM-DD format (required).
+        Start date for the report in YYYY-MM-DD format (optional, collects all data if not provided).
 
     --end-date DATE
         End date for the report in YYYY-MM-DD format (optional).
@@ -293,18 +324,35 @@ Options:
     --open
         Automatically open the generated PDF file after generation.
 
+    --app
+        Launch interactive Streamlit web app instead of generating PDF.
+        The JSON file will be generated first and automatically loaded in the app.
+        The app allows you to interactively change aggregation (workspace, project, user),
+        time units (year, month, week, day), and date ranges.
+
 Output:
-    Generates a PDF report containing:
-    - Summary statistics (total experiments, workspaces, metrics tracked)
-    - Average metrics by workspace charts
-    - Maximum metrics by month charts
+    Without --app:
+        Generates a PDF report and JSON file containing:
+        - Summary statistics (total experiments, workspaces, metrics tracked)
+        - Average metrics by workspace charts
+        - Maximum metrics by month charts
+        - JSON file: gpu_report_{workspace_projects}.json
+
+    With --app:
+        Launches an interactive web interface where you can:
+        - Change aggregation (workspace, project, user)
+        - Change time units (year, month, week, day)
+        - Select start and end dates from the data
+        - View charts for each metric
 
 Examples:
+    cometx admin gpu-report my-workspace
     cometx admin gpu-report my-workspace --start-date 2024-01-01
     cometx admin gpu-report my-workspace --start-date 2024-01-01 --end-date 2024-12-31
-    cometx admin gpu-report workspace1/project1 workspace2 --start-date 2024-01-01
-    cometx admin gpu-report my-workspace --start-date 2024-01-01 --metrics sys.gpu.0.gpu_utilization sys.gpu.0.memory_utilization
-    cometx admin gpu-report my-workspace --start-date 2024-01-01 --open
+    cometx admin gpu-report workspace1/project1 workspace2
+    cometx admin gpu-report my-workspace --metrics sys.gpu.0.gpu_utilization sys.gpu.0.memory_utilization
+    cometx admin gpu-report my-workspace --open
+    cometx admin gpu-report my-workspace --app
 """
     gpu_parser = subparsers.add_parser(
         "gpu-report",
@@ -316,16 +364,16 @@ Examples:
     add_global_arguments(gpu_parser)
     gpu_parser.add_argument(
         "WORKSPACE_PROJECT",
-        nargs="+",
+        nargs="*",
         help="One or more WORKSPACE or WORKSPACE/PROJECT to run GPU report for",
         metavar="WORKSPACE",
         type=str,
     )
     gpu_parser.add_argument(
         "--start-date",
-        help="Start date for the report in YYYY-MM-DD format (required)",
+        help="Start date for the report in YYYY-MM-DD format (optional, collects all data if not provided)",
         type=str,
-        required=True,
+        required=False,
     )
     gpu_parser.add_argument(
         "--end-date",
@@ -346,6 +394,60 @@ Examples:
         default=False,
         action="store_true",
     )
+    gpu_parser.add_argument(
+        "--app",
+        help="Launch interactive Streamlit web app instead of generating PDF",
+        default=False,
+        action="store_true",
+    )
+
+    # optimizer-report subcommand
+    optimizer_report_description = """Generate a report for an optimizer instance.
+
+Arguments:
+    OPTIMIZER_ID (required)
+        The optimizer instance ID to generate a report for.
+
+Options:
+    --app
+        Launch interactive Streamlit web app instead of generating JSON file.
+
+Output:
+    Without --app:
+        Generates a JSON file containing all dashboard/data items:
+        - optimizer-report-{OPTIMIZER_ID}.json
+
+    With --app:
+        Launches an interactive web interface where you can:
+        - View optimizer dashboard with statistics
+        - Filter by status
+        - Navigate through paginated job assignments
+        - View summary statistics
+
+Examples:
+    cometx admin optimizer-report abc123def456
+    cometx admin optimizer-report abc123def456 --app
+"""
+    optimizer_parser = subparsers.add_parser(
+        "optimizer-report",
+        help="Generate a report for an optimizer instance",
+        description=optimizer_report_description,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    # Add global arguments to subparser so they show in help
+    add_global_arguments(optimizer_parser)
+    optimizer_parser.add_argument(
+        "OPTIMIZER_ID",
+        help="The optimizer instance ID to generate a report for",
+        metavar="OPTIMIZER_ID",
+        type=str,
+    )
+    optimizer_parser.add_argument(
+        "--app",
+        help="Launch interactive Streamlit web app instead of generating JSON file",
+        default=False,
+        action="store_true",
+    )
 
 
 def admin(parsed_args, remaining=None):
@@ -391,8 +493,6 @@ def admin(parsed_args, remaining=None):
         elif parsed_args.ACTION == "usage-report":
             if parsed_args.app:
                 # Launch Streamlit app
-                import os
-                import sys
 
                 # Set environment variables if --api-key or --url-override were provided
                 if parsed_args.api_key:
@@ -440,44 +540,197 @@ def admin(parsed_args, remaining=None):
                     print("ERROR: " + str(e))
                     return
         elif parsed_args.ACTION == "gpu-report":
-            workspace_projects = parsed_args.WORKSPACE_PROJECT
+            workspace_projects = parsed_args.WORKSPACE_PROJECT or []
             start_date = parsed_args.start_date
             end_date = parsed_args.end_date
             metrics = parsed_args.metrics
 
-            try:
-                result = gpu_report_main(
-                    workspace_projects=workspace_projects,
-                    start_date=start_date,
-                    end_date=end_date,
-                    metrics=metrics,
-                    max_workers=None,  # Use default
-                )
-                if result:
-                    num_experiments = len(result.get("metrics", {}))
-                    num_charts = len(result.get("charts", []))
-                    pdf_file = result.get("pdf_file")
+            if parsed_args.app:
+                # Require workspace_projects for --app
+                if not workspace_projects:
+                    print(
+                        "ERROR: At least one workspace/project is required when using --app"
+                    )
+                    return
+
+                # Generate JSON first
+                json_file_path = None
+                try:
+                    print("Generating GPU report data...")
+                    result = gpu_report_main(
+                        workspace_projects=workspace_projects,
+                        start_date=start_date,
+                        end_date=end_date,
+                        metrics=metrics,
+                        max_workers=None,  # Use default
+                    )
+                    if result:
+                        json_file_path = result.get("json_file")
+                        if json_file_path:
+                            print(f"JSON report saved: {json_file_path}")
+                        else:
+                            print("ERROR: JSON file was not created")
+                            return
+                    else:
+                        print("ERROR: Failed to generate GPU report data")
+                        return
+                except Exception as e:
+                    print(f"ERROR: Could not generate JSON file: {e}")
+                    if parsed_args.debug:
+                        import traceback
+
+                        traceback.print_exc()
+                    return
+
+                # Launch Streamlit app
+                # Set environment variables if --api-key or --url-override were provided
+                if parsed_args.api_key:
+                    os.environ["COMET_API_KEY"] = parsed_args.api_key
+                if parsed_args.url_override:
+                    os.environ["COMET_URL_OVERRIDE"] = parsed_args.url_override
+
+                # Run the Streamlit app using streamlit's CLI
+                try:
+                    import tempfile
+
+                    import streamlit.web.cli as stcli
+
+                    # Get the path to admin_gpu_app.py
+                    gpu_app_path = os.path.join(
+                        os.path.dirname(os.path.abspath(__file__)), "admin_gpu_app.py"
+                    )
+
+                    # Write JSON file path to a temp file
+                    # The Streamlit app will read this file
+                    temp_dir = tempfile.gettempdir()
+                    json_path_file = os.path.join(
+                        temp_dir, "comet_gpu_report_json_path.txt"
+                    )
+                    with open(json_path_file, "w") as f:
+                        f.write(os.path.abspath(json_file_path))
+
+                    # Launch streamlit with the app
+                    sys.argv = ["streamlit", "run", gpu_app_path]
+                    stcli.main()
+                except Exception as e:
+                    print(f"ERROR launching Streamlit app: {e}")
+                    if parsed_args.debug:
+                        raise
+                    return
+            else:
+                # Generate report (JSON is always saved by gpu_report_main)
+                if not workspace_projects:
                     print(
-                        f"\nGPU report completed. Processed {num_experiments} experiments."
+                        "ERROR: At least one workspace/project is required when not using --app"
                     )
-                    if num_charts > 0:
-                        print(f"Generated {num_charts} charts:")
-                        for chart_file in result.get("charts", []):
-                            print(f"  - {chart_file}")
-                    if pdf_file:
-                        print(f"PDF report: {pdf_file}")
-                        # Open PDF if --open flag is set
-                        if parsed_args.open:
-                            from .admin_gpu_report import open_pdf
-
-                            open_pdf(pdf_file, debug=parsed_args.debug)
-            except Exception as e:
-                print("ERROR: " + str(e))
-                if parsed_args.debug:
-                    import traceback
-
-                    traceback.print_exc()
-                return
+                    return
+                try:
+                    result = gpu_report_main(
+                        workspace_projects=workspace_projects,
+                        start_date=start_date,
+                        end_date=end_date,
+                        metrics=metrics,
+                        max_workers=None,  # Use default
+                    )
+                    if result:
+                        num_experiments = len(result.get("metrics", {}))
+                        num_charts = len(result.get("charts", []))
+                        pdf_file = result.get("pdf_file")
+                        json_file = result.get("json_file")
+
+                        print(
+                            f"\nGPU report completed. Processed {num_experiments} experiments."
+                        )
+                        if num_charts > 0:
+                            print(f"Generated {num_charts} charts:")
+                            for chart_file in result.get("charts", []):
+                                print(f"  - {chart_file}")
+                        if json_file:
+                            print(f"JSON report: {json_file}")
+                        if pdf_file:
+                            print(f"PDF report: {pdf_file}")
+                            # Open PDF if --open flag is set
+                            if parsed_args.open:
+                                from .admin_gpu_report import open_pdf
+
+                                open_pdf(pdf_file, debug=parsed_args.debug)
+                except Exception as e:
+                    print("ERROR: " + str(e))
+                    if parsed_args.debug:
+                        import traceback
+
+                        traceback.print_exc()
+                    return
+        elif parsed_args.ACTION == "optimizer-report":
+            optimizer_id = parsed_args.OPTIMIZER_ID
+
+            if parsed_args.app:
+                # Launch Streamlit app
+                # Set environment variables if --api-key or --url-override were provided
+                if parsed_args.api_key:
+                    os.environ["COMET_API_KEY"] = parsed_args.api_key
+                if parsed_args.url_override:
+                    os.environ["COMET_URL_OVERRIDE"] = parsed_args.url_override
+
+                # Run the Streamlit app using streamlit's CLI
+                try:
+                    import streamlit.web.cli as stcli
+
+                    # Get the path to admin_optimizer_report.py
+                    optimizer_app_path = os.path.join(
+                        os.path.dirname(os.path.abspath(__file__)),
+                        "admin_optimizer_report.py",
+                    )
+
+                    # Set optimizer ID in environment so the app can access it
+                    # Use COMET_OPTIMIZER_ID if not already set
+                    if "COMET_OPTIMIZER_ID" not in os.environ:
+                        os.environ["COMET_OPTIMIZER_ID"] = optimizer_id
+
+                    # Launch streamlit with the app (Streamlit will automatically open the browser)
+                    sys.argv = ["streamlit", "run", optimizer_app_path]
+                    stcli.main()
+                except Exception as e:
+                    print(f"ERROR launching Streamlit app: {e}")
+                    if parsed_args.debug:
+                        raise
+                    return
+            else:
+                # Generate JSON report
+                try:
+                    # Get API key and optimizer URL from config or arguments
+                    from comet_ml.config import get_config, get_optimizer_address
+
+                    config_obj = get_config()
+
+                    # Get API key from parsed args, config, or environment
+                    api_key = parsed_args.api_key
+                    if not api_key:
+                        try:
+                            api_key = config_obj["comet.api_key"]
+                        except (KeyError, TypeError):
+                            api_key = os.environ.get("COMET_API_KEY")
+
+                    # Get optimizer URL from config
+                    optimizer_url = get_optimizer_address(config_obj)
+
+                    result = generate_json_report(
+                        optimizer_id=optimizer_id,
+                        api_key=api_key,
+                        optimizer_url=optimizer_url,
+                    )
+                    if result:
+                        print(f"\nOptimizer report generated successfully: {result}")
+                    else:
+                        print("ERROR: Failed to generate optimizer report")
+                        return
+                except Exception as e:
+                    print("ERROR: " + str(e))
+                    if parsed_args.debug:
+                        import traceback
+
+                        traceback.print_exc()
+                    return
 
     except KeyboardInterrupt:
         if parsed_args.debug: