ml-dash 0.6.0__py3-none-any.whl → 0.6.2__py3-none-any.whl

ml_dash/remote_auto_start.py

@@ -9,18 +9,18 @@ IMPORTANT: Before using rdxp, you must authenticate with the ML-Dash server:
     python -m ml_dash.cli login
 
 Usage:
-    from ml_dash import rdxp
+    from ml_dash.remote_auto_start import rdxp
 
     # Use with statement (recommended)
     with rdxp.run:
-        rdxp.log().info("Hello from rdxp!")
+        rdxp.log("Hello from rdxp!", level="info")
         rdxp.params.set(lr=0.001)
-        rdxp.metrics("loss").append(step=0, value=0.5)
+        rdxp.metrics("train").log(loss=0.5, step=0)
     # Automatically completes on exit from with block
 
     # Or start/complete manually
     rdxp.run.start()
-    rdxp.log().info("Training...")
+    rdxp.log("Training...", level="info")
     rdxp.run.complete()
 
 Configuration:
@@ -30,25 +30,28 @@ Configuration:
 """
 
 import atexit
-from .experiment import Experiment
 
 # Create pre-configured singleton experiment for remote mode
 # Uses remote API server - token auto-loaded from storage
-rdxp = Experiment(
-    name="rdxp",
-    project="scratch",
-    remote="https://api.dash.ml"
-)
+# Prefix format: {owner}/{project}/path...
+import getpass
+
+from .experiment import Experiment
+
+_owner = getpass.getuser()
+rdxp = Experiment(prefix=f"{_owner}/scratch/rdxp", dash_url="https://api.dash.ml")
+
 
 # Register cleanup handler to complete experiment on Python exit (if still open)
 def _cleanup():
-    """Complete the rdxp experiment on exit if still open."""
-    if rdxp._is_open:
-        try:
-            rdxp.run.complete()
-        except Exception:
-            # Silently ignore errors during cleanup
-            pass
+  """Complete the rdxp experiment on exit if still open."""
+  if rdxp._is_open:
+    try:
+      rdxp.run.complete()
+    except Exception:
+      # Silently ignore errors during cleanup
+      pass
+
 
 atexit.register(_cleanup)
 

ml_dash/run.py

@@ -0,0 +1,231 @@
+"""
+RUN - Global experiment configuration object for ML-Dash.
+
+This module provides a global RUN object that serves as the single source
+of truth for experiment metadata. Uses params-proto for configuration.
+
+Usage:
+    from ml_dash import RUN
+
+    # Configure via environment variable
+    # export ML_DASH_PREFIX="ge/myproject/experiments/exp1"
+
+    # Or set directly
+    RUN.PREFIX = "ge/myproject/experiments/exp1"
+
+    # Use in templates
+    prefix = "{RUN.PREFIX}/{RUN.name}.{RUN.id}".format(RUN=RUN)
+
+    # With Experiment (RUN is auto-populated)
+    from ml_dash import Experiment
+    with Experiment(prefix=RUN.PREFIX).run as exp:
+        exp.logs.info(f"Running {RUN.name}")
+"""
+
+import os
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Union
+
+from params_proto import EnvVar, proto
+
+PROJECT_ROOT_FILES = ("pyproject.toml", "requirements.txt", "setup.py", "setup.cfg")
+
+
+def find_project_root(
+  start: Union[str, Path] = None,
+  verbose: bool = False,
+) -> str:
+  """Find the nearest project root by looking for common project files.
+
+  Walks up the directory tree from `start` until it finds a directory
+  containing pyproject.toml, requirements.txt, setup.py, or setup.cfg.
+
+  Args:
+      start: Starting directory or file path. Defaults to cwd.
+      verbose: If True, print search progress.
+
+  Returns:
+      String path to the project root directory, or cwd if not found.
+  """
+  if start is None:
+    start = Path.cwd()
+  else:
+    start = Path(start)
+
+  if start.is_file():
+    start = start.parent
+
+  if verbose:
+    print(f"Searching for project root from: {start}")
+
+  for parent in [start, *start.parents]:
+    if verbose:
+      print(f"  Checking: {parent}")
+    for filename in PROJECT_ROOT_FILES:
+      if (parent / filename).exists():
+        if verbose:
+          print(f"  Found: {parent / filename}")
+        return str(parent)
+
+  if verbose:
+    print(f"  No project root found, using cwd: {Path.cwd()}")
+  return str(Path.cwd())
+
+
+@proto.prefix
+class RUN:
+  """
+  Global Experiment Run Configuration.
+
+  This class is the single source of truth for experiment metadata.
+  Configure it before starting an experiment, or through the Experiment
+  constructor.
+
+  Default prefix template:
+      {project}/{now:%Y/%m-%d}/{path_stem}/{job_name}
+
+  Example:
+      # Set prefix via environment variable
+      # export ML_DASH_PREFIX="ge/myproject/exp1"
+
+      # Or configure directly
+      from ml_dash.run import RUN
+
+      RUN.project = "my-project"
+      RUN.prefix = "{username}/{project}/{now:%Y-%m-%d}/{entry}"
+
+  Auto-detection:
+      project_root is auto-detected by searching for pyproject.toml,
+      requirements.txt, setup.py, or setup.cfg in parent directories.
+  """
+
+  user: str = EnvVar @ "ML_DASH_USER" @ "USER"
+
+  api_url: str = EnvVar @ "ML_DASH_API_URL" | "https://api.dash.ml"
+  """Remote API server URL"""
+
+  ### Experiment and project information
+  project = "{user}/scratch"  # default project name
+
+  prefix: str = (
+    EnvVar @ "ML_DASH_PREFIX" | "{project}/{now:%Y/%m-%d}/{path_stem}/{job_name}"
+  )
+  """Full experiment path: {owner}/{project}/path.../[name]"""
+
+  readme = None
+
+  id: int = None
+  """Unique experiment ID (snowflake, auto-generated at run start)"""
+
+  now = datetime.now()
+  """Timestamp at import time. Does not change during the session."""
+
+  timestamp: str = None
+  """Timestamp created at instantiation"""
+
+  ### file properties
+  project_root: str = None
+  """Root directory for experiment hierarchy (for auto-detection)"""
+
+  entry: Union[Path, str] = None
+  """Entry point file/directory path"""
+
+  path_stem: str = None
+
+  job_counter: int = 1  # Default to 0. Use True to increment by 1.
+
+  job_name: str = "{now:%H.%M.%S}/{job_counter:03d}"
+
+  """ 
+      Default to '{now:%H.%M.%S}'. use '{now:%H.%M.%S}/{job_counter:03d}'
+      
+      for multiple launches. You can do so by setting:
+
+      ```python
+      RUN.job_name += "/{job_counter}"
+
+      for params in sweep:
+         thunk = instr(main)
+         jaynes.run(thun)
+      jaynes.listen()
+      ```
+  """
+
+  debug = "pydevd" in sys.modules
+  "set to True automatically for pyCharm"
+
+  def __post_init__(self):
+    """
+    Initialize RUN with auto-detected prefix from entry path.
+
+    Args:
+        entry: Path to entry file/directory (e.g., __file__ or directory
+               containing sweep.jsonl). If not provided, uses caller's
+               __file__ automatically.
+
+    Computes prefix as relative path from project_root to entry's directory.
+
+    Example:
+        # experiments/__init__.py
+        from ml_dash import RUN
+
+        RUN.project_root = "/path/to/my-project/experiments"
+
+        # experiments/vision/resnet/train.py
+        from ml_dash import RUN
+
+        RUN.__post_init__(entry=__file__)
+        # Result: RUN.prefix = "vision/resnet", RUN.name = "resnet"
+    """
+
+    # Use provided entry or try to auto-detect from caller
+    if self.entry is None:
+      import inspect
+
+      # Walk up the stack to find the actual caller (skip params_proto frames)
+      frame = inspect.currentframe().f_back
+      while frame:
+        file_path = frame.f_globals.get("__file__", "")
+        if "params_proto" not in file_path and "ml_dash/run.py" not in file_path:
+          break
+        frame = frame.f_back
+
+      self.entry = frame.f_globals.get("__file__") if frame else None
+
+    if not self.path_stem:
+
+      def stem(path):
+        return os.path.splitext(str(path))[0]
+
+      def truncate(path, depth):
+        return "/".join(str(path).split("/")[depth:])
+
+      self.project_root = str(self.project_root or find_project_root(self.entry))
+      script_root_depth = self.project_root.split("/").__len__()
+
+      script_truncated = truncate(os.path.abspath(self.entry), depth=script_root_depth)
+
+      self.path_stem = stem(script_truncated)
+
+    if isinstance(RUN.job_counter, int) or isinstance(RUN.job_counter, float):
+      RUN.job_counter += 1
+
+    while "{" in self.prefix:
+      data = vars(self)
+      for k, v in data.items():
+        if isinstance(v, str):
+          setattr(self, k, v.format(**data))
+
+    # for k, v in data.items():
+    #   print(f"> {k:>30}: {v}")
+
+
+if __name__ == "__main__":
+  RUN.description = ""
+  RUN.entry = __file__
+  RUN.prefix = "you you"
+
+  run = RUN()
+  print(vars(run))

ml_dash/snowflake.py

@@ -0,0 +1,173 @@
+"""
+Snowflake ID generator for ML-Dash.
+
+Snowflake IDs are 64-bit unique identifiers with the following structure:
+- 1 bit: unused (always 0)
+- 41 bits: timestamp in milliseconds since custom epoch
+- 10 bits: worker/machine ID (0-1023)
+- 12 bits: sequence number (0-4095)
+
+This provides:
+- Unique IDs across distributed systems
+- Time-sortable (newer IDs are larger)
+- ~69 years of IDs from custom epoch
+- Up to 4096 IDs per millisecond per worker
+"""
+
+import time
+import threading
+import os
+
+
+class SnowflakeIDGenerator:
+    """
+    Thread-safe Snowflake ID generator.
+
+    Based on Twitter's Snowflake algorithm.
+    """
+
+    # Custom epoch: 2024-01-01 00:00:00 UTC (in milliseconds)
+    EPOCH = 1704067200000
+
+    # Bit lengths
+    TIMESTAMP_BITS = 41
+    WORKER_BITS = 10
+    SEQUENCE_BITS = 12
+
+    # Max values
+    MAX_WORKER_ID = (1 << WORKER_BITS) - 1  # 1023
+    MAX_SEQUENCE = (1 << SEQUENCE_BITS) - 1  # 4095
+
+    # Bit shifts
+    TIMESTAMP_SHIFT = WORKER_BITS + SEQUENCE_BITS  # 22
+    WORKER_SHIFT = SEQUENCE_BITS  # 12
+
+    def __init__(self, worker_id: int = None):
+        """
+        Initialize Snowflake ID generator.
+
+        Args:
+            worker_id: Worker/machine ID (0-1023). If None, derived from process ID.
+        """
+        if worker_id is None:
+            # Derive from process ID
+            worker_id = os.getpid() & self.MAX_WORKER_ID
+
+        if not 0 <= worker_id <= self.MAX_WORKER_ID:
+            raise ValueError(f"worker_id must be between 0 and {self.MAX_WORKER_ID}")
+
+        self.worker_id = worker_id
+        self.sequence = 0
+        self.last_timestamp = -1
+        self.lock = threading.Lock()
+
+    def _current_millis(self) -> int:
+        """Get current timestamp in milliseconds since custom epoch."""
+        return int(time.time() * 1000) - self.EPOCH
+
+    def _wait_next_millis(self, last_timestamp: int) -> int:
+        """Wait until next millisecond."""
+        timestamp = self._current_millis()
+        while timestamp <= last_timestamp:
+            timestamp = self._current_millis()
+        return timestamp
+
+    def generate(self) -> int:
+        """
+        Generate a new Snowflake ID.
+
+        Returns:
+            A unique 64-bit integer ID
+
+        Raises:
+            RuntimeError: If clock moves backwards
+        """
+        with self.lock:
+            timestamp = self._current_millis()
+
+            # Check for clock moving backwards
+            if timestamp < self.last_timestamp:
+                raise RuntimeError(
+                    f"Clock moved backwards. Refusing to generate ID. "
+                    f"Last: {self.last_timestamp}, Current: {timestamp}"
+                )
+
+            if timestamp == self.last_timestamp:
+                # Same millisecond - increment sequence
+                self.sequence = (self.sequence + 1) & self.MAX_SEQUENCE
+                if self.sequence == 0:
+                    # Sequence overflow - wait for next millisecond
+                    timestamp = self._wait_next_millis(self.last_timestamp)
+            else:
+                # New millisecond - reset sequence
+                self.sequence = 0
+
+            self.last_timestamp = timestamp
+
+            # Construct the ID
+            snowflake_id = (
+                (timestamp << self.TIMESTAMP_SHIFT) |
+                (self.worker_id << self.WORKER_SHIFT) |
+                self.sequence
+            )
+
+            return snowflake_id
+
+    def parse(self, snowflake_id: int) -> dict:
+        """
+        Parse a Snowflake ID into its components.
+
+        Args:
+            snowflake_id: The Snowflake ID to parse
+
+        Returns:
+            Dictionary with timestamp, worker_id, and sequence
+        """
+        timestamp = (snowflake_id >> self.TIMESTAMP_SHIFT) + self.EPOCH
+        worker_id = (snowflake_id >> self.WORKER_SHIFT) & self.MAX_WORKER_ID
+        sequence = snowflake_id & self.MAX_SEQUENCE
+
+        return {
+            "timestamp": timestamp,
+            "timestamp_ms": timestamp,
+            "worker_id": worker_id,
+            "sequence": sequence,
+        }
+
+
+# Global singleton instance
+_generator = None
+_generator_lock = threading.Lock()
+
+
+def get_generator() -> SnowflakeIDGenerator:
+    """Get or create the global Snowflake ID generator instance."""
+    global _generator
+    if _generator is None:
+        with _generator_lock:
+            if _generator is None:
+                _generator = SnowflakeIDGenerator()
+    return _generator
+
+
+def generate_id() -> int:
+    """
+    Generate a new Snowflake ID using the global generator.
+
+    Returns:
+        A unique 64-bit integer ID
+    """
+    return get_generator().generate()
+
+
+def parse_id(snowflake_id: int) -> dict:
+    """
+    Parse a Snowflake ID into its components.
+
+    Args:
+        snowflake_id: The Snowflake ID to parse
+
+    Returns:
+        Dictionary with timestamp, worker_id, and sequence
+    """
+    return get_generator().parse(snowflake_id)