clarifai 11.3.0rc2__py3-none-any.whl → 11.4.0__py3-none-any.whl

clarifai/utils/cli.py

@@ -12,157 +12,163 @@ from tabulate import tabulate
 
 
 def from_yaml(filename: str):
-  try:
-    with open(filename, 'r') as f:
-      return yaml.safe_load(f)
-  except yaml.YAMLError as e:
-    click.echo(f"Error reading YAML file: {e}", err=True)
-    return {}
+    try:
+        with open(filename, 'r') as f:
+            return yaml.safe_load(f)
+    except yaml.YAMLError as e:
+        click.echo(f"Error reading YAML file: {e}", err=True)
+        return {}
 
 
 def dump_yaml(data, filename: str):
-  try:
-    with open(filename, 'w') as f:
-      yaml.dump(data, f)
-  except Exception as e:
-    click.echo(f"Error writing YAML file: {e}", err=True)
+    try:
+        with open(filename, 'w') as f:
+            yaml.dump(data, f)
+    except Exception as e:
+        click.echo(f"Error writing YAML file: {e}", err=True)
 
 
 # Dynamically find and import all command modules from the cli directory
 def load_command_modules():
-  package_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'cli')
+    package_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'cli')
 
-  for _, module_name, _ in pkgutil.iter_modules([package_dir]):
-    if module_name not in ['base', '__main__']:  # Skip the base.py and __main__ file itself
-      importlib.import_module(f'clarifai.cli.{module_name}')
+    for _, module_name, _ in pkgutil.iter_modules([package_dir]):
+        if module_name not in ['base', '__main__']:  # Skip the base.py and __main__ file itself
+            importlib.import_module(f'clarifai.cli.{module_name}')
 
 
-def display_co_resources(response,
-                         custom_columns={
-                             'ID': lambda c: c.id,
-                             'USER_ID': lambda c: c.user_id,
-                             'DESCRIPTION': lambda c: c.description,
-                         }):
-  """Display compute orchestration resources listing results using rich."""
+def display_co_resources(
+    response,
+    custom_columns={
+        'ID': lambda c: c.id,
+        'USER_ID': lambda c: c.user_id,
+        'DESCRIPTION': lambda c: c.description,
+    },
+):
+    """Display compute orchestration resources listing results using rich."""
 
-  formatter = TableFormatter(custom_columns)
-  print(formatter.format(list(response), fmt="plain"))
+    formatter = TableFormatter(custom_columns)
+    print(formatter.format(list(response), fmt="plain"))
 
 
 class TableFormatter:
-
-  def __init__(self, custom_columns: OrderedDict):
-    """
+    def __init__(self, custom_columns: OrderedDict):
+        """
         Initializes the TableFormatter with column headers and custom column mappings.
 
         :param headers: List of column headers for the table.
         """
-    self.custom_columns = custom_columns
+        self.custom_columns = custom_columns
 
-  def format(self, objects, fmt='plain'):
-    """
+    def format(self, objects, fmt='plain'):
+        """
         Formats a list of objects into a table with custom columns.
 
         :param objects: List of objects to format into a table.
         :return: A string representing the table.
         """
-    # Prepare the rows by applying the custom column functions to each object
-    rows = []
-    for obj in objects:
-      #   row = [self.custom_columns[header](obj) for header in self.headers]
-      row = [f(obj) for f in self.custom_columns.values()]
-      rows.append(row)
+        # Prepare the rows by applying the custom column functions to each object
+        rows = []
+        for obj in objects:
+            #   row = [self.custom_columns[header](obj) for header in self.headers]
+            row = [f(obj) for f in self.custom_columns.values()]
+            rows.append(row)
 
-    # Create the table
-    table = tabulate(rows, headers=self.custom_columns.keys(), tablefmt=fmt)
-    return table
+        # Create the table
+        table = tabulate(rows, headers=self.custom_columns.keys(), tablefmt=fmt)
+        return table
 
 
 class AliasedGroup(click.Group):
-
-  def __init__(self,
-               name: t.Optional[str] = None,
-               commands: t.Optional[t.Union[t.MutableMapping[str, click.Command], t.Sequence[
-                   click.Command]]] = None,
-               **attrs: t.Any) -> None:
-    super().__init__(name, commands, **attrs)
-    self.alias_map = {}
-    self.command_to_aliases = defaultdict(list)
-
-  def add_alias(self, cmd: click.Command, alias: str) -> None:
-    self.alias_map[alias] = cmd
-    if alias != cmd.name:
-      self.command_to_aliases[cmd].append(alias)
-
-  def command(self, aliases=None, *args,
-              **kwargs) -> t.Callable[[t.Callable[..., t.Any]], click.Command]:
-    cmd_decorator = super().command(*args, **kwargs)
-    if aliases is None:
-      aliases = []
-
-    def aliased_decorator(f):
-      cmd = cmd_decorator(f)
-      if cmd.name:
-        self.add_alias(cmd, cmd.name)
-      for alias in aliases:
-        self.add_alias(cmd, alias)
-      return cmd
-
-    f = None
-    if args and callable(args[0]):
-      (f,) = args
-    if f is not None:
-      return aliased_decorator(f)
-    return aliased_decorator
-
-  def group(self, aliases=None, *args,
-            **kwargs) -> t.Callable[[t.Callable[..., t.Any]], click.Group]:
-    cmd_decorator = super().group(*args, **kwargs)
-    if aliases is None:
-      aliases = []
-
-    def aliased_decorator(f):
-      cmd = cmd_decorator(f)
-      if cmd.name:
-        self.add_alias(cmd, cmd.name)
-      for alias in aliases:
-        self.add_alias(cmd, alias)
-      return cmd
-
-    f = None
-    if args and callable(args[0]):
-      (f,) = args
-    if f is not None:
-      return aliased_decorator(f)
-    return aliased_decorator
-
-  def get_command(self, ctx: click.Context, cmd_name: str) -> t.Optional[click.Command]:
-    rv = click.Group.get_command(self, ctx, cmd_name)
-    if rv is not None:
-      return rv
-    return self.alias_map.get(cmd_name)
-
-  def format_commands(self, ctx, formatter):
-    sub_commands = self.list_commands(ctx)
-
-    rows = []
-    for sub_command in sub_commands:
-      cmd = self.get_command(ctx, sub_command)
-      if cmd is None or getattr(cmd, 'hidden', False):
-        continue
-      if cmd in self.command_to_aliases:
-        aliases = ', '.join(self.command_to_aliases[cmd])
-        sub_command = f'{sub_command} ({aliases})'
-      cmd_help = cmd.help
-      rows.append((sub_command, cmd_help))
-
-    if rows:
-      with formatter.section("Commands"):
-        formatter.write_dl(rows)
+    def __init__(
+        self,
+        name: t.Optional[str] = None,
+        commands: t.Optional[
+            t.Union[t.MutableMapping[str, click.Command], t.Sequence[click.Command]]
+        ] = None,
+        **attrs: t.Any,
+    ) -> None:
+        super().__init__(name, commands, **attrs)
+        self.alias_map = {}
+        self.command_to_aliases = defaultdict(list)
+
+    def add_alias(self, cmd: click.Command, alias: str) -> None:
+        self.alias_map[alias] = cmd
+        if alias != cmd.name:
+            self.command_to_aliases[cmd].append(alias)
+
+    def command(
+        self, aliases=None, *args, **kwargs
+    ) -> t.Callable[[t.Callable[..., t.Any]], click.Command]:
+        cmd_decorator = super().command(*args, **kwargs)
+        if aliases is None:
+            aliases = []
+
+        def aliased_decorator(f):
+            cmd = cmd_decorator(f)
+            if cmd.name:
+                self.add_alias(cmd, cmd.name)
+            for alias in aliases:
+                self.add_alias(cmd, alias)
+            return cmd
+
+        f = None
+        if args and callable(args[0]):
+            (f,) = args
+        if f is not None:
+            return aliased_decorator(f)
+        return aliased_decorator
+
+    def group(
+        self, aliases=None, *args, **kwargs
+    ) -> t.Callable[[t.Callable[..., t.Any]], click.Group]:
+        cmd_decorator = super().group(*args, **kwargs)
+        if aliases is None:
+            aliases = []
+
+        def aliased_decorator(f):
+            cmd = cmd_decorator(f)
+            if cmd.name:
+                self.add_alias(cmd, cmd.name)
+            for alias in aliases:
+                self.add_alias(cmd, alias)
+            return cmd
+
+        f = None
+        if args and callable(args[0]):
+            (f,) = args
+        if f is not None:
+            return aliased_decorator(f)
+        return aliased_decorator
+
+    def get_command(self, ctx: click.Context, cmd_name: str) -> t.Optional[click.Command]:
+        rv = click.Group.get_command(self, ctx, cmd_name)
+        if rv is not None:
+            return rv
+        return self.alias_map.get(cmd_name)
+
+    def format_commands(self, ctx, formatter):
+        sub_commands = self.list_commands(ctx)
+
+        rows = []
+        for sub_command in sub_commands:
+            cmd = self.get_command(ctx, sub_command)
+            if cmd is None or getattr(cmd, 'hidden', False):
+                continue
+            if cmd in self.command_to_aliases:
+                aliases = ', '.join(self.command_to_aliases[cmd])
+                sub_command = f'{sub_command} ({aliases})'
+            cmd_help = cmd.help
+            rows.append((sub_command, cmd_help))
+
+        if rows:
+            with formatter.section("Commands"):
+                formatter.write_dl(rows)
 
 
 def validate_context(ctx):
-  from clarifai.utils.logging import logger
-  if ctx.obj == {}:
-    logger.error("CLI config file missing. Run `clarifai login` to set up the CLI config.")
-    sys.exit(1)
+    from clarifai.utils.logging import logger
+
+    if ctx.obj == {}:
+        logger.error("CLI config file missing. Run `clarifai login` to set up the CLI config.")
+        sys.exit(1)

clarifai/utils/config.py

@@ -8,98 +8,138 @@ from clarifai.utils.constants import DEFAULT_CONFIG
 
 
 class Context(OrderedDict):
-  """
+    """
     A context which has a name and a set of key-values as a dict under env.
 
     You can access the keys directly.
     """
 
-  def __init__(self, name, **kwargs):
-    self['name'] = name
-    # when loading from config we may have the env: section in yaml already so we get it here.
-    if 'env' in kwargs:
-      self['env'] = kwargs['env']
-    else:  # when consructing as Context(name, key=value) we set it here.
-      self['env'] = kwargs
-
-  def __getattr__(self, key):
-    try:
-      if key == 'name':
-        return self[key]
-      if key == 'env':
-        raise AttributeError("Don't access .env directly")
-
-      # Allow accessing CLARIFAI_PAT type env var names from config as .pat
-      envvar_name = 'CLARIFAI_' + key.upper()
-      env = self['env']
-      if envvar_name in env:
-        value = env[envvar_name]
-        if value == "ENVVAR":
-          return os.environ[envvar_name]
-      else:
-        value = env[key]
-
-      if isinstance(value, dict):
-        return Context(value)
-
-      return value
-    except KeyError as e:
-      raise AttributeError(f"'{type(self).__name__}' object has no attribute '{key}'") from e
-
-  def __setattr__(self, key, value):
-    if key == "name":
-      self['name'] = value
-    else:
-      self['env'][key] = value
-
-  def __delattr__(self, key):
-    try:
-      del self['env'][key]
-    except KeyError as e:
-      raise AttributeError(f"'{type(self).__name__}' object has no attribute '{key}'") from e
-
-  def to_serializable_dict(self):
-    return dict(self['env'])
+    def __init__(self, name, **kwargs):
+        self['name'] = name
+        # when loading from config we may have the env: section in yaml already so we get it here.
+        if 'env' in kwargs:
+            self['env'] = kwargs['env']
+        else:  # when consructing as Context(name, key=value) we set it here.
+            self['env'] = kwargs
+
+    def __getattr__(self, key):
+        try:
+            if key == 'name':
+                return self[key]
+            if key == 'env':
+                raise AttributeError("Don't access .env directly")
+
+            # Allow accessing CLARIFAI_PAT type env var names from config as .pat
+            envvar_name = 'CLARIFAI_' + key.upper()
+            env = self['env']
+            if envvar_name in env:
+                value = env[envvar_name]
+                if value == "ENVVAR":
+                    if envvar_name not in os.environ:
+                        raise AttributeError(
+                            f"Environment variable '{envvar_name}' not set. Attempting to load it for config '{self['name']}'. Please set it in your terminal."
+                        )
+                    return os.environ[envvar_name]
+            else:
+                value = env[key]
+
+            if isinstance(value, dict):
+                return Context(value)
+
+            return value
+        except KeyError as e:
+            raise AttributeError(f"'{type(self).__name__}' object has no attribute '{key}'") from e
+
+    def __hasattr__(self, key):
+        if key == "name":
+            return True
+        else:
+            envvar_name = 'CLARIFAI_' + key.upper()
+            return envvar_name in self['env'] or key in self['env']
+
+    def __setattr__(self, key, value):
+        if key == "name":
+            self['name'] = value
+        else:
+            self['env'][key] = value
+
+    def __delattr__(self, key):
+        try:
+            del self['env'][key]
+        except KeyError as e:
+            raise AttributeError(f"'{type(self).__name__}' object has no attribute '{key}'") from e
+
+    def to_column_names(self):
+        """used for displaying on terminal."""
+        keys = []
+        for k in self['env'].keys():
+            if k.startswith("CLARIFAI_"):
+                keys.append(k.replace("CLARIFAI_", "", 1))
+        return keys
+
+    def to_stripped_lowercase(self):
+        dict(self['env'])
+
+    def to_serializable_dict(self):
+        return dict(self['env'])
+
+    def set_to_env(self):
+        """sets the context env vars to the current os.environ
+
+        Example:
+          # This is helpful in scripts so you can do
+
+          from clarifai.utils.config import Config
+
+          Config.from_yaml().current.set_to_env()
+
+        """
+        for k, v in self['env'].items():
+            if isinstance(v, dict):
+                continue
+            envvar_name = k.upper()
+            if not envvar_name.startswith('CLARIFAI_'):
+                envvar_name = 'CLARIFAI_' + envvar_name
+            os.environ[envvar_name] = str(v)
 
 
 @dataclass
-class Config():
-  current_context: str
-  filename: str
-  contexts: OrderedDict[str, Context] = field(default_factory=OrderedDict)
-
-  def __post_init__(self):
-    for k, v in self.contexts.items():
-      if 'name' not in v:
-        v['name'] = k
-    self.contexts = {k: Context(**v) for k, v in self.contexts.items()}
-
-  @classmethod
-  def from_yaml(cls, filename: str = DEFAULT_CONFIG):
-    with open(filename, 'r') as f:
-      cfg = yaml.safe_load(f)
-    return cls(**cfg, filename=filename)
-
-  def to_dict(self):
-    return {
-        'current_context': self.current_context,
-        'contexts': {k: v.to_serializable_dict()
-                     for k, v in self.contexts.items()}
-    }
-
-  def to_yaml(self, filename: str = None):
-    if filename is None:
-      filename = self.filename
-    dir = os.path.dirname(filename)
-    if len(dir):
-      os.makedirs(dir, exist_ok=True)
-    _dict = self.to_dict()
-    for k, v in _dict['contexts'].items():
-      v.pop('name', None)
-    with open(filename, 'w') as f:
-      yaml.safe_dump(_dict, f)
-
-  @property
-  def current(self) -> Context:
-    """ get the current Context """
-    return self.contexts[self.current_context]
+class Config:
+    current_context: str
+    filename: str
+    contexts: OrderedDict[str, Context] = field(default_factory=OrderedDict)
+
+    def __post_init__(self):
+        for k, v in self.contexts.items():
+            if 'name' not in v:
+                v['name'] = k
+        self.contexts = {k: Context(**v) for k, v in self.contexts.items()}
+
+    @classmethod
+    def from_yaml(cls, filename: str = DEFAULT_CONFIG):
+        with open(filename, 'r') as f:
+            cfg = yaml.safe_load(f)
+        return cls(**cfg, filename=filename)
+
+    def to_dict(self):
+        return {
+            'current_context': self.current_context,
+            'contexts': {k: v.to_serializable_dict() for k, v in self.contexts.items()},
+        }
+
+    def to_yaml(self, filename: str = None):
+        if filename is None:
+            filename = self.filename
+        dir = os.path.dirname(filename)
+        if len(dir):
+            os.makedirs(dir, exist_ok=True)
+        _dict = self.to_dict()
+        for k, v in _dict['contexts'].items():
+            v.pop('name', None)
+        with open(filename, 'w') as f:
+            yaml.safe_dump(_dict, f)
+
+    @property
+    def current(self) -> Context:
+        """get the current Context"""
+        return self.contexts[self.current_context]

clarifai/utils/constants.py

@@ -5,3 +5,52 @@ CLARIFAI_SESSION_TOKEN_ENV_VAR = "CLARIFAI_SESSION_TOKEN"
 CLARIFAI_USER_ID_ENV_VAR = "CLARIFAI_USER_ID"
 
 DEFAULT_CONFIG = f'{os.environ["HOME"]}/.config/clarifai/config'
+
+# Default clusters, etc. for local dev runner easy setup
+DEFAULT_LOCAL_DEV_COMPUTE_CLUSTER_ID = "local-dev-compute-cluster"
+DEFAULT_LOCAL_DEV_NODEPOOL_ID = "local-dev-nodepool"
+DEFAULT_LOCAL_DEV_DEPLOYMENT_ID = "local-dev-deployment"
+DEFAULT_LOCAL_DEV_MODEL_ID = "local-dev-model"
+DEFAULT_LOCAL_DEV_APP_ID = "local-dev-runner-app"
+
+# FIXME: should have any-to-any for these cases.
+DEFAULT_LOCAL_DEV_MODEL_TYPE = "text-to-text"
+
+DEFAULT_LOCAL_DEV_COMPUTE_CLUSTER_CONFIG = {
+    "compute_cluster": {
+        "id": DEFAULT_LOCAL_DEV_COMPUTE_CLUSTER_ID,
+        "description": "Default Local Dev Compute Cluster",
+        "cloud_provider": {
+            "id": "local",
+        },
+        "region": "na",
+        "managed_by": "user",
+        "cluster_type": "local-dev",
+    }
+}
+
+DEFAULT_LOCAL_DEV_NODEPOOL_CONFIG = {
+    "nodepool": {
+        "id": DEFAULT_LOCAL_DEV_NODEPOOL_ID,
+        "description": "Default Local Dev Nodepool",
+        "compute_cluster": {
+            "id": DEFAULT_LOCAL_DEV_COMPUTE_CLUSTER_ID,
+            "user_id": None,  # This will be set when creating the compute cluster
+        },
+        "instance_types": [
+            {
+                "id": "local-cpu",
+                "compute_info": {
+                    "cpu_limit": str(os.cpu_count()),
+                    "cpu_memory": "16Gi",  # made up as we don't schedule based on this for local dev.
+                    "num_accelerators": 0,  # TODO if we need accelerator detection for local dev.
+                },
+            }
+        ],
+        "node_capacity_type": {
+            "capacity_types": [1],
+        },
+        "min_instances": 1,
+        "max_instances": 1,
+    }
+}