lumen-resources 0.2.0__py3-none-any.whl

lumen_resources/schemas/config-schema.yaml

@@ -0,0 +1,249 @@
+# Lumen Services Configuration Schema
+# Version: 1.0.0
+# Specification: https://json-schema.org/draft-07/schema
+
+$schema: "http://json-schema.org/draft-07/schema#"
+$id: "https://github.com/Lumilio-Photos/lumen-config-spec/v1.0.0"
+
+title: Lumen Config
+description: Unified configuration schema for all Lumen ML services
+
+type: object
+required:
+    - metadata
+    - deployment
+    - server
+    - services
+
+properties:
+    metadata:
+        type: object
+        required:
+            - version
+            - region
+            - cache_dir
+        properties:
+            version:
+                type: string
+                pattern: "^\\d+\\.\\d+\\.\\d+$"
+                description: Configuration version (semantic versioning)
+                examples:
+                    - "1.0.0"
+                    - "2.1.3"
+
+            region:
+                type: string
+                enum: ["cn", "other"]
+                description: Platform region selection (cn=ModelScope, other=HuggingFace)
+
+            cache_dir:
+                type: string
+                description: Model cache directory path (supports ~ expansion)
+                examples:
+                    - "~/.lumen/models"
+                    - "/opt/lumen/models"
+
+    deployment:
+        type: object
+        required:
+            - mode
+        properties:
+            mode:
+                type: string
+                enum: ["single", "hub"]
+                description: Deployment mode
+
+            service:
+                type: string
+                description: Service name for single mode (required if mode=single)
+                pattern: "^[a-z][a-z0-9_]*$"
+
+            services:
+                type: array
+                items:
+                    type: string
+                    pattern: "^[a-z][a-z0-9_]*$"
+                minItems: 1
+                uniqueItems: true
+                description: Service names for hub mode (required if mode=hub)
+
+        # Conditional validation based on mode
+        oneOf:
+            - required: ["service"]
+              properties:
+                  mode:
+                      const: "single"
+            - required: ["services"]
+              properties:
+                  mode:
+                      const: "hub"
+
+    server:
+        type: object
+        required:
+            - port
+        properties:
+            port:
+                type: integer
+                minimum: 1024
+                maximum: 65535
+                default: 50051
+                description: gRPC server port
+
+            host:
+                type: string
+                default: "0.0.0.0"
+                description: Server bind address
+                examples:
+                    - "0.0.0.0"
+                    - "127.0.0.1"
+                    - "[::]"
+
+            mdns:
+                type: object
+                properties:
+                    enabled:
+                        type: boolean
+                        default: false
+                        description: Enable mDNS service discovery
+
+                    service_name:
+                        type: string
+                        description: mDNS service name (required if enabled=true)
+                        pattern: "^[a-z][a-z0-9-]*$"
+                        examples:
+                            - "lumen-clip"
+                            - "lumen-hub"
+
+                # If enabled=true, service_name is required
+                if:
+                    properties:
+                        enabled:
+                            const: true
+                then:
+                    required: ["service_name"]
+
+    services:
+        type: object
+        minProperties: 1
+        description: Service definitions
+
+        patternProperties:
+            "^[a-z][a-z0-9_]*$":
+                type: object
+                required:
+                    - enabled
+                    - package
+                    - import
+                    - models
+
+                properties:
+                    enabled:
+                        type: boolean
+                        description: Whether to load this service
+
+                    package:
+                        type: string
+                        pattern: "^[a-z][a-z0-9_]*$"
+                        description: Python package name
+                        examples:
+                            - "lumen_clip"
+                            - "lumen_face"
+
+                    import:
+                        type: object
+                        required:
+                            - registry_class
+                            - add_to_server
+                        properties:
+                            registry_class:
+                                type: string
+                                pattern: "^[a-z_][a-z0-9_.]*\\.[A-Z][a-zA-Z0-9]*$"
+                                description: Full dotted path to service registry class
+                                examples:
+                                    - "lumen_clip.service_registry.ClipService"
+                                    - "lumen_face.service_registry.FaceService"
+
+                            add_to_server:
+                                type: string
+                                pattern: "^[a-z_][a-z0-9_.]*\\.add_[A-Za-z0-9_]+_to_server$"
+                                description: Full dotted path to gRPC add_to_server function
+                                examples:
+                                    - "lumen_clip.proto.ml_service_pb2_grpc.add_InferenceServicer_to_server"
+                                    - "lumen_face.proto.ml_service_pb2_grpc.add_FaceServicer_to_server"
+                    backend_settings:
+                        $ref: "#/definitions/BackendSettings"
+                    models:
+                        type: object
+                        minProperties: 1
+                        description: Model configurations (alias → config)
+
+                        patternProperties:
+                            "^[a-z][a-z0-9_]*$":
+                                $ref: "#/definitions/ModelConfig"
+
+definitions:
+    BackendSettings:
+        type: object
+        description: Optional settings for inference backend configuration.
+        properties:
+            device:
+                type: [string, "null"] # Allow string or null
+                description: "Preferred device ('cuda', 'mps', 'cpu'). If null, auto-detects best available."
+                default: null
+            batch_size:
+                type: integer
+                description: Maximum batch size for inference.
+                minimum: 1
+                default: 8
+            onnx_providers:
+                type: [array, "null"] # Allow array or null
+                items:
+                    type: string
+                description: "List of ONNX execution providers. If null, uses ONNX Runtime defaults."
+                default: null
+        additionalProperties: false
+    ModelConfig:
+        type: object
+        required:
+            - model
+            - runtime
+        properties:
+            model:
+                type: string
+                description: Model repository name
+                examples:
+                    - "ViT-B-32"
+                    - "CN-CLIP-ViT-B-16"
+                    - "MobileCLIP-S2"
+
+            runtime:
+                type: string
+                enum: ["torch", "onnx", "rknn"]
+                description: Model runtime type
+
+            rknn_device:
+                type: string
+                pattern: "^rk\\d+$"
+                description: RKNN device identifier (required if runtime=rknn)
+                examples:
+                    - "rk3566"
+                    - "rk3588"
+
+            dataset:
+                type: string
+                description: Dataset name for zero-shot classification (optional)
+                examples:
+                    - "ImageNet_1k"
+                    - "TreeOfLife-10M"
+
+        # If runtime=rknn, rknn_device is required
+        if:
+            properties:
+                runtime:
+                    const: "rknn"
+        then:
+            required: ["rknn_device"]
+
+# Additional validation rules
+additionalProperties: false

lumen_resources/schemas/model_info-schema.json

@@ -0,0 +1,166 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$id": "https://example.com/model-schema.json",
+    "title": "Model Info",
+    "description": "Schema for Lumen AI model configuration files",
+    "type": "object",
+    "required": [
+        "name",
+        "version",
+        "description",
+        "model_type",
+        "embedding_dim",
+        "source",
+        "runtimes"
+    ],
+    "properties": {
+        "name": {
+            "type": "string",
+            "description": "Model name identifier, this is also openclip model identifier if openclip is set as source format",
+            "minLength": 1,
+            "maxLength": 100
+        },
+        "version": {
+            "type": "string",
+            "description": "Model version",
+            "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
+        },
+        "description": {
+            "type": "string",
+            "description": "Model description and purpose",
+            "minLength": 1,
+            "maxLength": 500
+        },
+        "model_type": {
+            "type": "string",
+            "description": "Type of the model"
+        },
+        "embedding_dim": {
+            "type": "integer",
+            "description": "Dimension of the embedding space",
+            "minimum": 1,
+            "maximum": 100000
+        },
+        "source": {
+            "type": "object",
+            "required": ["format", "repo_id"],
+            "properties": {
+                "format": {
+                    "type": "string",
+                    "enum": ["huggingface", "openclip", "modelscope", "custom"]
+                },
+                "repo_id": {
+                    "type": "string",
+                    "description": "Repository identifier for model source",
+                    "minLength": 1
+                }
+            },
+            "additionalProperties": false
+        },
+        "runtimes": {
+            "type": "object",
+            "minProperties": 1,
+            "patternProperties": {
+                "^[a-zA-Z0-9_-]+$": {
+                    "type": "object",
+                    "required": ["available"],
+                    "properties": {
+                        "available": {
+                            "type": "boolean"
+                        },
+                        "files": {
+                            "oneOf": [
+                                {
+                                    "type": "array",
+                                    "items": {
+                                        "type": "string"
+                                    }
+                                },
+                                {
+                                    "type": "object",
+                                    "patternProperties": {
+                                        "^[a-zA-Z0-9_-]+$": {
+                                            "type": "array",
+                                            "items": {
+                                                "type": "string"
+                                            }
+                                        }
+                                    },
+                                    "additionalProperties": false
+                                }
+                            ]
+                        },
+                        "devices": {
+                            "type": "array",
+                            "items": {
+                                "type": "string"
+                            }
+                        },
+                        "requirements": {
+                            "type": "object",
+                            "properties": {
+                                "python": {
+                                    "type": "string"
+                                },
+                                "dependencies": {
+                                    "type": "array",
+                                    "items": {
+                                        "type": "string"
+                                    }
+                                }
+                            }
+                        }
+                    },
+                    "additionalProperties": false
+                }
+            },
+            "additionalProperties": false
+        },
+        "datasets": {
+            "type": "object",
+            "patternProperties": {
+                "^[a-zA-Z0-9_-]+$": {
+                    "type": "object",
+                    "required": ["labels", "embeddings"],
+                    "properties": {
+                        "labels": {
+                            "type": "string"
+                        },
+                        "embeddings": {
+                            "type": "string"
+                        }
+                    },
+                    "additionalProperties": false
+                }
+            },
+            "additionalProperties": false
+        },
+        "metadata": {
+            "type": "object",
+            "properties": {
+                "license": {
+                    "type": "string"
+                },
+                "author": {
+                    "type": "string"
+                },
+                "created_at": {
+                    "type": "string",
+                    "format": "date"
+                },
+                "updated_at": {
+                    "type": "string",
+                    "format": "date-time"
+                },
+                "tags": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                }
+            },
+            "additionalProperties": false
+        }
+    },
+    "additionalProperties": false
+}

lumen_resources/schemas/result_schemas/embedding_v1.json

@@ -0,0 +1,35 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "EmbeddingV1",
+  "description": "Universal schema for embedding responses across all Lumen services (face, clip, ocr, etc.)",
+  "type": "object",
+  "properties": {
+    "vector": {
+      "type": "array",
+      "items": {
+        "type": "number"
+      },
+      "minItems": 1,
+      "description": "Embedding vector"
+    },
+    "dim": {
+      "type": "integer",
+      "minimum": 1,
+      "description": "Embedding dimension"
+    },
+    "model_id": {
+      "type": "string",
+      "minLength": 1,
+      "description": "Model identifier that generated the embedding"
+    }
+  },
+  "required": ["vector", "dim", "model_id"],
+  "additionalProperties": false,
+  "examples": [
+    {
+      "vector": [0.1, 0.2, 0.3],
+      "dim": 3,
+      "model_id": "embedding_model_1"
+    }
+  ]
+}

lumen_resources/schemas/result_schemas/face_v1.json

@@ -0,0 +1,61 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "FaceV1",
+  "description": "Universal schema for face detection and embedding responses across Lumen services",
+  "type": "object",
+  "properties": {
+    "faces": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "bbox": {
+            "type": "array",
+            "items": {
+              "type": "number",
+              "minimum": 0
+            },
+            "minItems": 4,
+            "maxItems": 4,
+            "description": "Bounding box coordinates [x1, y1, x2, y2] where (x1, y1) is top-left corner and (x2, y2) is bottom-right corner"
+          },
+          "confidence": {
+            "type": "number",
+            "minimum": 0.0,
+            "maximum": 1.0,
+            "description": "Detection confidence score"
+          },
+          "landmarks": {
+            "type": "array",
+            "items": {
+              "type": "number"
+            },
+            "description": "Facial landmark points (optional)"
+          },
+          "embedding": {
+            "type": "array",
+            "items": {
+              "type": "number"
+            },
+            "description": "Face embedding vector for recognition/comparison (optional)"
+          }
+        },
+        "required": ["bbox", "confidence"],
+        "additionalProperties": false
+      },
+      "description": "Detected faces with bounding boxes and metadata"
+    },
+    "count": {
+      "type": "integer",
+      "minimum": 0,
+      "description": "Number of detected faces"
+    },
+    "model_id": {
+      "type": "string",
+      "minLength": 1,
+      "description": "Model identifier"
+    }
+  },
+  "required": ["faces", "count", "model_id"],
+  "additionalProperties": false
+}

lumen_resources/schemas/result_schemas/labels_v1.json

@@ -0,0 +1,49 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "LabelsV1",
+  "description": "Classification response schema for Lumen services. Returns ranked labels with confidence scores.",
+  "type": "object",
+  "properties": {
+    "labels": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "label": {
+            "type": "string",
+            "description": "The classification label or class name"
+          },
+          "score": {
+            "type": "number",
+            "description": "Confidence score for this label"
+          }
+        },
+        "required": ["label", "score"],
+        "additionalProperties": false
+      },
+      "description": "Array of classification results"
+    },
+    "model_id": {
+      "type": "string",
+      "minLength": 1,
+      "description": "Identifier of the model that generated the classification"
+    }
+  },
+  "required": ["labels", "model_id"],
+  "additionalProperties": false,
+  "examples": [
+    {
+      "labels": [
+        {
+          "label": "cat",
+          "score": 0.9
+        },
+        {
+          "label": "dog",
+          "score": 0.1
+        }
+      ],
+      "model_id": "embedding_model_1"
+    }
+  ]
+}

lumen_resources-0.2.0.dist-info/METADATA

@@ -0,0 +1,133 @@
+Metadata-Version: 2.4
+Name: lumen-resources
+Version: 0.2.0
+Summary: Unified model resource management for Lumen ML services
+Author-email: EdwinZhanCN <support@lumilio.org>
+License: MIT
+Project-URL: Homepage, https://github.com/EdwinZhanCN/Lumen
+Project-URL: Issues, https://github.com/EdwinZhanCN/Lumen/issues
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: huggingface_hub>=0.20.0
+Requires-Dist: modelscope>=1.11.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: jsonschema>=4.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: datamodel-code-generator[http]>=0.25.0; extra == "dev"
+Requires-Dist: pre-commit>=3.0.0; extra == "dev"
+Provides-Extra: config
+Requires-Dist: pyyaml>=6.0.0; extra == "config"
+
+# Lumen Resources
+
+Lightweight tooling for shipping Lumen ML services. This package centralizes how models are described, validated, downloaded, and cached so every service (CLIP, face, etc.) follows the same playbook—whether weights live on Hugging Face, ModelScope, or a private registry.
+
+## Why use it?
+
+- **Single source of truth** – YAML configs describing deployments, devices, runtimes, and model aliases.
+- **Schema-backed validation** – JSON Schema plus Pydantic to catch errors before runtime.
+- **Cross-platform downloads** – Intelligent routing between Hugging Face and ModelScope with caching/resume support.
+- **CLI + Python API** – Automate in CI or embed in service bootstraps.
+- **Result schemas** – Typed response validators (`EmbeddingV1`, `FaceV1`, `LabelsV1`) for downstream services.
+
+## Installation
+
+```bash
+# project install
+pip install "lumen-resources @ git+https://github.com/EdwinZhanCN/Lumen.git@main#subdirectory=lumen-resources"
+
+# dev install
+git clone https://github.com/EdwinZhanCN/Lumen.git
+cd Lumen/lumen-resources
+pip install -e ".[dev,config]"
+```
+
+Optional extras depending on your targets:
+
+```bash
+pip install huggingface_hub
+pip install modelscope
+pip install torch torchvision
+pip install onnxruntime
+```
+
+## Usage
+
+### CLI
+
+```bash
+# download everything defined in config.yaml
+lumen-resources download config.yaml
+
+# strict config validation
+lumen-resources validate config.yaml
+
+# validate a model_info.json
+lumen-resources validate-model-info path/to/model_info.json
+
+# inspect cache contents (defaults to ~/.lumen/)
+lumen-resources list ~/.lumen/
+```
+
+### Python API
+
+```python
+from lumen_resources import (
+    load_and_validate_config,
+    Downloader,
+    load_and_validate_model_info,
+    EmbeddingV1,
+)
+
+config = load_and_validate_config("config.yaml")
+downloader = Downloader(config, verbose=True)
+results = downloader.download_all(force=False)
+
+model_info = load_and_validate_model_info("model_info.json")
+```
+
+## Configuration essentials
+
+```yaml
+metadata:
+  region: "other"      # or "cn" to prefer ModelScope
+  cache_dir: "~/.lumen/models"
+
+deployment:
+  mode: "single"       # or "hub"
+  service: "clip"
+
+services:
+  clip:
+    enabled: true
+    package: "lumen_clip"
+    backend_settings:
+      device: "cuda"
+      batch_size: 16
+      onnx_providers: ["CUDAExecutionProvider", "CPUExecutionProvider"]
+    models:
+      default:
+        model: "ViT-B-32"
+        runtime: "torch"
+      fp16:
+        model: "ViT-B-32"
+        runtime: "onnx"
+```
+
+- `metadata.region` decides whether downloads prefer ModelScope or Hugging Face.
+- `backend_settings` lets you declare execution providers, batch sizes, devices, etc.
+- Each entry in `models` becomes a cache namespace (`clip/default`, `clip/fp16`, …).
+
+## Reference
+
+- Source: `src/lumen_resources/`
+  - `lumen_config.py` – Typed config models
+  - `downloader.py` – Platform abstraction + caching
+  - `cli.py` – Command entrypoint
+  - `result_schemas/` – Response validators
+- Docs: https://doc.lumilio.org
+- Issues & support: open a ticket in the main Lumen monorepo.