openforma 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- forma_cli/__init__.py +5 -0
- forma_cli/app.py +1139 -0
- forma_cli/config.py +112 -0
- forma_cli/credentials.py +141 -0
- forma_cli/local.py +562 -0
- forma_cli/metadata_api.py +138 -0
- forma_cli/project_artifacts.py +182 -0
- forma_cli/sdk.py +571 -0
- forma_core/__init__.py +9 -0
- forma_core/__main__.py +4 -0
- forma_core/_version.py +3 -0
- forma_core/agents/__init__.py +50 -0
- forma_core/agents/clarification.py +194 -0
- forma_core/agents/component_reconciliation.py +99 -0
- forma_core/agents/context_gathering.py +115 -0
- forma_core/agents/continuous.py +537 -0
- forma_core/agents/contracts.py +247 -0
- forma_core/agents/firecrawl_mcp.py +484 -0
- forma_core/agents/lattice.py +239 -0
- forma_core/agents/orchestrator.py +3724 -0
- forma_core/agents/pipeline.py +843 -0
- forma_core/agents/project_correction.py +83 -0
- forma_core/agents/prompt_compaction.py +428 -0
- forma_core/agents/system_architecture.py +345 -0
- forma_core/agents/video_correction.py +119 -0
- forma_core/agents/web_research_workflow.py +1256 -0
- forma_core/agents/workflows.py +176 -0
- forma_core/assets/__init__.py +1 -0
- forma_core/assets/models.py +156 -0
- forma_core/assets/project.py +17 -0
- forma_core/assets/repository.py +48 -0
- forma_core/assets/service.py +142 -0
- forma_core/assets/validation.py +35 -0
- forma_core/benchmarks/__init__.py +5 -0
- forma_core/benchmarks/ir_triad_v1.py +477 -0
- forma_core/biology/__init__.py +28 -0
- forma_core/biology/benchmark.py +80 -0
- forma_core/biology/ewart_liverchip.py +279 -0
- forma_core/biology/ir.py +25 -0
- forma_core/biology/liverbench.py +416 -0
- forma_core/biology/liverbench_assay.py +251 -0
- forma_core/biology/liverbench_baselines.py +133 -0
- forma_core/biology/liverbench_independent.py +229 -0
- forma_core/biology/liverchip_demo.py +154 -0
- forma_core/biology/liverchip_evidence.py +149 -0
- forma_core/biology/models.py +247 -0
- forma_core/biology/validation.py +72 -0
- forma_core/cad_migrations/__init__.py +6 -0
- forma_core/cad_migrations/cli.py +101 -0
- forma_core/cad_migrations/emitters.py +227 -0
- forma_core/cad_migrations/models.py +121 -0
- forma_core/cad_migrations/planner.py +68 -0
- forma_core/chemistry/__init__.py +5 -0
- forma_core/chemistry/models.py +215 -0
- forma_core/chemistry/stoichiometry.py +272 -0
- forma_core/chemistry/validation.py +179 -0
- forma_core/cli/__init__.py +1 -0
- forma_core/cli/__main__.py +4 -0
- forma_core/cli/main.py +370 -0
- forma_core/config/__init__.py +28 -0
- forma_core/config/compatibility.py +282 -0
- forma_core/config/contract.py +184 -0
- forma_core/config/environment.py +129 -0
- forma_core/config/parti.py +34 -0
- forma_core/config/runtime.py +303 -0
- forma_core/control/__init__.py +10 -0
- forma_core/control/actions.py +136 -0
- forma_core/control/adapters.py +36 -0
- forma_core/control/cli.py +49 -0
- forma_core/control/client.py +124 -0
- forma_core/control/models.py +168 -0
- forma_core/control/opencode_worker.py +80 -0
- forma_core/control/package.py +318 -0
- forma_core/control/rover_scene/__init__.py +1 -0
- forma_core/control/rover_scene/__main__.py +34 -0
- forma_core/control/rover_scene/assets.py +65 -0
- forma_core/control/rover_scene/fleet.py +214 -0
- forma_core/control/rover_scene/host.py +258 -0
- forma_core/control/rover_scene/kinematics.py +35 -0
- forma_core/control/rover_scene/machine.py +347 -0
- forma_core/control/rover_scene/robot.py +166 -0
- forma_core/control/rover_scene/scene.py +206 -0
- forma_core/control/rover_scene/view.py +80 -0
- forma_core/control/rover_scene/web/index.html +25 -0
- forma_core/control/rover_scene/web/viewer.js +148 -0
- forma_core/control/runtime.py +356 -0
- forma_core/control/so101.py +235 -0
- forma_core/control/tasks.py +287 -0
- forma_core/database.py +2362 -0
- forma_core/debug.py +314 -0
- forma_core/domains/__init__.py +7 -0
- forma_core/domains/agents.py +85 -0
- forma_core/domains/base.py +69 -0
- forma_core/domains/benchmark.py +87 -0
- forma_core/domains/cli.py +48 -0
- forma_core/domains/compatibility.py +234 -0
- forma_core/domains/composition.py +372 -0
- forma_core/domains/diagnostics.py +15 -0
- forma_core/domains/generation.py +479 -0
- forma_core/domains/interaction.py +301 -0
- forma_core/domains/planning.py +140 -0
- forma_core/domains/providers.py +214 -0
- forma_core/domains/registry.py +267 -0
- forma_core/domains/selection.py +75 -0
- forma_core/domains/storage.py +60 -0
- forma_core/domains/tools.py +180 -0
- forma_core/electronics/__init__.py +1 -0
- forma_core/electronics/models.py +84 -0
- forma_core/electronics/primitives.py +43 -0
- forma_core/electronics/rules.py +340 -0
- forma_core/external_sources.py +497 -0
- forma_core/fabricator/__init__.py +33 -0
- forma_core/fabricator/__main__.py +7 -0
- forma_core/fabricator/main.py +578 -0
- forma_core/firmware/__init__.py +60 -0
- forma_core/firmware/drivers.py +140 -0
- forma_core/firmware/examples.py +503 -0
- forma_core/firmware/generate.py +520 -0
- forma_core/firmware/models.py +34 -0
- forma_core/firmware/sil/__init__.py +5 -0
- forma_core/firmware/sil/adapter.py +71 -0
- forma_core/firmware/sil/build.py +49 -0
- forma_core/firmware/sil/native/CMakeLists.txt +9 -0
- forma_core/firmware/sil/native/bridge.cpp +112 -0
- forma_core/firmware/sil/native/controller.cpp +123 -0
- forma_core/firmware/sil/native/controller.h +49 -0
- forma_core/firmware/sil/simulator.py +139 -0
- forma_core/firmware/targets.py +111 -0
- forma_core/firmware/validation.py +130 -0
- forma_core/fluidics/__init__.py +6 -0
- forma_core/fluidics/benchmark.py +217 -0
- forma_core/fluidics/calculations.py +79 -0
- forma_core/fluidics/examples.py +54 -0
- forma_core/fluidics/ir.py +25 -0
- forma_core/fluidics/models.py +132 -0
- forma_core/fluidics/topology.py +47 -0
- forma_core/fluidics/validation.py +134 -0
- forma_core/generation.py +32 -0
- forma_core/geometry_bindings/__init__.py +20 -0
- forma_core/geometry_bindings/cad_adapter.py +80 -0
- forma_core/geometry_bindings/models.py +163 -0
- forma_core/geometry_bindings/resolution.py +340 -0
- forma_core/geometry_bindings/tools.py +125 -0
- forma_core/image_providers.py +3049 -0
- forma_core/images.py +3 -0
- forma_core/integrations/__init__.py +1 -0
- forma_core/integrations/huggingface/__init__.py +29 -0
- forma_core/integrations/huggingface/huggingface_artifacts.py +272 -0
- forma_core/jobs/__init__.py +1 -0
- forma_core/jobs/context.py +277 -0
- forma_core/jobs/continuous.py +719 -0
- forma_core/jobs/metrics.py +124 -0
- forma_core/jobs/migrations.py +161 -0
- forma_core/jobs/persistence.py +28 -0
- forma_core/jobs/repositories.py +422 -0
- forma_core/jobs/schema.py +28 -0
- forma_core/jobs/source_usage.py +230 -0
- forma_core/jobs/store.py +476 -0
- forma_core/llm.py +3 -0
- forma_core/llm_providers.py +2682 -0
- forma_core/logs.py +107 -0
- forma_core/mechanics/__init__.py +1 -0
- forma_core/mechanics/gear_benchmark.py +85 -0
- forma_core/mechanics/mechanism_benchmarks.py +345 -0
- forma_core/mechanics/models.py +72 -0
- forma_core/mechanics/primitives.py +109 -0
- forma_core/mechanics/solid_cad.py +83 -0
- forma_core/observability.py +237 -0
- forma_core/openai_streams.py +780 -0
- forma_core/opencode/__init__.py +25 -0
- forma_core/opencode/architecture.py +82 -0
- forma_core/opencode/capabilities.py +130 -0
- forma_core/opencode/models.py +419 -0
- forma_core/opencode/persistence.py +62 -0
- forma_core/opencode/public_events.py +91 -0
- forma_core/opencode/schema.py +28 -0
- forma_core/opencode/store.py +874 -0
- forma_core/persistence/__init__.py +16 -0
- forma_core/persistence/base.py +38 -0
- forma_core/persistence/images.py +598 -0
- forma_core/persistence/migrations.py +200 -0
- forma_core/persistence/models.py +439 -0
- forma_core/persistence/project_artifacts.py +341 -0
- forma_core/persistence/project_reconciliation.py +1094 -0
- forma_core/persistence/providers/__init__.py +4 -0
- forma_core/persistence/providers/sqlite.py +114 -0
- forma_core/persistence/providers/supabase.py +31 -0
- forma_core/persistence/repositories/__init__.py +9 -0
- forma_core/persistence/repositories/base.py +335 -0
- forma_core/persistence/repositories/sqlite.py +1571 -0
- forma_core/persistence/repositories/supabase.py +1576 -0
- forma_core/persistence/schema.py +176 -0
- forma_core/processes/__init__.py +5 -0
- forma_core/processes/models.py +338 -0
- forma_core/processes/validation.py +141 -0
- forma_core/project_list_cache.py +316 -0
- forma_core/prompt_continuity.py +387 -0
- forma_core/providers/__init__.py +39 -0
- forma_core/providers/base.py +28 -0
- forma_core/providers/models.py +138 -0
- forma_core/providers/openai.py +92 -0
- forma_core/providers/openai_compatible.py +190 -0
- forma_core/providers/registry.py +68 -0
- forma_core/providers/registry_utils.py +46 -0
- forma_core/py.typed +1 -0
- forma_core/runtime.py +3 -0
- forma_core/selectors.py +36 -0
- forma_core/signups/__init__.py +5 -0
- forma_core/signups/models.py +49 -0
- forma_core/terminal/__init__.py +1 -0
- forma_core/terminal/dashboard.py +475 -0
- forma_core/terminal/images.py +156 -0
- forma_core/thermal/__init__.py +5 -0
- forma_core/thermal/models.py +361 -0
- forma_core/thermal/validation.py +175 -0
- forma_core/user_integrations.py +2069 -0
- forma_core/utils.py +291 -0
- forma_core/validation.py +83 -0
- forma_core/vertex_auth.py +92 -0
- forma_core/video_prompts.py +193 -0
- forma_core/video_review.py +1006 -0
- forma_core/wiring.py +356 -0
- forma_core/workers/__init__.py +120 -0
- forma_core/workers/contracts.py +183 -0
- forma_core/workers/generation.py +753 -0
- forma_core/workers/orchestration.py +805 -0
- forma_core/workers/registry.py +222 -0
- forma_core/workers/reverse_engineering.py +272 -0
- forma_core/workers/validation.py +291 -0
- forma_core/workspaces/__init__.py +5 -0
- forma_core/workspaces/chats/__init__.py +11 -0
- forma_core/workspaces/chats/models.py +48 -0
- forma_core/workspaces/context/__init__.py +15 -0
- forma_core/workspaces/context/agent.py +260 -0
- forma_core/workspaces/context/models.py +117 -0
- forma_core/workspaces/context/pdf.py +418 -0
- forma_core/workspaces/design_briefs/__init__.py +25 -0
- forma_core/workspaces/design_briefs/models.py +85 -0
- forma_core/workspaces/design_briefs/references.py +56 -0
- forma_core/workspaces/models.py +41 -0
- forma_core/workspaces/projects/__init__.py +88 -0
- forma_core/workspaces/projects/cad_generation.py +1016 -0
- forma_core/workspaces/projects/context_governance.py +365 -0
- forma_core/workspaces/projects/design_lifecycle.py +668 -0
- forma_core/workspaces/projects/fabrication/__init__.py +54 -0
- forma_core/workspaces/projects/fabrication/demo_printers.py +236 -0
- forma_core/workspaces/projects/fabrication/models.py +120 -0
- forma_core/workspaces/projects/fabrication/printer_config.py +205 -0
- forma_core/workspaces/projects/fabrication/slicers/__init__.py +7 -0
- forma_core/workspaces/projects/fabrication/slicers/base.py +25 -0
- forma_core/workspaces/projects/fabrication/slicers/cura.py +125 -0
- forma_core/workspaces/projects/fabrication/slicers/orca.py +208 -0
- forma_core/workspaces/projects/fabrication/slicing.py +202 -0
- forma_core/workspaces/projects/fabrication/validation.py +103 -0
- forma_core/workspaces/projects/gear_benchmark.py +6 -0
- forma_core/workspaces/projects/generation_mode.py +70 -0
- forma_core/workspaces/projects/identity.py +28 -0
- forma_core/workspaces/projects/iteration.py +680 -0
- forma_core/workspaces/projects/manifest.py +362 -0
- forma_core/workspaces/projects/mechanism_benchmarks.py +12 -0
- forma_core/workspaces/projects/models.py +913 -0
- forma_core/workspaces/projects/objects.py +731 -0
- forma_core/workspaces/projects/outcomes.py +47 -0
- forma_core/workspaces/projects/output.py +735 -0
- forma_core/workspaces/projects/resolver.py +397 -0
- forma_core/workspaces/projects/solid_cad.py +8 -0
- forma_core/workspaces/projects/state.py +622 -0
- forma_core/workspaces/readiness/__init__.py +28 -0
- forma_core/workspaces/readiness/evaluator.py +103 -0
- forma_core/workspaces/readiness/models.py +97 -0
- forma_core/workspaces/readiness/service.py +225 -0
- forma_core/workspaces/reverse_engineering/__init__.py +37 -0
- forma_core/workspaces/reverse_engineering/analyzer.py +246 -0
- forma_core/workspaces/reverse_engineering/models.py +128 -0
- forma_core/workspaces/validation/__init__.py +37 -0
- forma_core/workspaces/validation/evaluator.py +327 -0
- forma_core/workspaces/validation/models.py +146 -0
- forma_core/workspaces/validation/service.py +212 -0
- forma_core/workspaces/workflow/__init__.py +29 -0
- forma_core/workspaces/workflow/guard.py +33 -0
- forma_core/workspaces/workflow/models.py +77 -0
- forma_core/workspaces/workflow/service.py +287 -0
- openforma-0.4.0.dist-info/METADATA +247 -0
- openforma-0.4.0.dist-info/RECORD +288 -0
- openforma-0.4.0.dist-info/WHEEL +5 -0
- openforma-0.4.0.dist-info/entry_points.txt +4 -0
- openforma-0.4.0.dist-info/licenses/LICENSE +373 -0
- openforma-0.4.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
"""Private storage for artifacts transferred by the Forma OSS CLI."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import re
|
|
7
|
+
import tempfile
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from forma_core.config import config
|
|
13
|
+
from forma_core.config.runtime import forma_dev_mode_enabled, primary_database_backend_from_environment
|
|
14
|
+
from forma_core.workspaces.projects.manifest import normalize_artifact_media_type
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
DEFAULT_PROJECT_ARTIFACT_BUCKET = "cli-project-artifacts"
|
|
18
|
+
DEFAULT_PROJECT_ARTIFACT_DIRECTORY = ".forma/cli-artifacts"
|
|
19
|
+
DEFAULT_PROJECT_ARTIFACT_MAX_BYTES = 50 * 1024 * 1024
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class ProjectArtifactStorageError(RuntimeError):
|
|
23
|
+
"""Raised when private project artifact storage cannot be used."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class StoredProjectArtifact:
|
|
28
|
+
project_id: str
|
|
29
|
+
sha256: str
|
|
30
|
+
media_type: str
|
|
31
|
+
size_bytes: int
|
|
32
|
+
content: bytes | None = None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _local_content_matches(target: Path, content: bytes) -> bool:
|
|
36
|
+
"""Prove that a content-addressed put is already satisfied, or require a write."""
|
|
37
|
+
try:
|
|
38
|
+
return target.stat().st_size == len(content) and target.read_bytes() == content
|
|
39
|
+
except OSError:
|
|
40
|
+
# Missing or unreadable content cannot establish a successful put.
|
|
41
|
+
return False
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _storage_backend() -> str:
|
|
45
|
+
configured = (config.optional("FORMA_CLI_ARTIFACT_STORAGE_BACKEND") or "").lower()
|
|
46
|
+
aliases = {
|
|
47
|
+
"local": "local",
|
|
48
|
+
"sqlite": "local",
|
|
49
|
+
"supabase": "supabase",
|
|
50
|
+
"supabase-client": "supabase",
|
|
51
|
+
"s3": "s3-compatible",
|
|
52
|
+
"s3-compatible": "s3-compatible",
|
|
53
|
+
}
|
|
54
|
+
if configured:
|
|
55
|
+
return aliases.get(configured, "local")
|
|
56
|
+
if forma_dev_mode_enabled():
|
|
57
|
+
return "local"
|
|
58
|
+
return "supabase" if primary_database_backend_from_environment() == "supabase" else "local"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def get_project_artifact_storage_config() -> dict[str, Any]:
|
|
62
|
+
"""Resolve artifact storage without constructing a remote client."""
|
|
63
|
+
backend = _storage_backend()
|
|
64
|
+
bucket = config.optional("FORMA_CLI_ARTIFACT_BUCKET") or DEFAULT_PROJECT_ARTIFACT_BUCKET
|
|
65
|
+
directory = Path(
|
|
66
|
+
config.optional("FORMA_CLI_ARTIFACT_STORAGE_DIR") or DEFAULT_PROJECT_ARTIFACT_DIRECTORY
|
|
67
|
+
).expanduser()
|
|
68
|
+
try:
|
|
69
|
+
max_bytes = max(1, int(config.optional("FORMA_CLI_ARTIFACT_MAX_BYTES") or DEFAULT_PROJECT_ARTIFACT_MAX_BYTES))
|
|
70
|
+
except ValueError:
|
|
71
|
+
max_bytes = DEFAULT_PROJECT_ARTIFACT_MAX_BYTES
|
|
72
|
+
supabase_url = config.optional("SUPABASE_URL") or config.optional("NEXT_PUBLIC_SUPABASE_URL")
|
|
73
|
+
service_key = config.optional("SUPABASE_SERVICE_ROLE_KEY") or config.optional("SUPABASE_SECRET_KEY")
|
|
74
|
+
endpoint = config.optional("SUPABASE_S3_ENDPOINT")
|
|
75
|
+
access_key = config.optional("SUPABASE_S3_ACCESS_KEY_ID") or config.optional("AWS_ACCESS_KEY_ID")
|
|
76
|
+
secret_key = config.optional("SUPABASE_S3_SECRET_ACCESS_KEY") or config.optional("AWS_SECRET_ACCESS_KEY")
|
|
77
|
+
enabled = (
|
|
78
|
+
backend == "local"
|
|
79
|
+
or backend == "supabase" and bool(supabase_url and service_key)
|
|
80
|
+
or backend == "s3-compatible" and bool(endpoint and access_key and secret_key)
|
|
81
|
+
)
|
|
82
|
+
return {
|
|
83
|
+
"enabled": enabled,
|
|
84
|
+
"backend": backend,
|
|
85
|
+
"bucket": bucket,
|
|
86
|
+
"directory": directory,
|
|
87
|
+
"max_bytes": max_bytes,
|
|
88
|
+
"supabase_url_configured": bool(supabase_url),
|
|
89
|
+
"service_key_configured": bool(service_key),
|
|
90
|
+
"endpoint_configured": bool(endpoint),
|
|
91
|
+
"access_key_configured": bool(access_key),
|
|
92
|
+
"secret_key_configured": bool(secret_key),
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def project_artifact_storage_key(project_id: str, sha256: str) -> str:
|
|
97
|
+
"""Build a server-owned, project-scoped object key."""
|
|
98
|
+
project_text = str(project_id or "").strip()
|
|
99
|
+
if not project_text:
|
|
100
|
+
raise ValueError("Project artifact storage requires a project_id.")
|
|
101
|
+
digest = str(sha256 or "").strip().lower()
|
|
102
|
+
if not re.fullmatch(r"[0-9a-f]{64}", digest):
|
|
103
|
+
raise ValueError("Project artifact storage requires a SHA-256 hash.")
|
|
104
|
+
project_scope = hashlib.sha256(project_text.encode("utf-8")).hexdigest()
|
|
105
|
+
return f"projects/{project_scope}/artifacts/{digest}"
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class ProjectArtifactStorage:
|
|
109
|
+
"""Read and write private objects using Supabase, S3, or local disk."""
|
|
110
|
+
|
|
111
|
+
def __init__(self, storage_config: dict[str, Any] | None = None) -> None:
|
|
112
|
+
self.config = storage_config if storage_config is not None else get_project_artifact_storage_config()
|
|
113
|
+
|
|
114
|
+
def _require_enabled(self) -> None:
|
|
115
|
+
if self.config.get("enabled"):
|
|
116
|
+
return
|
|
117
|
+
backend = self.config.get("backend")
|
|
118
|
+
if backend == "supabase":
|
|
119
|
+
raise ProjectArtifactStorageError(
|
|
120
|
+
"CLI artifact storage requires SUPABASE_URL plus SUPABASE_SERVICE_ROLE_KEY or SUPABASE_SECRET_KEY."
|
|
121
|
+
)
|
|
122
|
+
if backend == "s3-compatible":
|
|
123
|
+
raise ProjectArtifactStorageError(
|
|
124
|
+
"CLI artifact S3 storage requires SUPABASE_S3_ENDPOINT and access credentials."
|
|
125
|
+
)
|
|
126
|
+
raise ProjectArtifactStorageError("CLI artifact storage is not configured.")
|
|
127
|
+
|
|
128
|
+
def _validate_size(self, content: bytes) -> None:
|
|
129
|
+
if len(content) > int(self.config["max_bytes"]):
|
|
130
|
+
raise ProjectArtifactStorageError(
|
|
131
|
+
f"Project artifact exceeds the {self.config['max_bytes']} byte size limit."
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
def _supabase_bucket(self):
|
|
135
|
+
supabase_url = config.optional("SUPABASE_URL") or config.optional("NEXT_PUBLIC_SUPABASE_URL")
|
|
136
|
+
service_key = config.optional("SUPABASE_SERVICE_ROLE_KEY") or config.optional("SUPABASE_SECRET_KEY")
|
|
137
|
+
if not supabase_url or not service_key:
|
|
138
|
+
raise ProjectArtifactStorageError(
|
|
139
|
+
"CLI artifact storage requires SUPABASE_URL plus SUPABASE_SERVICE_ROLE_KEY or SUPABASE_SECRET_KEY."
|
|
140
|
+
)
|
|
141
|
+
try:
|
|
142
|
+
from supabase import create_client
|
|
143
|
+
except ImportError as exc:
|
|
144
|
+
raise ProjectArtifactStorageError(
|
|
145
|
+
"Supabase client is not installed. Run pip install -r apps/api/requirements.txt."
|
|
146
|
+
) from exc
|
|
147
|
+
return create_client(supabase_url, service_key).storage.from_(self.config["bucket"])
|
|
148
|
+
|
|
149
|
+
def _s3_client(self):
|
|
150
|
+
try:
|
|
151
|
+
import boto3
|
|
152
|
+
from botocore.config import Config
|
|
153
|
+
except ImportError as exc:
|
|
154
|
+
raise ProjectArtifactStorageError("boto3 is required for S3-compatible artifact storage.") from exc
|
|
155
|
+
endpoint = config.optional("SUPABASE_S3_ENDPOINT")
|
|
156
|
+
access_key = config.optional("SUPABASE_S3_ACCESS_KEY_ID") or config.optional("AWS_ACCESS_KEY_ID")
|
|
157
|
+
secret_key = config.optional("SUPABASE_S3_SECRET_ACCESS_KEY") or config.optional("AWS_SECRET_ACCESS_KEY")
|
|
158
|
+
return boto3.client(
|
|
159
|
+
"s3",
|
|
160
|
+
endpoint_url=endpoint,
|
|
161
|
+
aws_access_key_id=access_key,
|
|
162
|
+
aws_secret_access_key=secret_key,
|
|
163
|
+
region_name=config.optional("SUPABASE_S3_REGION") or config.optional("AWS_REGION") or "us-east-1",
|
|
164
|
+
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
def put(self, project_id: str, sha256: str, content: bytes, media_type: str) -> StoredProjectArtifact:
|
|
168
|
+
self._require_enabled()
|
|
169
|
+
self._validate_size(content)
|
|
170
|
+
normalized_sha256 = str(sha256 or "").strip().lower()
|
|
171
|
+
key = project_artifact_storage_key(project_id, normalized_sha256)
|
|
172
|
+
actual_sha256 = hashlib.sha256(content).hexdigest()
|
|
173
|
+
if actual_sha256 != normalized_sha256:
|
|
174
|
+
raise ProjectArtifactStorageError(
|
|
175
|
+
f"Project artifact content does not match its declared SHA-256 hash: {normalized_sha256}."
|
|
176
|
+
)
|
|
177
|
+
try:
|
|
178
|
+
normalized_media_type = normalize_artifact_media_type(media_type)
|
|
179
|
+
except ValueError as exc:
|
|
180
|
+
raise ProjectArtifactStorageError("Project artifact media type is invalid.") from exc
|
|
181
|
+
backend = self.config["backend"]
|
|
182
|
+
if backend == "local":
|
|
183
|
+
target = Path(self.config["directory"]) / Path(*key.split("/"))
|
|
184
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
185
|
+
if not _local_content_matches(target, content):
|
|
186
|
+
# Give each writer its own file, and close it before publication.
|
|
187
|
+
temporary: Path | None = None
|
|
188
|
+
try:
|
|
189
|
+
with tempfile.NamedTemporaryFile(dir=target.parent, delete=False) as handle:
|
|
190
|
+
temporary = Path(handle.name)
|
|
191
|
+
handle.write(content)
|
|
192
|
+
try:
|
|
193
|
+
temporary.replace(target)
|
|
194
|
+
except PermissionError:
|
|
195
|
+
# A concurrent identical put may have won while a reader
|
|
196
|
+
# holds its target open (Windows denies replacement).
|
|
197
|
+
# Accept only verified bytes, never mere file existence.
|
|
198
|
+
if not _local_content_matches(target, content):
|
|
199
|
+
raise
|
|
200
|
+
finally:
|
|
201
|
+
if temporary is not None:
|
|
202
|
+
temporary.unlink(missing_ok=True)
|
|
203
|
+
elif backend == "supabase":
|
|
204
|
+
bucket = self._supabase_bucket()
|
|
205
|
+
options = {
|
|
206
|
+
"content-type": normalized_media_type,
|
|
207
|
+
"cache-control": "31536000",
|
|
208
|
+
"upsert": "true",
|
|
209
|
+
}
|
|
210
|
+
try:
|
|
211
|
+
bucket.upload(key, content, file_options=options)
|
|
212
|
+
except Exception:
|
|
213
|
+
# A retry may target an object that the first request created.
|
|
214
|
+
try:
|
|
215
|
+
bucket.update(key, content, file_options=options)
|
|
216
|
+
except Exception as update_exc:
|
|
217
|
+
raise ProjectArtifactStorageError("Project artifact storage upload failed.") from update_exc
|
|
218
|
+
else:
|
|
219
|
+
try:
|
|
220
|
+
self._s3_client().put_object(
|
|
221
|
+
Bucket=self.config["bucket"],
|
|
222
|
+
Key=key,
|
|
223
|
+
Body=content,
|
|
224
|
+
ContentType=normalized_media_type,
|
|
225
|
+
)
|
|
226
|
+
except Exception as exc:
|
|
227
|
+
raise ProjectArtifactStorageError("Project artifact storage upload failed.") from exc
|
|
228
|
+
return StoredProjectArtifact(project_id, normalized_sha256, normalized_media_type, len(content))
|
|
229
|
+
|
|
230
|
+
def get(self, project_id: str, sha256: str, media_type: str) -> StoredProjectArtifact:
|
|
231
|
+
self._require_enabled()
|
|
232
|
+
normalized_sha256 = str(sha256 or "").strip().lower()
|
|
233
|
+
key = project_artifact_storage_key(project_id, normalized_sha256)
|
|
234
|
+
try:
|
|
235
|
+
normalized_media_type = normalize_artifact_media_type(media_type)
|
|
236
|
+
except ValueError as exc:
|
|
237
|
+
raise ProjectArtifactStorageError("Project artifact media type is invalid.") from exc
|
|
238
|
+
backend = self.config["backend"]
|
|
239
|
+
if backend == "local":
|
|
240
|
+
content = (Path(self.config["directory"]) / Path(*key.split("/"))).read_bytes()
|
|
241
|
+
elif backend == "supabase":
|
|
242
|
+
try:
|
|
243
|
+
content = bytes(self._supabase_bucket().download(key))
|
|
244
|
+
except FileNotFoundError:
|
|
245
|
+
raise
|
|
246
|
+
except Exception as exc:
|
|
247
|
+
raise ProjectArtifactStorageError("Project artifact storage download failed.") from exc
|
|
248
|
+
else:
|
|
249
|
+
try:
|
|
250
|
+
content = self._s3_client().get_object(Bucket=self.config["bucket"], Key=key)["Body"].read()
|
|
251
|
+
except Exception as exc:
|
|
252
|
+
raise ProjectArtifactStorageError("Project artifact storage download failed.") from exc
|
|
253
|
+
self._validate_size(content)
|
|
254
|
+
return StoredProjectArtifact(project_id, normalized_sha256, normalized_media_type, len(content), content)
|
|
255
|
+
|
|
256
|
+
def contains(self, project_id: str, sha256: str) -> bool:
|
|
257
|
+
"""Return whether an object with the declared SHA-256 is already stored."""
|
|
258
|
+
self._require_enabled()
|
|
259
|
+
normalized_sha256 = str(sha256 or "").strip().lower()
|
|
260
|
+
if not re.fullmatch(r"[0-9a-f]{64}", normalized_sha256):
|
|
261
|
+
raise ValueError("Project artifact storage requires a SHA-256 hash.")
|
|
262
|
+
key = project_artifact_storage_key(project_id, normalized_sha256)
|
|
263
|
+
backend = self.config["backend"]
|
|
264
|
+
if backend == "local":
|
|
265
|
+
return (Path(self.config["directory"]) / Path(*key.split("/"))).is_file()
|
|
266
|
+
if backend == "supabase":
|
|
267
|
+
bucket = self._supabase_bucket()
|
|
268
|
+
folder, _, name = key.rpartition("/")
|
|
269
|
+
try:
|
|
270
|
+
# Supabase storage treats a list prefix as a folder, so listing
|
|
271
|
+
# the full object key would only return its children. List the
|
|
272
|
+
# containing folder and match the object name instead.
|
|
273
|
+
items = bucket.list(folder, {"limit": 1000, "search": name}) or []
|
|
274
|
+
except Exception as exc:
|
|
275
|
+
raise ProjectArtifactStorageError("Project artifact presence check failed.") from exc
|
|
276
|
+
return any(
|
|
277
|
+
isinstance(item, dict) and item.get("name") == name
|
|
278
|
+
for item in items
|
|
279
|
+
)
|
|
280
|
+
try:
|
|
281
|
+
self._s3_client().head_object(Bucket=self.config["bucket"], Key=key)
|
|
282
|
+
return True
|
|
283
|
+
except Exception:
|
|
284
|
+
return False
|
|
285
|
+
|
|
286
|
+
def delete_project(self, project_id: str) -> int:
|
|
287
|
+
"""Delete all objects for a project; used by privacy purge workers."""
|
|
288
|
+
self._require_enabled()
|
|
289
|
+
project_text = str(project_id or "").strip()
|
|
290
|
+
if not project_text:
|
|
291
|
+
raise ValueError("Project artifact deletion requires a project_id.")
|
|
292
|
+
scope = hashlib.sha256(project_text.encode("utf-8")).hexdigest()
|
|
293
|
+
prefix = f"projects/{scope}/artifacts/"
|
|
294
|
+
backend = self.config["backend"]
|
|
295
|
+
if backend == "local":
|
|
296
|
+
root = Path(self.config["directory"]) / Path(*prefix.split("/")[:-1])
|
|
297
|
+
if not root.exists():
|
|
298
|
+
return 0
|
|
299
|
+
count = sum(1 for item in root.iterdir() if item.is_file())
|
|
300
|
+
for item in root.iterdir():
|
|
301
|
+
if item.is_file():
|
|
302
|
+
item.unlink()
|
|
303
|
+
return count
|
|
304
|
+
if backend == "supabase":
|
|
305
|
+
bucket = self._supabase_bucket()
|
|
306
|
+
items: list[dict[str, Any]] = []
|
|
307
|
+
offset = 0
|
|
308
|
+
while True:
|
|
309
|
+
page = bucket.list(prefix.removesuffix("/"), {"limit": 1000, "offset": offset}) or []
|
|
310
|
+
items.extend(item for item in page if isinstance(item, dict))
|
|
311
|
+
if len(page) < 1000:
|
|
312
|
+
break
|
|
313
|
+
offset += 1000
|
|
314
|
+
keys = [f"{prefix}{item['name']}" for item in items if item.get("name")]
|
|
315
|
+
for start in range(0, len(keys), 1000):
|
|
316
|
+
bucket.remove(keys[start : start + 1000])
|
|
317
|
+
return len(keys)
|
|
318
|
+
client = self._s3_client()
|
|
319
|
+
paginator = client.get_paginator("list_objects_v2")
|
|
320
|
+
keys = [
|
|
321
|
+
item["Key"]
|
|
322
|
+
for page in paginator.paginate(Bucket=self.config["bucket"], Prefix=prefix)
|
|
323
|
+
for item in page.get("Contents") or []
|
|
324
|
+
if item.get("Key")
|
|
325
|
+
]
|
|
326
|
+
for start in range(0, len(keys), 1000):
|
|
327
|
+
client.delete_objects(
|
|
328
|
+
Bucket=self.config["bucket"],
|
|
329
|
+
Delete={"Objects": [{"Key": key} for key in keys[start : start + 1000]], "Quiet": True},
|
|
330
|
+
)
|
|
331
|
+
return len(keys)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
__all__ = [
|
|
335
|
+
"DEFAULT_PROJECT_ARTIFACT_BUCKET",
|
|
336
|
+
"ProjectArtifactStorage",
|
|
337
|
+
"ProjectArtifactStorageError",
|
|
338
|
+
"StoredProjectArtifact",
|
|
339
|
+
"get_project_artifact_storage_config",
|
|
340
|
+
"project_artifact_storage_key",
|
|
341
|
+
]
|