takos-control 1.0.0

package/src/application/services/r2/orphaned-object-gc.ts

@@ -0,0 +1,268 @@
+import type { D1Database, R2Bucket } from '../../../shared/types/bindings.ts';
+import { getDb, blobs, sessionFiles, sessions, snapshots } from '../../../infra/db';
+import { eq, and, inArray } from 'drizzle-orm';
+
+type OffloadEnv = {
+  DB: D1Database;
+  TENANT_SOURCE?: R2Bucket;
+  TAKOS_OFFLOAD?: R2Bucket;
+};
+
+type CursorState = {
+  version: 1;
+  last_success_at: string;
+  cursors: { blobs?: string; trees?: string };
+};
+
+export interface R2OrphanedObjectGcSummary {
+  skipped: boolean;
+  reason?: string;
+  dry_run: boolean;
+  started_at: string;
+  min_age_minutes: number;
+  scanned: { blobs: number; trees: number };
+  candidates: { blobs: number; trees: number };
+  deleted: { blobs: number; trees: number };
+  next_cursors: { blobs?: string; trees?: string };
+}
+
+const STATE_KEY = 'ops/job-state/r2-orphaned-object-gc.json';
+const AUDIT_PREFIX = 'ops/r2-orphaned-object-gc';
+
+function stableIsoNoMillis(date: Date): string {
+  return date.toISOString().replace(/\.\d{3}Z$/, 'Z');
+}
+
+function safeKeyTimestamp(iso: string): string {
+  return iso.replace(/:/g, '-');
+}
+
+async function readJson<T>(bucket: R2Bucket, key: string): Promise<T | null> {
+  const obj = await bucket.get(key);
+  if (!obj) return null;
+  try {
+    return await obj.json<T>();
+  } catch {
+    return null;
+  }
+}
+
+async function writeJson(bucket: R2Bucket, key: string, value: unknown): Promise<void> {
+  await bucket.put(key, JSON.stringify(value), {
+    httpMetadata: { contentType: 'application/json; charset=utf-8' },
+  });
+}
+
+function uploadedMs(obj: { uploaded?: Date }): number | null {
+  if (!(obj.uploaded instanceof Date)) return null;
+  const ms = obj.uploaded.getTime();
+  return Number.isFinite(ms) ? ms : null;
+}
+
+const WORKSPACE_ID_RE = /^[a-zA-Z0-9_-]{6,128}$/;
+
+function parseBlobKey(key: string): { spaceId: string; hash: string } | null {
+  if (!key.startsWith('blobs/')) return null;
+  const parts = key.slice(6).split('/');
+  if (parts.length !== 2) return null;
+  const [spaceId, hash] = parts;
+  if (!WORKSPACE_ID_RE.test(spaceId) || !/^[0-9a-f]{64}$/.test(hash)) return null;
+  return { spaceId, hash };
+}
+
+function parseTreeKey(key: string): { spaceId: string; snapshotId: string } | null {
+  if (!key.startsWith('trees/') || !key.endsWith('.json.gz')) return null;
+  const parts = key.slice(6).split('/');
+  if (parts.length !== 2) return null;
+  const [spaceId, name] = parts;
+  const snapshotId = name.slice(0, -8);
+  if (!WORKSPACE_ID_RE.test(spaceId) || !WORKSPACE_ID_RE.test(snapshotId)) return null;
+  return { spaceId, snapshotId };
+}
+
+function groupBySpaceId<T extends { spaceId: string }>(items: T[]): Map<string, T[]> {
+  const map = new Map<string, T[]>();
+  for (const item of items) {
+    let list = map.get(item.spaceId);
+    if (!list) {
+      list = [];
+      map.set(item.spaceId, list);
+    }
+    list.push(item);
+  }
+  return map;
+}
+
+function skippedSummary(reason: string): R2OrphanedObjectGcSummary {
+  return {
+    skipped: true,
+    reason,
+    dry_run: true,
+    started_at: stableIsoNoMillis(new Date()),
+    min_age_minutes: 0,
+    scanned: { blobs: 0, trees: 0 },
+    candidates: { blobs: 0, trees: 0 },
+    deleted: { blobs: 0, trees: 0 },
+    next_cursors: {},
+  };
+}
+
+function clamp(value: number, min: number, max: number): number {
+  return Math.max(min, Math.min(value, max));
+}
+
+export async function runR2OrphanedObjectGcBatch(
+  env: OffloadEnv,
+  options?: {
+    dryRun?: boolean;
+    listLimit?: number;
+    maxDeletes?: number;
+    minAgeMinutes?: number;
+  }
+): Promise<R2OrphanedObjectGcSummary> {
+  const source = env.TENANT_SOURCE;
+  if (!source) return skippedSummary('TENANT_SOURCE bucket not configured');
+
+  const offload = env.TAKOS_OFFLOAD;
+  if (!offload) return skippedSummary('TAKOS_OFFLOAD bucket not configured');
+
+  const dryRun = options?.dryRun ?? false;
+  const listLimit = clamp(options?.listLimit ?? 200, 1, 1000);
+  const maxDeletes = clamp(options?.maxDeletes ?? 200, 0, 1000);
+  const minAgeMinutes = clamp(options?.minAgeMinutes ?? 24 * 60, 0, 30 * 24 * 60);
+
+  const startedAt = stableIsoNoMillis(new Date());
+  const cutoffMs = Date.now() - minAgeMinutes * 60_000;
+
+  const prior = await readJson<CursorState>(offload, STATE_KEY);
+  const cursors = prior?.version === 1 && prior.cursors ? { ...prior.cursors } : {};
+
+  const db = getDb(env.DB);
+
+  const summary: R2OrphanedObjectGcSummary = {
+    skipped: false,
+    dry_run: dryRun,
+    started_at: startedAt,
+    min_age_minutes: minAgeMinutes,
+    scanned: { blobs: 0, trees: 0 },
+    candidates: { blobs: 0, trees: 0 },
+    deleted: { blobs: 0, trees: 0 },
+    next_cursors: {},
+  };
+
+  const deletedKeys: { blobs: string[]; trees: string[] } = { blobs: [], trees: [] };
+
+  // Blob GC: delete objects whose DB blob row is missing AND not referenced by any session_file.
+  const blobsPage = await source.list({ prefix: 'blobs/', cursor: cursors.blobs, limit: listLimit });
+  summary.next_cursors.blobs = blobsPage.truncated ? blobsPage.cursor : undefined;
+
+  const blobCandidates: Array<{ key: string; spaceId: string; hash: string }> = [];
+  for (const obj of blobsPage.objects) {
+    summary.scanned.blobs += 1;
+    if (minAgeMinutes > 0) {
+      const ms = uploadedMs(obj);
+      if (ms === null || ms > cutoffMs) continue;
+    }
+    const parsed = parseBlobKey(obj.key);
+    if (parsed) blobCandidates.push({ key: obj.key, ...parsed });
+  }
+  summary.candidates.blobs = blobCandidates.length;
+
+  const blobDeletes: string[] = [];
+  for (const [spaceId, items] of groupBySpaceId(blobCandidates)) {
+    const hashes = [...new Set(items.map((i) => i.hash))];
+    if (hashes.length === 0) continue;
+
+    const existing = new Set(
+      (await db.select({ hash: blobs.hash }).from(blobs)
+        .where(and(eq(blobs.accountId, spaceId), inArray(blobs.hash, hashes)))
+        .all()
+      ).map((b) => b.hash)
+    );
+
+    const referenced = new Set(
+      (await db.selectDistinct({ hash: sessionFiles.hash })
+        .from(sessionFiles)
+        .innerJoin(sessions, eq(sessionFiles.sessionId, sessions.id))
+        .where(and(inArray(sessionFiles.hash, hashes), eq(sessions.accountId, spaceId)))
+        .all()
+      ).map((r) => r.hash)
+    );
+
+    for (const item of items) {
+      if (!existing.has(item.hash) && !referenced.has(item.hash)) {
+        blobDeletes.push(item.key);
+      }
+    }
+  }
+
+  const blobDeletesLimited = blobDeletes.slice(0, maxDeletes);
+  if (!dryRun && blobDeletesLimited.length > 0) {
+    await source.delete(blobDeletesLimited);
+  }
+  summary.deleted.blobs = blobDeletesLimited.length;
+  deletedKeys.blobs.push(...blobDeletesLimited);
+
+  // Tree GC: delete objects whose DB snapshot row is missing.
+  const treesPage = await source.list({ prefix: 'trees/', cursor: cursors.trees, limit: listLimit });
+  summary.next_cursors.trees = treesPage.truncated ? treesPage.cursor : undefined;
+
+  const treeCandidates: Array<{ key: string; spaceId: string; snapshotId: string }> = [];
+  for (const obj of treesPage.objects) {
+    summary.scanned.trees += 1;
+    if (minAgeMinutes > 0) {
+      const ms = uploadedMs(obj);
+      if (ms === null || ms > cutoffMs) continue;
+    }
+    const parsed = parseTreeKey(obj.key);
+    if (parsed) treeCandidates.push({ key: obj.key, ...parsed });
+  }
+  summary.candidates.trees = treeCandidates.length;
+
+  const treeDeletes: string[] = [];
+  for (const [spaceId, items] of groupBySpaceId(treeCandidates)) {
+    const ids = [...new Set(items.map((i) => i.snapshotId))];
+    if (ids.length === 0) continue;
+
+    const existingIds = new Set(
+      (await db.select({ id: snapshots.id }).from(snapshots)
+        .where(and(eq(snapshots.accountId, spaceId), inArray(snapshots.id, ids)))
+        .all()
+      ).map((s) => s.id)
+    );
+
+    for (const item of items) {
+      if (!existingIds.has(item.snapshotId)) {
+        treeDeletes.push(item.key);
+      }
+    }
+  }
+
+  const treeDeletesLimited = treeDeletes.slice(0, maxDeletes);
+  if (!dryRun && treeDeletesLimited.length > 0) {
+    await source.delete(treeDeletesLimited);
+  }
+  summary.deleted.trees = treeDeletesLimited.length;
+  deletedKeys.trees.push(...treeDeletesLimited);
+
+  const nextCursors: CursorState['cursors'] = {};
+  if (summary.next_cursors.blobs) nextCursors.blobs = summary.next_cursors.blobs;
+  if (summary.next_cursors.trees) nextCursors.trees = summary.next_cursors.trees;
+
+  await writeJson(offload, STATE_KEY, {
+    version: 1,
+    last_success_at: stableIsoNoMillis(new Date()),
+    cursors: nextCursors,
+  } satisfies CursorState);
+
+  if (deletedKeys.blobs.length > 0 || deletedKeys.trees.length > 0) {
+    await writeJson(offload, `${AUDIT_PREFIX}/deleted/${safeKeyTimestamp(startedAt)}.json`, {
+      started_at: startedAt,
+      dry_run: dryRun,
+      min_age_minutes: minAgeMinutes,
+      deleted: deletedKeys,
+    });
+  }
+
+  return summary;
+}

package/src/application/services/resources/access.ts

@@ -0,0 +1,180 @@
+import type { D1Database } from '../../../shared/types/bindings.ts';
+import type { ResourcePermission } from '../../../shared/types';
+import { getDb, resourceAccess } from '../../../infra/db';
+import { eq, and, inArray } from 'drizzle-orm';
+import { generateId, now, toRequiredIsoString } from '../../../shared/utils';
+import { toApiResourceAccess } from './format';
+import { getResourceById } from './store';
+import { resolveAccessibleAccountIds } from '../identity/membership-resolver';
+
+const RESOURCE_PERMISSIONS: readonly string[] = ['read', 'write', 'admin'];
+
+function isResourcePermission(value: string): value is ResourcePermission {
+  return RESOURCE_PERMISSIONS.includes(value);
+}
+
+export async function listResourceAccess(db: D1Database, resourceId: string) {
+  const drizzle = getDb(db);
+
+  // We need account name, so join with accounts
+  const { accounts } = await import('../../../infra/db');
+  const accessGrants = await drizzle.select({
+    id: resourceAccess.id,
+    resourceId: resourceAccess.resourceId,
+    accountId: resourceAccess.accountId,
+    permission: resourceAccess.permission,
+    grantedByAccountId: resourceAccess.grantedByAccountId,
+    createdAt: resourceAccess.createdAt,
+    accountName: accounts.name,
+  }).from(resourceAccess)
+    .leftJoin(accounts, eq(resourceAccess.accountId, accounts.id))
+    .where(eq(resourceAccess.resourceId, resourceId))
+    .orderBy(resourceAccess.createdAt)
+    .all();
+
+  return accessGrants.map((ra) => ({
+    ...toApiResourceAccess({
+      id: ra.id,
+      resourceId: ra.resourceId,
+      accountId: ra.accountId,
+      permission: ra.permission,
+      grantedByAccountId: ra.grantedByAccountId,
+      createdAt: toRequiredIsoString(ra.createdAt),
+    }),
+    workspace_name: ra.accountName,
+  }));
+}
+
+export async function upsertResourceAccess(
+  db: D1Database,
+  input: { resource_id: string; space_id: string; permission: ResourcePermission; granted_by: string }
+) {
+  const drizzle = getDb(db);
+  const id = generateId();
+  const timestamp = now();
+
+  try {
+    await drizzle.insert(resourceAccess).values({
+      id,
+      resourceId: input.resource_id,
+      accountId: input.space_id,
+      permission: input.permission,
+      grantedByAccountId: input.granted_by,
+      createdAt: timestamp,
+    });
+
+    return {
+      created: true,
+      access: {
+        id,
+        resource_id: input.resource_id,
+        space_id: input.space_id,
+        permission: input.permission,
+        granted_by: input.granted_by,
+        created_at: timestamp,
+      },
+    };
+  } catch (err) {
+    if (String(err).includes('UNIQUE constraint')) {
+      await drizzle.update(resourceAccess)
+        .set({ permission: input.permission })
+        .where(and(
+          eq(resourceAccess.resourceId, input.resource_id),
+          eq(resourceAccess.accountId, input.space_id),
+        ));
+
+      return {
+        created: false,
+        permission: input.permission,
+      };
+    }
+    throw err;
+  }
+}
+
+export async function deleteResourceAccess(db: D1Database, resourceId: string, spaceId: string) {
+  const drizzle = getDb(db);
+  await drizzle.delete(resourceAccess)
+    .where(and(
+      eq(resourceAccess.resourceId, resourceId),
+      eq(resourceAccess.accountId, spaceId),
+    ));
+}
+
+export async function checkResourceAccess(
+  db: D1Database,
+  resourceId: string,
+  userId: string,
+  requiredPermissions?: ResourcePermission[]
+): Promise<boolean> {
+  const drizzle = getDb(db);
+
+  // Find accounts the user is a member of
+  const accountIds = await resolveAccessibleAccountIds(db, userId, { activeOnly: true });
+
+  const access = await drizzle.select({ permission: resourceAccess.permission })
+    .from(resourceAccess)
+    .where(and(
+      eq(resourceAccess.resourceId, resourceId),
+      inArray(resourceAccess.accountId, accountIds),
+    ))
+    .get();
+
+  if (!access) return false;
+
+  if (!requiredPermissions || requiredPermissions.length === 0) return true;
+
+  if (!isResourcePermission(access.permission)) return false;
+
+  return requiredPermissions.includes(access.permission);
+}
+
+export async function canAccessResource(
+  db: D1Database,
+  resourceId: string,
+  userId: string,
+  requiredPermissions?: ResourcePermission[]
+): Promise<{ canAccess: boolean; isOwner: boolean; permission?: ResourcePermission }> {
+  const resource = await getResourceById(db, resourceId);
+
+  if (!resource) {
+    return { canAccess: false, isOwner: false };
+  }
+
+  if (resource.owner_id === userId) {
+    return { canAccess: true, isOwner: true, permission: 'admin' };
+  }
+
+  const drizzle = getDb(db);
+
+  // Find accounts the user is a member of
+  const accountIds = await resolveAccessibleAccountIds(db, userId, { activeOnly: true });
+
+  const access = await drizzle.select({ permission: resourceAccess.permission })
+    .from(resourceAccess)
+    .where(and(
+      eq(resourceAccess.resourceId, resourceId),
+      inArray(resourceAccess.accountId, accountIds),
+    ))
+    .get();
+
+  if (!access) {
+    return { canAccess: false, isOwner: false };
+  }
+
+  if (!isResourcePermission(access.permission)) {
+    return { canAccess: false, isOwner: false };
+  }
+
+  const permission = access.permission;
+
+  if (!requiredPermissions || requiredPermissions.length === 0) {
+    return { canAccess: true, isOwner: false, permission };
+  }
+
+  return {
+    canAccess: requiredPermissions.includes(permission),
+    isOwner: false,
+    permission,
+  };
+}

package/src/application/services/resources/bindings.ts

@@ -0,0 +1,185 @@
+import type { D1Database } from '../../../shared/types/bindings.ts';
+import { getDb, serviceBindings, services } from '../../../infra/db';
+import { eq, and } from 'drizzle-orm';
+import { toRequiredIsoString } from '../../../shared/utils';
+import { toApiServiceBinding } from './format';
+import { getResourceById } from './store';
+
+export async function listServiceBindings(db: D1Database, resourceId: string) {
+  const drizzle = getDb(db);
+
+  const bindingsResult = await drizzle.select({
+    id: serviceBindings.id,
+    serviceId: serviceBindings.serviceId,
+    resourceId: serviceBindings.resourceId,
+    bindingName: serviceBindings.bindingName,
+    bindingType: serviceBindings.bindingType,
+    config: serviceBindings.config,
+    createdAt: serviceBindings.createdAt,
+    serviceHostname: services.hostname,
+    serviceSlug: services.slug,
+    serviceStatus: services.status,
+  }).from(serviceBindings)
+    .leftJoin(services, eq(serviceBindings.serviceId, services.id))
+    .where(eq(serviceBindings.resourceId, resourceId))
+    .orderBy(serviceBindings.createdAt)
+    .all();
+
+  return bindingsResult.map((wb) => ({
+    ...toApiServiceBinding({
+      id: wb.id,
+      serviceId: wb.serviceId,
+      resourceId: wb.resourceId,
+      bindingName: wb.bindingName,
+      bindingType: wb.bindingType,
+      config: wb.config,
+      createdAt: toRequiredIsoString(wb.createdAt),
+    }),
+    service_hostname: wb.serviceHostname,
+    service_slug: wb.serviceSlug,
+    service_status: wb.serviceStatus,
+  }));
+}
+
+export async function countServiceBindings(db: D1Database, resourceId: string) {
+  const drizzle = getDb(db);
+  const { count } = await import('drizzle-orm');
+  const result = await drizzle.select({ count: count() }).from(serviceBindings)
+    .where(eq(serviceBindings.resourceId, resourceId))
+    .get();
+  return { count: result?.count ?? 0 };
+}
+
+export async function createServiceBinding(
+  db: D1Database,
+  input: {
+    id: string;
+    service_id: string;
+    resource_id: string;
+    binding_name: string;
+    binding_type: string;
+    config: Record<string, unknown>;
+    created_at: string;
+  }
+) {
+  const drizzle = getDb(db);
+  await drizzle.insert(serviceBindings).values({
+    id: input.id,
+    serviceId: input.service_id,
+    resourceId: input.resource_id,
+    bindingName: input.binding_name,
+    bindingType: input.binding_type,
+    config: JSON.stringify(input.config || {}),
+    createdAt: input.created_at,
+  });
+}
+
+export async function deleteServiceBinding(db: D1Database, resourceId: string, serviceId: string) {
+  const drizzle = getDb(db);
+  await drizzle.delete(serviceBindings)
+    .where(and(
+      eq(serviceBindings.resourceId, resourceId),
+      eq(serviceBindings.serviceId, serviceId),
+    ));
+}
+
+export const listResourceBindings = listServiceBindings;
+export const countResourceBindings = countServiceBindings;
+export const createWorkerBinding = createServiceBinding;
+export const deleteWorkerBinding = deleteServiceBinding;
+
+export async function buildBindingFromResource(
+  db: D1Database,
+  resourceId: string,
+  bindingName: string
+): Promise<{
+  type: 'd1' | 'r2' | 'kv' | 'queue' | 'analytics_engine' | 'workflow' | 'vectorize' | 'durable_object_namespace';
+  name: string;
+  id?: string;
+  bucket_name?: string;
+  namespace_id?: string;
+  queue_name?: string;
+  dataset?: string;
+  workflow_name?: string;
+  index_name?: string;
+  class_name?: string;
+  script_name?: string;
+} | null> {
+  const resource = await getResourceById(db, resourceId);
+
+  if (!resource || resource.status !== 'active') {
+    return null;
+  }
+
+  switch (resource.type) {
+    case 'd1':
+      return {
+        type: 'd1',
+        name: bindingName,
+        id: resource.cf_id || undefined,
+      };
+
+    case 'r2':
+      return {
+        type: 'r2',
+        name: bindingName,
+        bucket_name: resource.cf_name || undefined,
+      };
+
+    case 'kv':
+      return {
+        type: 'kv',
+        name: bindingName,
+        namespace_id: resource.cf_id || undefined,
+      };
+
+    case 'vectorize':
+      return {
+        type: 'vectorize',
+        name: bindingName,
+        index_name: resource.cf_name || undefined,
+      };
+    case 'queue':
+      return {
+        type: 'queue',
+        name: bindingName,
+        queue_name: resource.cf_name || resource.cf_id || undefined,
+      };
+    case 'analytics_engine':
+    case 'analyticsEngine':
+      return {
+        type: 'analytics_engine',
+        name: bindingName,
+        dataset: resource.cf_name || resource.cf_id || undefined,
+      };
+    case 'workflow':
+      return {
+        type: 'workflow',
+        name: bindingName,
+        workflow_name: resource.cf_name || resource.cf_id || undefined,
+      };
+
+    case 'durable_object': {
+      let config: Record<string, unknown> = {};
+      if (resource.config) {
+        try {
+          config = (typeof resource.config === 'string' ? JSON.parse(resource.config) : resource.config) as Record<string, unknown>;
+        } catch {
+          config = {};
+        }
+      }
+      const className = (config.className as string) || resource.cf_name || undefined;
+      if (!className) return null;
+      const scriptName = config.scriptName as string | undefined;
+      return {
+        type: 'durable_object_namespace',
+        name: bindingName,
+        class_name: className,
+        ...(scriptName ? { script_name: scriptName } : {}),
+      };
+    }
+
+    default:
+      return null;
+  }
+}

package/src/application/services/resources/format.ts

@@ -0,0 +1,76 @@
+import type { Resource, ResourceAccess, ResourcePermission, ResourceStatus, ResourceType, ServiceBinding } from '../../../shared/types';
+import { toIsoString } from '../../../shared/utils';
+
+export function toApiResource(r: {
+  id: string;
+  ownerId: string;
+  spaceId: string | null;
+  name: string;
+  type: string;
+  status: string;
+  cfId: string | null;
+  cfName: string | null;
+  config: string;
+  metadata: string;
+  sizeBytes: number | null;
+  itemCount: number | null;
+  lastUsedAt: string | Date | null;
+  createdAt: string | Date;
+  updatedAt: string | Date;
+}): Resource {
+  return {
+    id: r.id,
+    owner_id: r.ownerId,
+    space_id: r.spaceId,
+    name: r.name,
+    type: r.type as ResourceType,
+    status: r.status as ResourceStatus,
+    cf_id: r.cfId,
+    cf_name: r.cfName,
+    config: r.config,
+    metadata: r.metadata,
+    size_bytes: r.sizeBytes,
+    item_count: r.itemCount,
+    last_used_at: toIsoString(r.lastUsedAt),
+    created_at: toIsoString(r.createdAt) ?? new Date(0).toISOString(),
+    updated_at: toIsoString(r.updatedAt) ?? new Date(0).toISOString(),
+  } as Resource;
+}
+
+export function toApiResourceAccess(ra: {
+  id: string;
+  resourceId: string;
+  accountId: string;
+  permission: string;
+  grantedByAccountId: string | null;
+  createdAt: string | Date;
+}): ResourceAccess {
+  return {
+    id: ra.id,
+    resource_id: ra.resourceId,
+    space_id: ra.accountId,
+    permission: ra.permission as ResourcePermission,
+    granted_by: ra.grantedByAccountId,
+    created_at: toIsoString(ra.createdAt) ?? new Date(0).toISOString(),
+  } as ResourceAccess;
+}
+
+export function toApiServiceBinding(wb: {
+  id: string;
+  serviceId: string;
+  resourceId: string;
+  bindingName: string;
+  bindingType: string;
+  config: string;
+  createdAt: string | Date;
+}): ServiceBinding {
+  return {
+    id: wb.id,
+    service_id: wb.serviceId,
+    resource_id: wb.resourceId,
+    binding_name: wb.bindingName,
+    binding_type: wb.bindingType,
+    config: wb.config,
+    created_at: toIsoString(wb.createdAt) ?? new Date(0).toISOString(),
+  } as ServiceBinding;
+}