takos-control 1.0.0

package/src/application/services/source/info-units.ts

@@ -0,0 +1,383 @@
+import type { Ai, VectorizeIndex, D1Database, R2Bucket } from '../../../shared/types/bindings.ts';
+import type { Env } from '../../../shared/types';
+import { getDb, runs, runEvents, infoUnits, nodes, edges, sessionRepos, repositories } from '../../../infra/db';
+import { eq, and, asc } from 'drizzle-orm';
+import { generateId, now, toIsoString } from '../../../shared/utils';
+import { getRunEventsAfterFromR2 } from '../offload/run-events';
+import { logWarn } from '../../../shared/utils/logger';
+
+
+import { EMBEDDING_MODEL } from '../../../shared/config/limits.ts';
+const MAX_INFO_UNIT_TOKENS = 2048;
+const MAX_EVENT_TEXT_CHARS = 4000;
+const CHARS_PER_TOKEN = 4;
+
+type RunEventRecord = {
+  id: number;
+  type: string;
+  data: string;
+  createdAt: string;
+};
+
+function estimateTokens(text: string): number {
+  return Math.ceil(text.length / CHARS_PER_TOKEN);
+}
+
+function truncateText(text: string, maxChars: number): string {
+  if (text.length <= maxChars) return text;
+  return text.slice(0, maxChars) + `... [truncated:${text.length} chars]`;
+}
+
+function stringifySafe(value: unknown): string {
+  if (typeof value === 'string') return value;
+  try {
+    return JSON.stringify(value);
+  } catch {
+    return String(value);
+  }
+}
+
+function formatEvent(event: RunEventRecord): string | null {
+  let data: Record<string, unknown> = {};
+  try {
+    data = JSON.parse(event.data);
+  } catch {
+    data = {};
+  }
+
+  if (event.type === 'message') {
+    const content = data.content || data.text || data.message;
+    if (!content) return null;
+    return `[assistant] ${truncateText(String(content), MAX_EVENT_TEXT_CHARS)}`;
+  }
+
+  if (event.type === 'tool_call') {
+    const tool = data.tool || data.name || 'unknown';
+    const args = data.arguments || data.args || {};
+    return `[tool_call] ${tool} ${truncateText(stringifySafe(args), MAX_EVENT_TEXT_CHARS)}`;
+  }
+
+  if (event.type === 'tool_result') {
+    const tool = data.tool || data.name || 'unknown';
+    const output = data.output || data.result || data.error || '';
+    return `[tool_result] ${tool} ${truncateText(stringifySafe(output), MAX_EVENT_TEXT_CHARS)}`;
+  }
+
+  if (event.type === 'error') {
+    const error = data.error || data.message || 'unknown error';
+    return `[error] ${truncateText(String(error), MAX_EVENT_TEXT_CHARS)}`;
+  }
+
+  if (event.type === 'progress') {
+    const message = data.message || data.status;
+    if (!message) return null;
+    return `[progress] ${truncateText(String(message), MAX_EVENT_TEXT_CHARS)}`;
+  }
+
+  return null;
+}
+
+function buildSegments(entries: string[]): string[] {
+  const segments: string[] = [];
+  let current = '';
+
+  for (const entry of entries) {
+    const next = current ? `${current}\n${entry}` : entry;
+    if (estimateTokens(next) > MAX_INFO_UNIT_TOKENS && current) {
+      segments.push(current);
+      current = entry;
+      continue;
+    }
+    current = next;
+  }
+
+  if (current) {
+    segments.push(current);
+  }
+
+  return segments;
+}
+
+async function upsertVector(
+  ai: Ai,
+  vectorize: VectorizeIndex,
+  infoUnitId: string,
+  spaceId: string,
+  runId: string | null,
+  threadId: string | null,
+  content: string,
+  segmentIndex: number,
+  segmentCount: number,
+  repoIds?: string[]
+): Promise<string> {
+  const result = await ai.run(EMBEDDING_MODEL, {
+    text: [content],
+  }) as { data: number[][] };
+
+  if (!result.data || result.data.length === 0) {
+    throw new Error('Failed to generate embedding');
+  }
+
+  const vectorId = `info_unit:${spaceId}:${infoUnitId}`;
+  await vectorize.upsert([{
+    id: vectorId,
+    values: result.data[0],
+    metadata: {
+      kind: 'info_unit',
+      spaceId,
+      ...(runId ? { runId } : {}),
+      ...(threadId ? { threadId } : {}),
+      segmentIndex,
+      segmentCount,
+      repoIds: repoIds || [],
+      content: content.slice(0, 1000),
+    },
+  }]);
+
+  return vectorId;
+}
+
+async function ensureNode(
+  dbBinding: D1Database,
+  spaceId: string,
+  type: string,
+  refId: string,
+  label?: string,
+  metadata?: Record<string, unknown>
+): Promise<string> {
+  const db = getDb(dbBinding);
+  const existing = await db.select({ id: nodes.id })
+    .from(nodes)
+    .where(and(
+      eq(nodes.accountId, spaceId),
+      eq(nodes.type, type),
+      eq(nodes.refId, refId),
+    ))
+    .get();
+
+  if (existing) {
+    return existing.id;
+  }
+
+  const id = generateId();
+  await db.insert(nodes).values({
+    id,
+    accountId: spaceId,
+    type,
+    refId,
+    label: label || null,
+    metadata: JSON.stringify(metadata || {}),
+    createdAt: now(),
+  });
+
+  return id;
+}
+
+async function ensureEdge(
+  dbBinding: D1Database,
+  spaceId: string,
+  sourceId: string,
+  targetId: string,
+  type: string,
+  metadata?: Record<string, unknown>
+): Promise<void> {
+  const db = getDb(dbBinding);
+  const existing = await db.select({ id: edges.id })
+    .from(edges)
+    .where(and(
+      eq(edges.accountId, spaceId),
+      eq(edges.sourceId, sourceId),
+      eq(edges.targetId, targetId),
+      eq(edges.type, type),
+    ))
+    .get();
+
+  if (existing) return;
+
+  await db.insert(edges).values({
+    id: generateId(),
+    accountId: spaceId,
+    sourceId,
+    targetId,
+    type,
+    weight: 1.0,
+    metadata: JSON.stringify(metadata || {}),
+    createdAt: now(),
+  });
+}
+
+export class InfoUnitIndexer {
+  private ai?: Ai;
+  private vectorize?: VectorizeIndex;
+  private dbBinding: D1Database;
+  private offloadBucket?: R2Bucket;
+
+  constructor(env: Pick<Env, 'AI' | 'VECTORIZE' | 'DB' | 'TAKOS_OFFLOAD'>) {
+    this.ai = env.AI;
+    this.vectorize = env.VECTORIZE;
+    this.dbBinding = env.DB;
+    this.offloadBucket = env.TAKOS_OFFLOAD;
+  }
+
+  async indexRun(spaceId: string, runId: string): Promise<void> {
+    const db = getDb(this.dbBinding);
+
+    const run = await db.select({
+      id: runs.id,
+      accountId: runs.accountId,
+      threadId: runs.threadId,
+      sessionId: runs.sessionId,
+      status: runs.status,
+      startedAt: runs.startedAt,
+      completedAt: runs.completedAt,
+    }).from(runs)
+      .where(eq(runs.id, runId))
+      .get();
+
+    if (!run || run.accountId !== spaceId) {
+      return;
+    }
+
+    const existing = await db.select({ id: infoUnits.id })
+      .from(infoUnits)
+      .where(eq(infoUnits.runId, runId))
+      .get();
+    if (existing) return;
+
+    const events = this.offloadBucket
+      ? (await getRunEventsAfterFromR2(this.offloadBucket, runId, 0, 5000)).map((e) => ({
+          id: e.event_id,
+          type: e.type,
+          data: e.data,
+          createdAt: e.created_at,
+        }))
+      : (await db.select().from(runEvents)
+          .where(eq(runEvents.runId, runId))
+          .orderBy(asc(runEvents.id))
+          .all()
+        ).map((event) => ({
+          ...event,
+          createdAt: toIsoString(event.createdAt) ?? new Date(0).toISOString(),
+        }));
+
+    const entries = events
+      .map(formatEvent)
+      .filter((entry): entry is string => Boolean(entry));
+
+    const fallback = entries.length === 0
+      ? `[summary] run ${runId} (${run.status})`
+      : null;
+
+    if (fallback) {
+      entries.push(fallback);
+    }
+
+    const segments = buildSegments(entries);
+    const segmentCount = Math.max(1, segments.length);
+    const createdAt = now();
+
+    const sessionRepoResults = run.sessionId
+      ? await db.select({
+          repoId: sessionRepos.repoId,
+          branch: sessionRepos.branch,
+          mountPath: sessionRepos.mountPath,
+          isPrimary: sessionRepos.isPrimary,
+          repoName: repositories.name,
+        }).from(sessionRepos)
+          .leftJoin(repositories, eq(sessionRepos.repoId, repositories.id))
+          .where(eq(sessionRepos.sessionId, run.sessionId))
+          .all()
+      : [];
+    const repoMetadata = sessionRepoResults.map((repo) => ({
+      repo_id: repo.repoId,
+      repo_name: repo.repoName,
+      branch: repo.branch,
+      mount_path: repo.mountPath,
+      is_primary: repo.isPrimary,
+    }));
+
+    for (let index = 0; index < segments.length; index++) {
+      const content = segments[index];
+      const tokenCount = estimateTokens(content);
+      const infoUnitId = generateId();
+
+      let vectorId: string | null = null;
+      if (this.ai && this.vectorize) {
+        try {
+          vectorId = await upsertVector(
+            this.ai,
+            this.vectorize,
+            infoUnitId,
+            spaceId,
+            runId,
+            run.threadId,
+            content,
+            index,
+            segmentCount,
+            repoMetadata.map((repo) => repo.repo_id)
+          );
+        } catch (err) {
+          logWarn(`Embedding failed for run ${runId}`, { module: 'info_unit', detail: err });
+        }
+      }
+
+      await db.insert(infoUnits).values({
+        id: infoUnitId,
+        accountId: spaceId,
+        threadId: run.threadId,
+        runId,
+        sessionId: run.sessionId,
+        kind: segmentCount > 1 ? 'segment' : 'session',
+        title: `Run ${runId} (${run.status})`,
+        content,
+        tokenCount,
+        segmentIndex: index,
+        segmentCount,
+        vectorId,
+        metadata: JSON.stringify({
+          run_status: run.status,
+          started_at: run.startedAt,
+          completed_at: run.completedAt,
+          segment_index: index,
+          segment_count: segmentCount,
+          repos: repoMetadata,
+        }),
+        createdAt,
+        updatedAt: createdAt,
+      });
+
+      const infoNodeId = await ensureNode(
+        this.dbBinding,
+        spaceId,
+        'info_unit',
+        infoUnitId,
+        `Run ${runId} (${index + 1}/${segmentCount})`,
+        { runId, segmentIndex: index, segmentCount, repos: repoMetadata }
+      );
+
+      if (run.threadId) {
+        const threadNodeId = await ensureNode(
+          this.dbBinding,
+          spaceId,
+          'thread',
+          run.threadId,
+          `Thread ${run.threadId}`,
+          {}
+        );
+        await ensureEdge(
+          this.dbBinding,
+          spaceId,
+          infoNodeId,
+          threadNodeId,
+          'generated_from',
+          { runId }
+        );
+      }
+    }
+  }
+}
+
+export function createInfoUnitIndexer(env: Pick<Env, 'AI' | 'VECTORIZE' | 'DB' | 'TAKOS_OFFLOAD'>): InfoUnitIndexer | null {
+  if (!env.DB) return null;
+  return new InfoUnitIndexer(env);
+}

package/src/application/services/source/official-packages.ts

@@ -0,0 +1,50 @@
+/**
+ * Official Packages
+ *
+ * First-party packages that always appear in the store catalog.
+ * These are defined in code (not DB) so they're always available
+ * regardless of database state.
+ *
+ * Similar pattern to official-skills.ts.
+ */
+
+export interface OfficialPackage {
+  /** Unique identifier */
+  id: string;
+  /** Package name */
+  name: string;
+  /** Short description */
+  description: string;
+  /** Category for filtering */
+  category: 'app' | 'service' | 'library' | 'template' | 'tool';
+  /** Git clone URL */
+  url: string;
+  /** Owner display info */
+  owner: {
+    name: string;
+    username: string;
+  };
+  /** Tags for search/filtering */
+  tags: string[];
+  /** Whether this is recommended for new users */
+  recommended: boolean;
+  /** Display priority (higher = shown first) */
+  priority: number;
+}
+
+export const OFFICIAL_PACKAGES: OfficialPackage[] = [
+  {
+    id: 'official/takos-computer',
+    name: 'Takos Computer',
+    description: 'Browser automation and agent executor',
+    category: 'tool',
+    url: 'https://github.com/tako0614/takos-computer.git',
+    owner: {
+      name: 'Takos',
+      username: 'takos',
+    },
+    tags: ['browser', 'executor', 'agent', 'playwright', 'automation'],
+    recommended: true,
+    priority: 100,
+  },
+];

package/src/application/services/source/repo-release-assets.ts

@@ -0,0 +1,55 @@
+import { safeJsonParseOrDefault, toIsoString } from '../../../shared/utils';
+
+export interface ReleaseAsset {
+  id: string;
+  name: string;
+  content_type: string;
+  size: number;
+  r2_key: string;
+  download_count: number;
+  bundle_format?: string;
+  bundle_meta?: {
+    name?: string;
+    app_id?: string;
+    version: string;
+    description?: string;
+    icon?: string;
+    category?: 'app' | 'service' | 'library' | 'template' | 'social';
+    tags?: string[];
+    dependencies?: Array<{ repo: string; version: string }>;
+  };
+  created_at: string;
+}
+
+type ReleaseAssetRow = {
+  id: string;
+  assetKey: string;
+  name: string;
+  contentType: string | null;
+  sizeBytes: number | null;
+  downloadCount: number;
+  bundleFormat: string | null;
+  bundleMetaJson: string | null;
+  createdAt: string | Date;
+};
+
+export function toReleaseAsset(row: ReleaseAssetRow): ReleaseAsset {
+  const bundleMeta = row.bundleMetaJson
+    ? safeJsonParseOrDefault<ReleaseAsset['bundle_meta'] | undefined>(row.bundleMetaJson, undefined)
+    : undefined;
+  return {
+    id: row.id,
+    name: row.name,
+    content_type: row.contentType || 'application/octet-stream',
+    size: row.sizeBytes ?? 0,
+    r2_key: row.assetKey,
+    download_count: row.downloadCount,
+    bundle_format: row.bundleFormat || undefined,
+    bundle_meta: bundleMeta,
+    created_at: toIsoString(row.createdAt),
+  };
+}
+
+export function toReleaseAssets(rows: ReleaseAssetRow[]): ReleaseAsset[] {
+  return rows.map(toReleaseAsset);
+}

package/src/application/services/source/repos.ts

@@ -0,0 +1,231 @@
+import type { D1Database, R2Bucket } from '../../../shared/types/bindings.ts';
+import type { Env, Repository, RepositoryVisibility, SpaceRole } from '../../../shared/types';
+import type { SelectOf } from '../../../shared/types/drizzle-helpers';
+import { checkSpaceAccess } from '../../../shared/utils';
+import { getDb, accounts, repositories } from '../../../infra/db';
+import { and, desc, eq } from 'drizzle-orm';
+import { isValidOpaqueId } from '../../../shared/utils/db-guards';
+import { generateId, now, sanitizeRepoName } from '../../../shared/utils';
+import * as gitStore from '../git-smart';
+import { logError } from '../../../shared/utils/logger';
+
+export interface RepoAccess {
+  repo: Repository;
+  spaceId: string;
+  role: SpaceRole;
+}
+
+export interface CheckRepoAccessOptions {
+  allowPublicRead?: boolean;
+}
+
+export interface CreateRepositoryInput {
+  spaceId: string;
+  name: string;
+  description?: string | null;
+  visibility?: RepositoryVisibility | 'internal';
+  actorAccountId?: string;
+}
+
+export class RepositoryCreationError extends Error {
+  constructor(
+    message: string,
+    public readonly code:
+      | 'INVALID_NAME'
+      | 'SPACE_NOT_FOUND'
+      | 'REPOSITORY_EXISTS'
+      | 'GIT_STORAGE_NOT_CONFIGURED'
+      | 'INIT_FAILED',
+  ) {
+    super(message);
+    this.name = 'RepositoryCreationError';
+  }
+}
+
+function toRepositoryVisibility(value: string): RepositoryVisibility {
+  return value === 'public' ? 'public' : 'private';
+}
+
+type RepositoryRow = SelectOf<typeof repositories>;
+
+export function toApiRepositoryFromDb(row: RepositoryRow): Repository {
+  const repository = {
+    id: row.id,
+    space_id: row.accountId,
+    name: row.name,
+    description: row.description,
+    visibility: toRepositoryVisibility(row.visibility),
+    default_branch: row.defaultBranch,
+    forked_from_id: row.forkedFromId,
+    stars: row.stars,
+    forks: row.forks,
+    git_enabled: row.gitEnabled,
+    is_official: row.isOfficial,
+    official_category: row.officialCategory,
+    official_maintainer: row.officialMaintainer,
+    featured: row.featured,
+    install_count: row.installCount,
+    created_at: row.createdAt,
+    updated_at: row.updatedAt,
+  };
+
+  return repository;
+}
+
+export async function checkRepoAccess(
+  env: Env,
+  repoId: string,
+  userId: string | null | undefined,
+  requiredRoles?: SpaceRole[],
+  options: CheckRepoAccessOptions = {},
+): Promise<RepoAccess | null> {
+  if (!isValidOpaqueId(repoId)) return null;
+
+  const normalizedUserId = typeof userId === 'string' && isValidOpaqueId(userId)
+    ? userId
+    : null;
+
+  const drizzle = getDb(env.DB);
+  const row = await drizzle.select().from(repositories).where(eq(repositories.id, repoId)).get();
+  const repo = row ? toApiRepositoryFromDb(row) : null;
+
+  if (!repo) return null;
+
+  if (normalizedUserId) {
+    const access = await checkSpaceAccess(env.DB, repo.space_id, normalizedUserId, requiredRoles);
+    if (access) {
+      return { repo, spaceId: repo.space_id, role: access.membership.role };
+    }
+  }
+
+  if (options.allowPublicRead && !requiredRoles && repo.visibility === 'public') {
+    return { repo, spaceId: repo.space_id, role: 'viewer' };
+  }
+
+  return null;
+}
+
+export async function getRepositoryById(db: D1Database, repoId: string): Promise<Repository | null> {
+  if (!isValidOpaqueId(repoId)) return null;
+
+  const drizzle = getDb(db);
+  const row = await drizzle.select().from(repositories).where(eq(repositories.id, repoId)).get();
+  return row ? toApiRepositoryFromDb(row) : null;
+}
+
+export async function listRepositoriesBySpace(db: D1Database, spaceId: string): Promise<Repository[]> {
+  const drizzle = getDb(db);
+  const rows = await drizzle.select().from(repositories)
+    .where(eq(repositories.accountId, spaceId))
+    .orderBy(desc(repositories.updatedAt))
+    .all();
+  return rows.map(toApiRepositoryFromDb);
+}
+
+async function resolveRepositoryInitActor(
+  db: ReturnType<typeof getDb>,
+  actorAccountId?: string,
+): Promise<{ name: string; email: string }> {
+  if (!actorAccountId) {
+    return {
+      name: 'Takos Agent',
+      email: 'agent@users.takos.local',
+    };
+  }
+
+  const actor = await db.select({
+    name: accounts.name,
+    slug: accounts.slug,
+    email: accounts.email,
+  }).from(accounts).where(eq(accounts.id, actorAccountId)).get();
+
+  if (!actor) {
+    return {
+      name: 'Takos Agent',
+      email: 'agent@users.takos.local',
+    };
+  }
+
+  const fallbackLocalPart = actor.slug?.trim() || actorAccountId;
+
+  return {
+    name: actor.name || 'Takos Agent',
+    email: actor.email || `${fallbackLocalPart}@users.takos.local`,
+  };
+}
+
+export async function createRepository(
+  dbBinding: D1Database,
+  bucket: R2Bucket | undefined,
+  input: CreateRepositoryInput,
+): Promise<Repository> {
+  const db = getDb(dbBinding);
+  const name = sanitizeRepoName(input.name);
+
+  if (!name) {
+    throw new RepositoryCreationError('Invalid repository name', 'INVALID_NAME');
+  }
+
+  const workspace = await db.select({ id: accounts.id })
+    .from(accounts)
+    .where(eq(accounts.id, input.spaceId))
+    .get();
+
+  if (!workspace) {
+    throw new RepositoryCreationError('Workspace not found', 'SPACE_NOT_FOUND');
+  }
+
+  const existing = await db.select({ id: repositories.id })
+    .from(repositories)
+    .where(and(
+      eq(repositories.accountId, input.spaceId),
+      eq(repositories.name, name),
+    ))
+    .get();
+
+  if (existing) {
+    throw new RepositoryCreationError('Repository with this name already exists', 'REPOSITORY_EXISTS');
+  }
+
+  if (!bucket) {
+    throw new RepositoryCreationError('Git storage not configured', 'GIT_STORAGE_NOT_CONFIGURED');
+  }
+
+  const id = generateId();
+  const timestamp = now();
+  const actor = await resolveRepositoryInitActor(db, input.actorAccountId);
+
+  await db.insert(repositories).values({
+    id,
+    accountId: input.spaceId,
+    name,
+    description: input.description || null,
+    visibility: input.visibility || 'private',
+    defaultBranch: 'main',
+    stars: 0,
+    forks: 0,
+    gitEnabled: true,
+    createdAt: timestamp,
+    updatedAt: timestamp,
+  });
+
+  try {
+    await gitStore.initRepository(dbBinding, bucket, id, 'main', {
+      name: actor.name,
+      email: actor.email,
+      timestamp: Math.floor(new Date(timestamp).getTime() / 1000),
+      tzOffset: '+0000',
+    });
+  } catch (error) {
+    await db.delete(repositories).where(eq(repositories.id, id));
+    logError('Failed to initialize repository', error, { module: 'services/source/repos' });
+    throw new RepositoryCreationError('Failed to initialize repository', 'INIT_FAILED');
+  }
+
+  const row = await db.select().from(repositories).where(eq(repositories.id, id)).get();
+  if (!row) {
+    throw new RepositoryCreationError('Failed to create repository', 'INIT_FAILED');
+  }
+
+  return toApiRepositoryFromDb(row);
+}