ttsc 0.32.1 → 0.33.0-next.1

package/cmd/ttsc-cache/account.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+  "errors"
+  "fmt"
+  "io"
+  "os"
+  "path/filepath"
+)
+
+type artifactAccounting struct {
+  BaseBytes    int64 `json:"baseBytes"`
+  OverlayBytes int64 `json:"overlayBytes"`
+  StagingBytes int64 `json:"stagingBytes"`
+  PinnedBytes  int64 `json:"pinnedBytes"`
+  BaseFiles    int64 `json:"baseFiles"`
+  OverlayFiles int64 `json:"overlayFiles"`
+  StagingFiles int64 `json:"stagingFiles"`
+}
+
+// accountArtifactStore reads bounded rooted metadata for every retained cache
+// population. The byte totals are logical file lengths, not free-space or
+// physical allocated-byte claims; concurrent writers can change them later.
+func accountArtifactStore(store *artifactStore) (artifactAccounting, error) {
+  var result artifactAccounting
+  var entries int
+  var walk func(string, int, *int64, *int64) error
+  walk = func(directory string, depth int, bytes, files *int64) error {
+    if depth > 4 {
+      return errors.New("artifact storage exceeds directory depth policy")
+    }
+    opened, err := store.root.Open(directory)
+    if err != nil {
+      return err
+    }
+    defer opened.Close()
+    for {
+      batch, readErr := opened.ReadDir(256)
+      for _, entry := range batch {
+        entries++
+        if entries > 100000 {
+          return errors.New("artifact storage exceeds entry count policy")
+        }
+        child := filepath.ToSlash(filepath.Join(directory, entry.Name()))
+        info, err := store.root.Lstat(child)
+        if err != nil {
+          return err
+        }
+        if info.IsDir() && info.Mode()&os.ModeSymlink == 0 {
+          if err := walk(child, depth+1, bytes, files); err != nil {
+            return err
+          }
+        } else if info.Mode().IsRegular() {
+          if info.Size() < 0 || info.Size() > 1<<40 || *bytes > (1<<50)-info.Size() {
+            return fmt.Errorf("artifact storage has an invalid file size: %s", child)
+          }
+          *bytes += info.Size()
+          *files += 1
+        } else {
+          return fmt.Errorf("artifact storage contains an aliased or unknown entry: %s", child)
+        }
+      }
+      if errors.Is(readErr, io.EOF) {
+        return nil
+      }
+      if readErr != nil {
+        return readErr
+      }
+    }
+  }
+  for _, part := range []struct {
+    path  string
+    bytes *int64
+    files *int64
+  }{
+    {"base/packs", &result.BaseBytes, &result.BaseFiles},
+    {"base/staging", &result.StagingBytes, &result.StagingFiles},
+    {"overlay/bodies", &result.OverlayBytes, &result.OverlayFiles},
+    {"overlay/actions", &result.OverlayBytes, &result.OverlayFiles},
+    {"overlay/staging", &result.StagingBytes, &result.StagingFiles},
+  } {
+    if err := walk(part.path, 0, part.bytes, part.files); err != nil {
+      return artifactAccounting{}, err
+    }
+  }
+  for _, pinned := range store.pins {
+    info, err := pinned.Stat()
+    if err != nil || !info.Mode().IsRegular() || info.Size() < 0 || result.PinnedBytes > (1<<50)-info.Size() {
+      return artifactAccounting{}, errors.New("artifact pinned body accounting failed")
+    }
+    result.PinnedBytes += info.Size()
+  }
+  return result, nil
+}

package/cmd/ttsc-cache/interruption_test.go

@@ -0,0 +1,176 @@
+package main
+
+import (
+  "bufio"
+  "bytes"
+  "crypto/sha256"
+  "encoding/base64"
+  "encoding/hex"
+  "encoding/json"
+  "errors"
+  "fmt"
+  "io"
+  "os"
+  "os/exec"
+  "path/filepath"
+  "strings"
+  "testing"
+  "time"
+)
+
+// TestCacheHelperInterruption verifies that an interrupted protocol operation cannot publish an incomplete action.
+//
+// Verifies: killing a helper during a streamed put leaves the new action absent, and a broken get response leaves an older complete action readable.
+// 1. Seed a complete action, start the production helper protocol, and send only the beginning of a larger put body.
+// 2. Kill the helper after its staging file appears, then reopen the store and compare the old hit and interrupted miss.
+// 3. Break the response pipe during a get and reopen the store to require the same complete hit.
+//
+// @evidence contracts/testing.md#behavioral-verification Runs the production helper as a child through its public protocol, kills it during a partial streamed put, and interrupts the direct server's response writer on a real get.
+// @evidence contracts/testing.md#independent-expectations An action mapping requires a complete body matching its SHA-256 OutputID; a partly received body has no valid mapping, while an earlier completed mapping must survive both failures.
+// @evidence contracts/testing.md#distinguishing-cases Covers process death after a partial body is sent and a get response failure with a prior valid hit and a new missing action; stock Go's reaction to a bad helper is owned by TestStockGoCacheFailures.
+// @evidence contracts/testing.md#execution-ownership This Go test uses a temporary local helper process and store; it does not install a consumer or run a product E2E build.
+func TestCacheHelperInterruption(t *testing.T) {
+  root := t.TempDir()
+  oldAction := bytes.Repeat([]byte{0x31}, 32)
+  newAction := bytes.Repeat([]byte{0x32}, 32)
+  oldBody := []byte("completed archive")
+  oldOutput := sha256.Sum256(oldBody)
+  initial, err := openArtifactStore(root, nil)
+  if err != nil {
+    t.Fatal(err)
+  }
+  if _, err := initial.put(hex.EncodeToString(oldAction), hex.EncodeToString(oldOutput[:]), int64(len(oldBody)), bytes.NewReader(oldBody)); err != nil {
+    t.Fatal(err)
+  }
+  if err := initial.close(); err != nil {
+    t.Fatal(err)
+  }
+
+  helper := filepath.Join(t.TempDir(), "cache helper"+exeSuffix())
+  build := exec.Command("go", "build", "-trimpath", "-o", helper, ".")
+  build.Env = append(os.Environ(), "GOCACHEPROG=", "GOTOOLCHAIN=local")
+  if output, err := build.CombinedOutput(); err != nil {
+    t.Fatalf("building cache helper: %v\n%s", err, output)
+  }
+  command := exec.Command(helper, "-root", root)
+  stdin, err := command.StdinPipe()
+  if err != nil {
+    t.Fatal(err)
+  }
+  stdout, err := command.StdoutPipe()
+  if err != nil {
+    t.Fatal(err)
+  }
+  var stderr bytes.Buffer
+  command.Stderr = &stderr
+  if err := command.Start(); err != nil {
+    t.Fatal(err)
+  }
+  defer func() {
+    if command.ProcessState == nil {
+      command.Process.Kill()
+      command.Wait()
+    }
+  }()
+  startup, err := bufio.NewReader(stdout).ReadBytes('\n')
+  if err != nil {
+    t.Fatalf("reading helper startup: %v; stderr: %s", err, &stderr)
+  }
+  var capability response
+  if err := json.Unmarshal(startup, &capability); err != nil || fmt.Sprint(capability.KnownCommands) != "[get put close]" {
+    t.Fatalf("helper did not advertise the Go cache protocol: %s; %v", startup, err)
+  }
+  body := bytes.Repeat([]byte("interrupted archive body"), 50000)
+  outputID := sha256.Sum256(body)
+  header, err := json.Marshal(request{ID: 1, Command: "put", ActionID: newAction, OutputID: outputID[:], BodySize: int64(len(body))})
+  if err != nil {
+    t.Fatal(err)
+  }
+  if _, err := fmt.Fprintf(stdin, "%s\n\n\"%s", header, base64.StdEncoding.EncodeToString(body[:98304])); err != nil {
+    t.Fatalf("sending partial Go cache body: %v", err)
+  }
+  staging := filepath.Join(root, "overlay", "staging")
+  deadline := time.Now().Add(10 * time.Second)
+  staged := false
+  for time.Now().Before(deadline) {
+    entries, err := os.ReadDir(staging)
+    if err != nil {
+      t.Fatal(err)
+    }
+    staged = len(entries) > 0
+    if staged {
+      break
+    }
+    time.Sleep(10 * time.Millisecond)
+  }
+  if !staged {
+    entries, _ := os.ReadDir(staging)
+    command.Process.Kill()
+    command.Wait()
+    remaining, _ := io.ReadAll(stdout)
+    t.Fatalf("helper never opened a put staging file: entries=%d remaining=%q stderr=%s", len(entries), remaining, &stderr)
+  }
+  if err := command.Process.Kill(); err != nil {
+    t.Fatal(err)
+  }
+  command.Wait()
+  stdin.Close()
+  stdout.Close()
+
+  reopened, err := openArtifactStore(root, nil)
+  if err != nil {
+    t.Fatal(err)
+  }
+  _, _, found, err := reopened.get(hex.EncodeToString(newAction))
+  if err != nil || found {
+    t.Fatalf("interrupted put became a hit: found=%v error=%v", found, err)
+  }
+  oldValue, oldPath, found, err := reopened.get(hex.EncodeToString(oldAction))
+  if err != nil || !found || oldValue.OutputID != hex.EncodeToString(oldOutput[:]) {
+    t.Fatalf("interrupted put damaged the complete action: value=%+v found=%v error=%v", oldValue, found, err)
+  }
+  if raw, err := os.ReadFile(oldPath); err != nil || !bytes.Equal(raw, oldBody) {
+    t.Fatalf("complete body changed after interruption: %q %v", raw, err)
+  }
+  if err := reopened.close(); err != nil {
+    t.Fatal(err)
+  }
+
+  interruptedGet, err := openArtifactStore(root, nil)
+  if err != nil {
+    t.Fatal(err)
+  }
+  getHeader, err := json.Marshal(request{ID: 2, Command: "get", ActionID: oldAction})
+  if err != nil {
+    t.Fatal(err)
+  }
+  broken := &failSecondResponse{}
+  err = serve(strings.NewReader(string(getHeader)+"\n"), broken, interruptedGet)
+  if !errors.Is(err, errBrokenResponse) {
+    t.Fatalf("interrupted get did not report its broken response: %v", err)
+  }
+  if err := interruptedGet.close(); err != nil {
+    t.Fatal(err)
+  }
+  settled, err := openArtifactStore(root, nil)
+  if err != nil {
+    t.Fatal(err)
+  }
+  defer settled.close()
+  value, location, found, err := settled.get(hex.EncodeToString(oldAction))
+  if err != nil || !found || value.OutputID != hex.EncodeToString(oldOutput[:]) {
+    t.Fatalf("interrupted get damaged the complete action: value=%+v path=%q found=%v error=%v", value, location, found, err)
+  }
+}
+
+var errBrokenResponse = errors.New("client response pipe closed")
+
+type failSecondResponse struct{ writes int }
+
+func (writer *failSecondResponse) Write(value []byte) (int, error) {
+  writer.writes++
+  if writer.writes == 2 {
+    return 0, errBrokenResponse
+  }
+  return len(value), nil
+}

package/cmd/ttsc-cache/main.go

@@ -0,0 +1,252 @@
+package main
+
+import (
+  "bufio"
+  "bytes"
+  "context"
+  "encoding/base64"
+  "encoding/hex"
+  "encoding/json"
+  "errors"
+  "flag"
+  "fmt"
+  "io"
+  "os"
+  "path/filepath"
+  "strings"
+
+  "github.com/samchon/ttsc/packages/ttsc/internal/packageartifacts"
+)
+
+type request struct {
+  ID       int64
+  Command  string
+  ActionID []byte
+  OutputID []byte
+  BodySize int64
+}
+
+type response struct {
+  ID            int64
+  Err           string   `json:",omitempty"`
+  KnownCommands []string `json:",omitempty"`
+  Miss          bool     `json:",omitempty"`
+  OutputID      []byte   `json:",omitempty"`
+  Size          int64    `json:",omitempty"`
+  DiskPath      string   `json:",omitempty"`
+}
+
+func main() {
+  if err := run(); err != nil {
+    fmt.Fprintln(os.Stderr, "ttsc cache helper:", err)
+    os.Exit(1)
+  }
+}
+
+func run() error {
+  rootPath := flag.String("root", "", "owned artifact cache root")
+  manifestPath := flag.String("manifest", "", "authenticated publisher manifest")
+  manifestDigest := flag.String("manifest-sha256", "", "trusted release metadata digest")
+  prepare := flag.Bool("prepare", false, "download and verify selected packs before a Go build")
+  packNames := flag.String("packs", "", "comma-separated selected pack names")
+  tupleMode := flag.Bool("tuple", false, "report the effective stock-Go artifact tuple")
+  goBinary := flag.String("go", "", "regular Go executable used by the source builder")
+  seal := flag.Bool("seal", false, "seal recorded publisher actions into an immutable pack")
+  merge := flag.Bool("merge", false, "merge independently sealed publisher packs")
+  plan := flag.String("plan", "", "absolute JSON release merge plan")
+  verify := flag.Bool("verify", false, "validate a trusted release manifest without using a cache")
+  actionGraph := flag.String("actiongraph", "", "Go -debug-actiongraph JSON from the recording build")
+  packageReceipts := flag.String("packages", "", "Go list -deps -json -compiled receipts from the same workspace")
+  release := flag.String("release", "", "ttsc release version")
+  sourceCommit := flag.String("source-commit", "", "full source commit")
+  packURLBase := flag.String("pack-url-base", "", "immutable pack URL directory")
+  packName := flag.String("pack-name", "", "safe transport pack name")
+  outputDir := flag.String("out", "", "publisher output directory")
+  receiptName := flag.String("receipt-name", "", "optional local per-build usage receipt ID")
+  flag.Parse()
+  if *merge {
+    if *plan == "" || *rootPath != "" || *manifestPath != "" || *manifestDigest != "" || *prepare || *packNames != "" || *tupleMode || *goBinary != "" || *seal || *verify || *actionGraph != "" || *packageReceipts != "" || *release != "" || *sourceCommit != "" || *packURLBase != "" || *packName != "" || *outputDir != "" || *receiptName != "" {
+      return errors.New("supply only -merge and -plan for release union")
+    }
+    return MergeRecordedPacks(*plan)
+  }
+  if *plan != "" {
+    return errors.New("-plan requires -merge")
+  }
+  if *verify {
+    if *manifestPath == "" || *manifestDigest == "" || *rootPath != "" || *prepare || *packNames != "" || *tupleMode || *goBinary != "" || *seal || *actionGraph != "" || *packageReceipts != "" || *release != "" || *sourceCommit != "" || *packURLBase != "" || *packName != "" || *outputDir != "" || *receiptName != "" {
+      return errors.New("supply only -verify and trusted manifest arguments")
+    }
+    _, err := loadTrustedManifest(*manifestPath, *manifestDigest)
+    return err
+  }
+  if *seal {
+    if *rootPath == "" || *goBinary == "" || *actionGraph == "" || *packageReceipts == "" || *release == "" || *sourceCommit == "" || *packURLBase == "" || *packName == "" || *outputDir == "" || *manifestPath != "" || *manifestDigest != "" || *prepare || *tupleMode || *packNames != "" || *receiptName != "" {
+      return errors.New("incomplete or conflicting publisher seal arguments")
+    }
+    return SealRecordedActions(sealOptions{root: *rootPath, goBinary: *goBinary, actionGraph: *actionGraph, packages: *packageReceipts, release: *release, sourceCommit: *sourceCommit, packURLBase: *packURLBase, packName: *packName, outputDir: *outputDir})
+  }
+  if *tupleMode {
+    if *goBinary == "" || *rootPath != "" || *manifestPath != "" || *manifestDigest != "" || *prepare || *packNames != "" || *receiptName != "" {
+      return errors.New("supply only -tuple and -go for tuple inspection")
+    }
+    value, err := readGoTuple(*goBinary)
+    if err != nil {
+      return err
+    }
+    return json.NewEncoder(os.Stdout).Encode(value)
+  }
+  if *rootPath == "" || (*manifestPath == "") != (*manifestDigest == "") || (*prepare && (*manifestPath == "" || *packNames == "")) || (!*prepare && *packNames != "") || (*prepare && *receiptName != "") || (*receiptName != "" && !validReceiptName(*receiptName)) {
+    return errors.New("supply -root and either both publisher manifest arguments or neither")
+  }
+  var manifest *packageartifacts.Manifest
+  if *manifestPath != "" {
+    decoded, err := loadTrustedManifest(*manifestPath, *manifestDigest)
+    if err != nil {
+      return err
+    }
+    manifest = &decoded
+  }
+  store, err := openArtifactStore(*rootPath, manifest)
+  if err != nil {
+    return err
+  }
+  defer store.close()
+  store.receiptName = *receiptName
+  if *prepare {
+    receipt, err := preparePacks(context.Background(), store, *manifest, strings.Split(*packNames, ","))
+    if err != nil {
+      return err
+    }
+    return json.NewEncoder(os.Stdout).Encode(receipt)
+  }
+  return serve(os.Stdin, os.Stdout, store)
+}
+
+func loadTrustedManifest(location, digest string) (packageartifacts.Manifest, error) {
+  raw, err := readBoundedFile(location, packageartifacts.MaxManifestBytes)
+  if err != nil {
+    return packageartifacts.Manifest{}, err
+  }
+  return packageartifacts.ParseTrustedManifest(raw, digest)
+}
+
+func serve(source io.Reader, destination io.Writer, store *artifactStore) error {
+  usage := usageReceipt{
+    BaseActionIDs: []string{}, OverlayActionIDs: []string{},
+    MissActionIDs: []string{}, PutActionIDs: []string{},
+  }
+  input := bufio.NewReaderSize(source, 64<<10)
+  output := bufio.NewWriter(destination)
+  if err := writeResponse(output, response{KnownCommands: []string{"get", "put", "close"}}); err != nil {
+    return err
+  }
+  for {
+    line, err := input.ReadSlice('\n')
+    if err != nil {
+      return fmt.Errorf("reading Go cache request: %w", err)
+    }
+    if len(bytes.TrimSpace(line)) == 0 {
+      continue
+    }
+    decoder := json.NewDecoder(bytes.NewReader(line))
+    decoder.DisallowUnknownFields()
+    var current request
+    if err := decoder.Decode(&current); err != nil {
+      return fmt.Errorf("decoding Go cache request: %w", err)
+    }
+    answer := response{ID: current.ID}
+    switch current.Command {
+    case "get":
+      usage.Gets++
+      if len(current.ActionID) != 32 {
+        answer.Err = "invalid full Go ActionID"
+        break
+      }
+      actionID := hex.EncodeToString(current.ActionID)
+      value, location, found, err := store.get(actionID)
+      if err != nil {
+        answer.Err = err.Error()
+      } else if !found {
+        answer.Miss = true
+        usage.Misses++
+        usage.record(&usage.MissActionIDs, actionID)
+      } else {
+        if strings.HasPrefix(location, filepath.Join(store.path, "base")+string(os.PathSeparator)) {
+          usage.BaseHits++
+          usage.record(&usage.BaseActionIDs, actionID)
+        } else {
+          usage.OverlayHits++
+          usage.record(&usage.OverlayActionIDs, actionID)
+        }
+        answer.OutputID, _ = hex.DecodeString(value.OutputID)
+        answer.Size = value.Size
+        answer.DiskPath = location
+      }
+    case "put":
+      if len(current.ActionID) != 32 || len(current.OutputID) != 32 || current.BodySize < 0 || current.BodySize > 512<<20 {
+        return errors.New("invalid full Go cache put identity or body size")
+      }
+      // Go 1.26 writes an extra newline after the encoded request.
+      separator, err := input.ReadByte()
+      if err != nil || separator != '\n' {
+        return errors.New("missing Go cache request separator")
+      }
+      var body io.Reader = bytes.NewReader(nil)
+      if current.BodySize > 0 {
+        quote, err := input.ReadByte()
+        if err != nil || quote != '"' {
+          return errors.New("missing Go cache body framing")
+        }
+        encodedLength := ((current.BodySize + 2) / 3) * 4
+        body = base64.NewDecoder(base64.StdEncoding, io.LimitReader(input, encodedLength))
+      }
+      location, putErr := store.put(hex.EncodeToString(current.ActionID), hex.EncodeToString(current.OutputID), current.BodySize, body)
+      if current.BodySize > 0 {
+        end, err := input.ReadByte()
+        if err != nil || end != '"' {
+          return errors.New("truncated Go cache body framing")
+        }
+        end, err = input.ReadByte()
+        if err != nil || end != '\n' {
+          return errors.New("invalid Go cache body terminator")
+        }
+      }
+      if putErr != nil {
+        answer.Err = putErr.Error()
+      } else {
+        answer.DiskPath = location
+        usage.Puts++
+        usage.PutBytes += current.BodySize
+        usage.record(&usage.PutActionIDs, hex.EncodeToString(current.ActionID))
+      }
+    case "close":
+      if store.receiptName != "" {
+        if accounting, err := accountArtifactStore(store); err == nil {
+          usage.Storage = &accounting
+        } else {
+          fmt.Fprintln(os.Stderr, "ttsc cache helper: storage accounting unavailable:", err)
+        }
+        if err := writeUsageReceipt(store, usage); err != nil {
+          fmt.Fprintln(os.Stderr, "ttsc cache helper: usage receipt unavailable:", err)
+        }
+      }
+      if err := writeResponse(output, answer); err != nil {
+        return err
+      }
+      return nil
+    default:
+      answer.Err = "unsupported Go cache command"
+    }
+    if err := writeResponse(output, answer); err != nil {
+      return err
+    }
+  }
+}
+
+func writeResponse(output *bufio.Writer, value response) error {
+  if err := json.NewEncoder(output).Encode(value); err != nil {
+    return err
+  }
+  return output.Flush()
+}