Refactor snapshot management: integrate logging, enhance concurrency with mutex, add clean extraction option, and update gRPC ListSnapshots with ListOptions.
This commit is contained in:
185
grpc_test.go
185
grpc_test.go
@ -1,9 +1,12 @@
|
||||
package agate
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -202,3 +205,185 @@ func TestGRPCServerClient(t *testing.T) {
|
||||
t.Errorf("file2.txt should not exist in the downloaded snapshot")
|
||||
}
|
||||
}
|
||||
|
||||
// TestGRPC_GetRemoteSnapshot_Incremental tests the incremental download functionality
|
||||
// of GetRemoteSnapshot, verifying that it reuses files from a local parent snapshot
|
||||
// instead of downloading them again.
|
||||
func TestGRPC_GetRemoteSnapshot_Incremental(t *testing.T) {
|
||||
// Skip this test in short mode
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping incremental GetRemoteSnapshot test in short mode")
|
||||
}
|
||||
|
||||
// Create a temporary directory for the server
|
||||
serverDir, err := os.MkdirTemp("", "agate-server-*")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create server temp directory: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(serverDir)
|
||||
|
||||
// Create a temporary directory for the client
|
||||
clientDir, err := os.MkdirTemp("", "agate-client-*")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create client temp directory: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(clientDir)
|
||||
|
||||
// Create a buffer to capture client logs
|
||||
var clientLogBuffer bytes.Buffer
|
||||
clientLogger := log.New(&clientLogBuffer, "", 0)
|
||||
|
||||
// Create Agate options for the server
|
||||
serverOptions := AgateOptions{
|
||||
WorkDir: serverDir,
|
||||
}
|
||||
|
||||
// Create Agate options for the client with logger
|
||||
clientOptions := AgateOptions{
|
||||
WorkDir: clientDir,
|
||||
Logger: clientLogger,
|
||||
}
|
||||
|
||||
// Create Agate instances for server and client
|
||||
serverAgate, err := New(serverOptions)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create server Agate instance: %v", err)
|
||||
}
|
||||
defer serverAgate.Close()
|
||||
|
||||
clientAgate, err := New(clientOptions)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create client Agate instance: %v", err)
|
||||
}
|
||||
defer clientAgate.Close()
|
||||
|
||||
// Create a data directory on the server
|
||||
serverDataDir := serverAgate.options.BlobStore.GetActiveDir()
|
||||
if err := os.MkdirAll(serverDataDir, 0755); err != nil {
|
||||
t.Fatalf("Failed to create server data directory: %v", err)
|
||||
}
|
||||
|
||||
// Create test files for snapshot A on the server
|
||||
if err := os.MkdirAll(filepath.Join(serverDataDir, "subdir"), 0755); err != nil {
|
||||
t.Fatalf("Failed to create subdirectory: %v", err)
|
||||
}
|
||||
|
||||
snapshotAFiles := map[string]string{
|
||||
filepath.Join(serverDataDir, "file1.txt"): "Content of file 1",
|
||||
filepath.Join(serverDataDir, "file2.txt"): "Content of file 2",
|
||||
filepath.Join(serverDataDir, "subdir/file3.txt"): "Content of file 3",
|
||||
}
|
||||
|
||||
for path, content := range snapshotAFiles {
|
||||
if err := os.WriteFile(path, []byte(content), 0644); err != nil {
|
||||
t.Fatalf("Failed to create test file %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create snapshot A on the server
|
||||
ctx := context.Background()
|
||||
snapshotAID, err := serverAgate.SaveSnapshot(ctx, "Snapshot A", "")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create snapshot A: %v", err)
|
||||
}
|
||||
t.Logf("Created snapshot A with ID: %s", snapshotAID)
|
||||
|
||||
// Modify some files and add a new file for snapshot B
|
||||
snapshotBChanges := map[string]string{
|
||||
filepath.Join(serverDataDir, "file1.txt"): "Modified content of file 1", // Modified file
|
||||
filepath.Join(serverDataDir, "file4.txt"): "Content of new file 4", // New file
|
||||
filepath.Join(serverDataDir, "subdir/file5.txt"): "Content of new file 5", // New file in subdir
|
||||
}
|
||||
|
||||
for path, content := range snapshotBChanges {
|
||||
// Ensure parent directory exists
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
|
||||
t.Fatalf("Failed to create directory for %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0644); err != nil {
|
||||
t.Fatalf("Failed to create/modify test file %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create snapshot B on the server (with A as parent)
|
||||
snapshotBID, err := serverAgate.SaveSnapshot(ctx, "Snapshot B", snapshotAID)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create snapshot B: %v", err)
|
||||
}
|
||||
t.Logf("Created snapshot B with ID: %s", snapshotBID)
|
||||
|
||||
// Start the gRPC server
|
||||
serverAddress := "localhost:50052" // Use a different port than the other test
|
||||
server, err := remote.RunServer(ctx, serverAgate.manager, serverAddress)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to start gRPC server: %v", err)
|
||||
}
|
||||
defer server.Stop(ctx)
|
||||
|
||||
// Give the server a moment to start
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// Step 1: Client downloads snapshot A
|
||||
err = clientAgate.GetRemoteSnapshot(ctx, serverAddress, snapshotAID, "")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to download snapshot A: %v", err)
|
||||
}
|
||||
t.Log("Client successfully downloaded snapshot A")
|
||||
|
||||
// Clear the log buffer to capture only logs from the incremental download
|
||||
clientLogBuffer.Reset()
|
||||
|
||||
// Step 2: Client downloads snapshot B, specifying A as the local parent
|
||||
err = clientAgate.GetRemoteSnapshot(ctx, serverAddress, snapshotBID, snapshotAID)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to download snapshot B: %v", err)
|
||||
}
|
||||
t.Log("Client successfully downloaded snapshot B")
|
||||
|
||||
// Step 3: Verify that snapshot B was correctly imported
|
||||
// Restore snapshot B to a directory
|
||||
restoreDir := filepath.Join(clientDir, "restore")
|
||||
if err := os.MkdirAll(restoreDir, 0755); err != nil {
|
||||
t.Fatalf("Failed to create restore directory: %v", err)
|
||||
}
|
||||
|
||||
err = clientAgate.RestoreSnapshotToDir(ctx, snapshotBID, restoreDir)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to restore snapshot B: %v", err)
|
||||
}
|
||||
|
||||
// Verify the restored files match the expected content
|
||||
expectedFiles := map[string]string{
|
||||
filepath.Join(restoreDir, "file1.txt"): "Modified content of file 1", // Modified file
|
||||
filepath.Join(restoreDir, "file2.txt"): "Content of file 2", // Unchanged file
|
||||
filepath.Join(restoreDir, "file4.txt"): "Content of new file 4", // New file
|
||||
filepath.Join(restoreDir, "subdir/file3.txt"): "Content of file 3", // Unchanged file
|
||||
filepath.Join(restoreDir, "subdir/file5.txt"): "Content of new file 5", // New file
|
||||
}
|
||||
|
||||
for path, expectedContent := range expectedFiles {
|
||||
content, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read restored file %s: %v", path, err)
|
||||
}
|
||||
if string(content) != expectedContent {
|
||||
t.Errorf("Restored file %s has wrong content: got %s, want %s", path, string(content), expectedContent)
|
||||
}
|
||||
}
|
||||
|
||||
// Step 4: Analyze logs to verify incremental download behavior
|
||||
logs := clientLogBuffer.String()
|
||||
|
||||
// Check for evidence of file reuse
|
||||
if !strings.Contains(logs, "Reusing file") {
|
||||
t.Errorf("No evidence of file reuse in logs")
|
||||
}
|
||||
|
||||
// Check for evidence of downloading only new/changed files
|
||||
if !strings.Contains(logs, "Downloading file") {
|
||||
t.Errorf("No evidence of downloading new files in logs")
|
||||
}
|
||||
|
||||
// Log the relevant parts for debugging
|
||||
t.Logf("Log evidence of incremental download:\n%s", logs)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user