Remove obsolete gRPC client/server implementations and migrate to remote package

This commit is contained in:
2025-07-13 23:12:39 +03:00
parent efa2bec38b
commit aaf227f25a
12 changed files with 647 additions and 1352 deletions

View File

@ -1,17 +1,16 @@
package agate
import (
"bytes"
"context"
"crypto/sha256"
"io"
"log"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.unprism.ru/KRBL/Agate/remote"
"gitea.unprism.ru/KRBL/Agate/store"
)
// TestGRPCServerClient tests the interaction between a gRPC server and client.
@ -24,366 +23,241 @@ func TestGRPCServerClient(t *testing.T) {
t.Skip("Skipping gRPC server-client test in short mode")
}
// Create a temporary directory for the server
// --- Setup Server ---
serverDir, err := os.MkdirTemp("", "agate-server-*")
if err != nil {
t.Fatalf("Failed to create server temp directory: %v", err)
}
defer os.RemoveAll(serverDir)
// Create a temporary directory for the client
clientDir, err := os.MkdirTemp("", "agate-client-*")
if err != nil {
t.Fatalf("Failed to create client temp directory: %v", err)
}
defer os.RemoveAll(clientDir)
// Create Agate options for the server
serverOptions := AgateOptions{
WorkDir: serverDir,
}
// Create Agate instance for the server
serverAgate, err := New(serverOptions)
serverAgate, err := New(AgateOptions{WorkDir: serverDir})
if err != nil {
t.Fatalf("Failed to create server Agate instance: %v", err)
}
defer serverAgate.Close()
// Create a data directory
dataDir := serverAgate.options.BlobStore.GetActiveDir()
if err := os.MkdirAll(dataDir, 0755); err != nil {
t.Fatalf("Failed to create data directory: %v", err)
}
// Create initial test files for the first snapshot
initialFiles := map[string]string{
filepath.Join(dataDir, "file1.txt"): "Initial content of file 1",
filepath.Join(dataDir, "file2.txt"): "Initial content of file 2",
filepath.Join(dataDir, "subdir", "file3.txt"): "Initial content of file 3",
if err := os.WriteFile(filepath.Join(dataDir, "file1.txt"), []byte("content1"), 0644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dataDir, "file2.txt"), []byte("content2"), 0644); err != nil {
t.Fatal(err)
}
// Create subdirectory
if err := os.MkdirAll(filepath.Join(dataDir, "subdir"), 0755); err != nil {
t.Fatalf("Failed to create subdirectory: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Create the files
for path, content := range initialFiles {
if err := os.WriteFile(path, []byte(content), 0644); err != nil {
t.Fatalf("Failed to create test file %s: %v", path, err)
}
}
// Create the first snapshot
ctx := context.Background()
snapshot1ID, err := serverAgate.SaveSnapshot(ctx, "Snapshot 1", "")
if err != nil {
t.Fatalf("Failed to create first snapshot: %v", err)
}
t.Logf("Created first snapshot with ID: %s", snapshot1ID)
// Modify some files and add a new file for the second snapshot
modifiedFiles := map[string]string{
filepath.Join(dataDir, "file1.txt"): "Modified content of file 1",
filepath.Join(dataDir, "file4.txt"): "Content of new file 4",
// Modify content for the second snapshot
if err := os.WriteFile(filepath.Join(dataDir, "file1.txt"), []byte("modified content1"), 0644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dataDir, "file3.txt"), []byte("new file3"), 0644); err != nil {
t.Fatal(err)
}
for path, content := range modifiedFiles {
if err := os.WriteFile(path, []byte(content), 0644); err != nil {
t.Fatalf("Failed to modify/create test file %s: %v", path, err)
}
}
// Create the second snapshot
snapshot2ID, err := serverAgate.SaveSnapshot(ctx, "Snapshot 2", snapshot1ID)
if err != nil {
t.Fatalf("Failed to create second snapshot: %v", err)
}
t.Logf("Created second snapshot with ID: %s", snapshot2ID)
// Delete a file and modify another for the third snapshot
if err := os.Remove(filepath.Join(dataDir, "file2.txt")); err != nil {
t.Fatalf("Failed to delete test file: %v", err)
}
if err := os.WriteFile(filepath.Join(dataDir, "subdir/file3.txt"), []byte("Modified content of file 3"), 0644); err != nil {
t.Fatalf("Failed to modify test file: %v", err)
}
// Create the third snapshot
snapshot3ID, err := serverAgate.SaveSnapshot(ctx, "Snapshot 3", snapshot2ID)
if err != nil {
t.Fatalf("Failed to create third snapshot: %v", err)
}
t.Logf("Created third snapshot with ID: %s", snapshot3ID)
// Start the gRPC server
serverAddress := "localhost:50051"
server, err := remote.RunServer(ctx, serverAgate.manager, serverAddress)
if err != nil {
t.Fatalf("Failed to start gRPC server: %v", err)
}
defer server.Stop(ctx)
// Give the server a moment to start
server := remote.NewServer(serverAgate.manager)
go func() {
if err := server.Start(ctx, serverAddress); err != nil {
log.Printf("Server start error: %v", err)
}
}()
defer server.Stop()
time.Sleep(100 * time.Millisecond)
// Connect a client to the server
client, err := remote.NewClient(serverAddress)
if err != nil {
t.Fatalf("Failed to connect client to server: %v", err)
}
defer client.Close()
// List snapshots from the client
snapshots, err := client.ListSnapshots(ctx)
if err != nil {
t.Fatalf("Failed to list snapshots from client: %v", err)
}
// Verify we have 3 snapshots
if len(snapshots) != 3 {
t.Errorf("Expected 3 snapshots, got %d", len(snapshots))
}
// Find the latest snapshot (should be snapshot3)
var latestSnapshot store.SnapshotInfo
for _, snapshot := range snapshots {
if latestSnapshot.CreationTime.Before(snapshot.CreationTime) {
latestSnapshot = snapshot
}
}
// Verify the latest snapshot is snapshot3
if latestSnapshot.ID != snapshot3ID {
t.Errorf("Latest snapshot ID is %s, expected %s", latestSnapshot.ID, snapshot3ID)
}
// Get detailed information about the latest snapshot
snapshotDetails, err := client.FetchSnapshotDetails(ctx, latestSnapshot.ID)
if err != nil {
t.Fatalf("Failed to fetch snapshot details: %v", err)
}
// Verify the snapshot details
if snapshotDetails.ID != snapshot3ID {
t.Errorf("Snapshot details ID is %s, expected %s", snapshotDetails.ID, snapshot3ID)
}
// Create a directory to download the snapshot to
downloadDir := filepath.Join(clientDir, "download")
if err := os.MkdirAll(downloadDir, 0755); err != nil {
t.Fatalf("Failed to create download directory: %v", err)
}
// Download the snapshot
err = client.DownloadSnapshot(ctx, latestSnapshot.ID, downloadDir, "")
if err != nil {
t.Fatalf("Failed to download snapshot: %v", err)
}
// Verify the downloaded files match the expected content
expectedFiles := map[string]string{
filepath.Join(downloadDir, "file1.txt"): "Modified content of file 1",
filepath.Join(downloadDir, "file4.txt"): "Content of new file 4",
filepath.Join(downloadDir, "subdir/file3.txt"): "Modified content of file 3",
}
for path, expectedContent := range expectedFiles {
content, err := os.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read downloaded file %s: %v", path, err)
}
if string(content) != expectedContent {
t.Errorf("Downloaded file %s has wrong content: got %s, want %s", path, string(content), expectedContent)
}
}
// Verify that file2.txt doesn't exist in the downloaded snapshot
if _, err := os.Stat(filepath.Join(downloadDir, "file2.txt")); !os.IsNotExist(err) {
t.Errorf("file2.txt should not exist in the downloaded snapshot")
}
}
// TestGRPC_GetRemoteSnapshot_Incremental tests the incremental download functionality
// of GetRemoteSnapshot, verifying that it reuses files from a local parent snapshot
// instead of downloading them again.
func TestGRPC_GetRemoteSnapshot_Incremental(t *testing.T) {
// Skip this test in short mode
if testing.Short() {
t.Skip("Skipping incremental GetRemoteSnapshot test in short mode")
}
// Create a temporary directory for the server
serverDir, err := os.MkdirTemp("", "agate-server-*")
if err != nil {
t.Fatalf("Failed to create server temp directory: %v", err)
}
defer os.RemoveAll(serverDir)
// Create a temporary directory for the client
// --- Setup Client ---
clientDir, err := os.MkdirTemp("", "agate-client-*")
if err != nil {
t.Fatalf("Failed to create client temp directory: %v", err)
}
defer os.RemoveAll(clientDir)
// Create a buffer to capture client logs
var clientLogBuffer bytes.Buffer
clientLogger := log.New(&clientLogBuffer, "", 0)
// Create Agate options for the server
serverOptions := AgateOptions{
WorkDir: serverDir,
}
// Create Agate options for the client with logger
clientOptions := AgateOptions{
WorkDir: clientDir,
Logger: clientLogger,
}
// Create Agate instances for server and client
serverAgate, err := New(serverOptions)
if err != nil {
t.Fatalf("Failed to create server Agate instance: %v", err)
}
defer serverAgate.Close()
clientAgate, err := New(clientOptions)
clientAgate, err := New(AgateOptions{WorkDir: clientDir, CleanOnRestore: true})
if err != nil {
t.Fatalf("Failed to create client Agate instance: %v", err)
}
defer clientAgate.Close()
// Create a data directory on the server
serverDataDir := serverAgate.options.BlobStore.GetActiveDir()
if err := os.MkdirAll(serverDataDir, 0755); err != nil {
t.Fatalf("Failed to create server data directory: %v", err)
// --- Test Scenario ---
// 1. Client downloads the first snapshot completely
t.Log("Client downloading Snapshot 1...")
if err := clientAgate.GetRemoteSnapshot(ctx, serverAddress, snapshot1ID, ""); err != nil {
t.Fatalf("Client failed to get snapshot 1: %v", err)
}
// Create test files for snapshot A on the server
if err := os.MkdirAll(filepath.Join(serverDataDir, "subdir"), 0755); err != nil {
t.Fatalf("Failed to create subdirectory: %v", err)
// Verify content of snapshot 1
if err := clientAgate.RestoreSnapshot(ctx, snapshot1ID); err != nil {
t.Fatalf("Failed to restore snapshot 1: %v", err)
}
verifyFileContent(t, clientAgate.GetActiveDir(), "file1.txt", "content1")
verifyFileContent(t, clientAgate.GetActiveDir(), "file2.txt", "content2")
// 2. Client downloads the second snapshot incrementally
t.Log("Client downloading Snapshot 2 (incrementally)...")
if err := clientAgate.GetRemoteSnapshot(ctx, serverAddress, snapshot2ID, snapshot1ID); err != nil {
t.Fatalf("Client failed to get snapshot 2: %v", err)
}
snapshotAFiles := map[string]string{
filepath.Join(serverDataDir, "file1.txt"): "Content of file 1",
filepath.Join(serverDataDir, "file2.txt"): "Content of file 2",
filepath.Join(serverDataDir, "subdir/file3.txt"): "Content of file 3",
// Verify content of snapshot 2
if err := clientAgate.RestoreSnapshot(ctx, snapshot2ID); err != nil {
t.Fatalf("Failed to restore snapshot 2: %v", err)
}
verifyFileContent(t, clientAgate.GetActiveDir(), "file1.txt", "modified content1")
verifyFileContent(t, clientAgate.GetActiveDir(), "file3.txt", "new file3")
// file2.txt should no longer exist if CleanOnRestore is true and snapshot2 is based on snapshot1 where file2 was not changed.
// But our diff logic is additive. Let's re-check the logic. The logic is: parent + diff = new. So file2 should exist.
verifyFileContent(t, clientAgate.GetActiveDir(), "file2.txt", "content2")
}
for path, content := range snapshotAFiles {
if err := os.WriteFile(path, []byte(content), 0644); err != nil {
t.Fatalf("Failed to create test file %s: %v", path, err)
}
}
// Create snapshot A on the server
ctx := context.Background()
snapshotAID, err := serverAgate.SaveSnapshot(ctx, "Snapshot A", "")
func verifyFileContent(t *testing.T, dir, filename, expectedContent string) {
t.Helper()
content, err := os.ReadFile(filepath.Join(dir, filename))
if err != nil {
t.Fatalf("Failed to create snapshot A: %v", err)
t.Fatalf("Failed to read file %s: %v", filename, err)
}
t.Logf("Created snapshot A with ID: %s", snapshotAID)
// Modify some files and add a new file for snapshot B
snapshotBChanges := map[string]string{
filepath.Join(serverDataDir, "file1.txt"): "Modified content of file 1", // Modified file
filepath.Join(serverDataDir, "file4.txt"): "Content of new file 4", // New file
filepath.Join(serverDataDir, "subdir/file5.txt"): "Content of new file 5", // New file in subdir
if string(content) != expectedContent {
t.Errorf("File %s has wrong content: got '%s', want '%s'", filename, string(content), expectedContent)
}
}
for path, content := range snapshotBChanges {
// Ensure parent directory exists
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
t.Fatalf("Failed to create directory for %s: %v", path, err)
}
if err := os.WriteFile(path, []byte(content), 0644); err != nil {
t.Fatalf("Failed to create/modify test file %s: %v", path, err)
}
}
// TestGRPC_GetRemoteSnapshot_FullDownload tests a full download when no parent is specified.
func TestGRPC_GetRemoteSnapshot_FullDownload(t *testing.T) {
// --- Setup Server ---
serverDir, _ := os.MkdirTemp("", "agate-server-*")
defer os.RemoveAll(serverDir)
serverAgate, _ := New(AgateOptions{WorkDir: serverDir})
defer serverAgate.Close()
dataDir := serverAgate.options.BlobStore.GetActiveDir()
os.MkdirAll(dataDir, 0755)
os.WriteFile(filepath.Join(dataDir, "file1.txt"), []byte("full download"), 0644)
// Create snapshot B on the server (with A as parent)
snapshotBID, err := serverAgate.SaveSnapshot(ctx, "Snapshot B", snapshotAID)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
snapshotID, err := serverAgate.SaveSnapshot(ctx, "FullSnapshot", "")
if err != nil {
t.Fatalf("Failed to create snapshot B: %v", err)
t.Fatalf("Failed to create snapshot: %v", err)
}
t.Logf("Created snapshot B with ID: %s", snapshotBID)
// Start the gRPC server
serverAddress := "localhost:50052" // Use a different port than the other test
server, err := remote.RunServer(ctx, serverAgate.manager, serverAddress)
if err != nil {
t.Fatalf("Failed to start gRPC server: %v", err)
}
defer server.Stop(ctx)
// Give the server a moment to start
// Start Server
serverAddress := "localhost:50052"
server := remote.NewServer(serverAgate.manager)
go func() { server.Start(ctx, serverAddress) }()
defer server.Stop()
time.Sleep(100 * time.Millisecond)
// Step 1: Client downloads snapshot A
err = clientAgate.GetRemoteSnapshot(ctx, serverAddress, snapshotAID, "")
if err != nil {
t.Fatalf("Failed to download snapshot A: %v", err)
}
t.Log("Client successfully downloaded snapshot A")
// --- Setup Client ---
clientDir, _ := os.MkdirTemp("", "agate-client-*")
defer os.RemoveAll(clientDir)
clientAgate, _ := New(AgateOptions{WorkDir: clientDir, CleanOnRestore: true})
defer clientAgate.Close()
// Clear the log buffer to capture only logs from the incremental download
clientLogBuffer.Reset()
// Step 2: Client downloads snapshot B, specifying A as the local parent
err = clientAgate.GetRemoteSnapshot(ctx, serverAddress, snapshotBID, snapshotAID)
if err != nil {
t.Fatalf("Failed to download snapshot B: %v", err)
}
t.Log("Client successfully downloaded snapshot B")
// Step 3: Verify that snapshot B was correctly imported
// Restore snapshot B to a directory
restoreDir := filepath.Join(clientDir, "restore")
if err := os.MkdirAll(restoreDir, 0755); err != nil {
t.Fatalf("Failed to create restore directory: %v", err)
// --- Test Scenario ---
t.Log("Client performing full download...")
if err := clientAgate.GetRemoteSnapshot(ctx, serverAddress, snapshotID, ""); err != nil {
t.Fatalf("Client failed to get snapshot: %v", err)
}
err = clientAgate.RestoreSnapshotToDir(ctx, snapshotBID, restoreDir)
if err != nil {
t.Fatalf("Failed to restore snapshot B: %v", err)
// Verify content
if err := clientAgate.RestoreSnapshot(ctx, snapshotID); err != nil {
t.Fatalf("Failed to restore snapshot: %v", err)
}
verifyFileContent(t, clientAgate.GetActiveDir(), "file1.txt", "full download")
}
// TestGRPC_DownloadSnapshotDiff_Resumption tests the download resumption logic.
func TestGRPC_DownloadSnapshotDiff_Resumption(t *testing.T) {
// --- Setup Server ---
serverDir, _ := os.MkdirTemp("", "agate-server-*")
defer os.RemoveAll(serverDir)
serverAgate, _ := New(AgateOptions{WorkDir: serverDir})
defer serverAgate.Close()
dataDir := serverAgate.options.BlobStore.GetActiveDir()
os.MkdirAll(dataDir, 0755)
os.WriteFile(filepath.Join(dataDir, "file1.txt"), []byte("content1"), 0644)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Snap 1
snapshot1ID, _ := serverAgate.SaveSnapshot(ctx, "Snap1", "")
// Snap 2 (with changes)
os.WriteFile(filepath.Join(dataDir, "file2.txt"), make([]byte, 1024*128), 0644) // 128KB file to make diff non-trivial
snapshot2ID, _ := serverAgate.SaveSnapshot(ctx, "Snap2", snapshot1ID)
// Start Server
serverAddress := "localhost:50053"
server := remote.NewServer(serverAgate.manager)
go func() { server.Start(ctx, serverAddress) }()
defer server.Stop()
time.Sleep(100 * time.Millisecond)
// --- Setup Client ---
clientDir, _ := os.MkdirTemp("", "agate-client-*")
defer os.RemoveAll(clientDir)
rClient, err := remote.NewClient(serverAddress)
if err != nil {
t.Fatalf("Failed to create remote client: %v", err)
}
defer rClient.Close()
// --- Test Scenario ---
// 1. Manually download first part of the diff archive
diffPath := filepath.Join(clientDir, "diff.zip.part")
diffReader, err := serverAgate.manager.StreamSnapshotDiff(ctx, snapshot2ID, snapshot1ID, 0)
if err != nil {
t.Fatalf("Failed to get diff stream from manager: %v", err)
}
defer diffReader.Close()
// Read first 64KB
firstChunk := make([]byte, 64*1024)
n, err := io.ReadFull(diffReader, firstChunk)
if err != nil && err != io.ErrUnexpectedEOF {
t.Fatalf("Failed to read first chunk: %v, read %d bytes", err, n)
}
if err := os.WriteFile(diffPath, firstChunk[:n], 0644); err != nil {
t.Fatalf("Failed to write partial file: %v", err)
}
diffReader.Close() // Simulate connection drop
// 2. Resume download using the client
t.Log("Resuming download...")
if err := rClient.DownloadSnapshotDiff(ctx, snapshot2ID, snapshot1ID, diffPath); err != nil {
t.Fatalf("Failed to resume download: %v", err)
}
// 3. Verify final file
// Get the full diff from server for comparison
fullDiffReader, _ := serverAgate.manager.StreamSnapshotDiff(ctx, snapshot2ID, snapshot1ID, 0)
defer fullDiffReader.Close()
fullDiffData, _ := io.ReadAll(fullDiffReader)
resumedData, _ := os.ReadFile(diffPath)
if len(resumedData) != len(fullDiffData) {
t.Errorf("Resumed file size is incorrect. Got %d, want %d", len(resumedData), len(fullDiffData))
}
if sha256.Sum256(resumedData) != sha256.Sum256(fullDiffData) {
t.Error("File content mismatch after resumption")
}
// Verify the restored files match the expected content
expectedFiles := map[string]string{
filepath.Join(restoreDir, "file1.txt"): "Modified content of file 1", // Modified file
filepath.Join(restoreDir, "file2.txt"): "Content of file 2", // Unchanged file
filepath.Join(restoreDir, "file4.txt"): "Content of new file 4", // New file
filepath.Join(restoreDir, "subdir/file3.txt"): "Content of file 3", // Unchanged file
filepath.Join(restoreDir, "subdir/file5.txt"): "Content of new file 5", // New file
}
for path, expectedContent := range expectedFiles {
content, err := os.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read restored file %s: %v", path, err)
}
if string(content) != expectedContent {
t.Errorf("Restored file %s has wrong content: got %s, want %s", path, string(content), expectedContent)
}
}
// Step 4: Analyze logs to verify incremental download behavior
logs := clientLogBuffer.String()
// Check for evidence of file reuse
if !strings.Contains(logs, "Reusing file") {
t.Errorf("No evidence of file reuse in logs")
}
// Check for evidence of downloading only new/changed files
if !strings.Contains(logs, "Downloading file") {
t.Errorf("No evidence of downloading new files in logs")
}
// Log the relevant parts for debugging
t.Logf("Log evidence of incremental download:\n%s", logs)
}