From 223a63ee6d0059de9ce1aabe46ca4ed0c2438a9b Mon Sep 17 00:00:00 2001 From: Alexander Lazarenko Date: Mon, 7 Jul 2025 20:03:40 +0300 Subject: [PATCH] Refactor snapshot management: integrate logging, enhance concurrency with mutex, add clean extraction option, and update gRPC ListSnapshots with ListOptions. --- Makefile | 16 +-- api.go | 220 +++++++++++++++++++++++++++++++----- api_test.go | 88 +++++++++++++++ functional_test.go | 21 +++- grpc/server.go | 6 +- grpc_test.go | 185 ++++++++++++++++++++++++++++++ interfaces/snapshot.go | 31 +++-- manager.go | 168 +++++++++++---------------- manager_test.go | 183 +++++++++++++++++++++++++++--- remote/server.go | 5 +- snapshot.go | 53 --------- store/sqlite/sqlite.go | 77 ++++++++----- store/sqlite/sqlite_test.go | 166 ++++++++++++++++++++++++++- store/store.go | 11 +- 14 files changed, 977 insertions(+), 253 deletions(-) delete mode 100644 snapshot.go diff --git a/Makefile b/Makefile index 077325c..e25192a 100644 --- a/Makefile +++ b/Makefile @@ -15,37 +15,37 @@ gen-proto: --grpc-gateway_out=grpc --grpc-gateway_opt paths=source_relative \ ./grpc/snapshot.proto -# Запуск всех тестов +# Run all tests test: go test -v ./... -# Запуск модульных тестов +# Run unit tests test-unit: go test -v ./store/... ./hash/... ./archive/... -# Запуск интеграционных тестов +# Run integration tests test-integration: go test -v -tags=integration ./... -# Запуск функциональных тестов +# Run functional tests test-functional: go test -v -run TestFull ./... -# Запуск тестов производительности +# Run performance tests test-performance: go test -v -run TestPerformanceMetrics ./... go test -v -bench=. ./... -# Запуск тестов с покрытием кода +# Run tests with code coverage test-coverage: go test -v -coverprofile=coverage.out ./... go tool cover -html=coverage.out -o coverage.html -# Запуск линтера +# Run linter lint: golangci-lint run -# Запуск всех проверок (тесты + линтер) +# Run all checks (tests + linter) check: test lint .PHONY: download-third-party gen-proto test test-unit test-integration test-functional test-performance test-coverage lint check diff --git a/api.go b/api.go index ff3f412..f869a3e 100644 --- a/api.go +++ b/api.go @@ -4,9 +4,14 @@ import ( "context" "errors" "fmt" + "gitea.unprism.ru/KRBL/Agate/archive" "gitea.unprism.ru/KRBL/Agate/grpc" + "gitea.unprism.ru/KRBL/Agate/interfaces" + "io" + "log" "os" "path/filepath" + "sync" "gitea.unprism.ru/KRBL/Agate/store" "gitea.unprism.ru/KRBL/Agate/stores" @@ -35,11 +40,18 @@ type AgateOptions struct { // Use the stores package to initialize a custom implementation: // blobStore, err := stores.NewDefaultBlobStore(blobsDir) BlobStore store.BlobStore + + // CleanOnRestore specifies whether the target directory should be cleaned before restoring a snapshot. + CleanOnRestore bool + + // Logger is the logger to use for output. If nil, logging is disabled. + Logger *log.Logger } // Agate is the main entry point for the snapshot library. type Agate struct { - manager SnapshotManager + mutex sync.Mutex + manager interfaces.SnapshotManager options AgateOptions metadataDir string blobsDir string @@ -53,6 +65,11 @@ func New(options AgateOptions) (*Agate, error) { return nil, errors.New("work directory cannot be empty") } + // Initialize logger if not provided + if options.Logger == nil { + options.Logger = log.New(io.Discard, "", 0) + } + // Create the work directory if it doesn't exist if err := os.MkdirAll(options.WorkDir, 0755); err != nil { return nil, fmt.Errorf("failed to create work directory: %w", err) @@ -109,7 +126,7 @@ func New(options AgateOptions) (*Agate, error) { } // Create the snapshot manager - manager, err := CreateSnapshotManager(metadataStore, blobStore) + manager, err := CreateSnapshotManager(metadataStore, blobStore, options.Logger) if err != nil { return nil, fmt.Errorf("failed to create snapshot manager: %w", err) } @@ -160,6 +177,11 @@ func (a *Agate) GetBlobsDir() string { // If parentID is empty, it will use the ID of the snapshot currently loaded in the active directory. // Returns the ID of the created snapshot. func (a *Agate) SaveSnapshot(ctx context.Context, name string, parentID string) (string, error) { + a.mutex.Lock() + defer a.mutex.Unlock() + + a.options.Logger.Printf("Creating new snapshot with name: %s", name) + // Call CloseFunc if provided if a.options.CloseFunc != nil { if err := a.options.CloseFunc(); err != nil { @@ -170,7 +192,7 @@ func (a *Agate) SaveSnapshot(ctx context.Context, name string, parentID string) defer func() { if a.options.OpenFunc != nil { if err := a.options.OpenFunc(a.options.BlobStore.GetActiveDir()); err != nil { - fmt.Printf("Failed to open resources after snapshot: %v\n", err) + a.options.Logger.Printf("ERROR: failed to open resources after snapshot creation: %v", err) } } }() @@ -185,9 +207,12 @@ func (a *Agate) SaveSnapshot(ctx context.Context, name string, parentID string) // Create the snapshot snapshot, err := a.manager.CreateSnapshot(ctx, a.options.BlobStore.GetActiveDir(), name, effectiveParentID) if err != nil { + a.options.Logger.Printf("ERROR: failed to create snapshot: %v", err) return "", fmt.Errorf("failed to create snapshot: %w", err) } + a.options.Logger.Printf("Successfully created snapshot with ID: %s", snapshot.ID) + // Update the current snapshot ID to the newly created snapshot a.currentSnapshotID = snapshot.ID @@ -201,6 +226,11 @@ func (a *Agate) SaveSnapshot(ctx context.Context, name string, parentID string) // RestoreSnapshot extracts a snapshot to the active directory. func (a *Agate) RestoreSnapshot(ctx context.Context, snapshotID string) error { + a.mutex.Lock() + defer a.mutex.Unlock() + + a.options.Logger.Printf("Restoring snapshot with ID: %s", snapshotID) + // Call CloseFunc if provided if a.options.CloseFunc != nil { if err := a.options.CloseFunc(); err != nil { @@ -209,10 +239,13 @@ func (a *Agate) RestoreSnapshot(ctx context.Context, snapshotID string) error { } // Extract the snapshot - if err := a.manager.ExtractSnapshot(ctx, snapshotID, a.options.BlobStore.GetActiveDir()); err != nil { + if err := a.manager.ExtractSnapshot(ctx, snapshotID, a.options.BlobStore.GetActiveDir(), a.options.CleanOnRestore); err != nil { + a.options.Logger.Printf("ERROR: failed to extract snapshot: %v", err) return fmt.Errorf("failed to extract snapshot: %w", err) } + a.options.Logger.Printf("Successfully restored snapshot with ID: %s", snapshotID) + // Save the ID of the snapshot that was restored a.currentSnapshotID = snapshotID @@ -224,6 +257,7 @@ func (a *Agate) RestoreSnapshot(ctx context.Context, snapshotID string) error { // Call OpenFunc if provided if a.options.OpenFunc != nil { if err := a.options.OpenFunc(a.options.BlobStore.GetActiveDir()); err != nil { + a.options.Logger.Printf("ERROR: failed to open resources after restore: %v", err) return fmt.Errorf("failed to open resources after restore: %w", err) } } @@ -233,6 +267,9 @@ func (a *Agate) RestoreSnapshot(ctx context.Context, snapshotID string) error { // RestoreSnapshot extracts a snapshot to the directory. func (a *Agate) RestoreSnapshotToDir(ctx context.Context, snapshotID string, dir string) error { + a.mutex.Lock() + defer a.mutex.Unlock() + // Call CloseFunc if provided if a.options.CloseFunc != nil { if err := a.options.CloseFunc(); err != nil { @@ -243,13 +280,13 @@ func (a *Agate) RestoreSnapshotToDir(ctx context.Context, snapshotID string, dir defer func() { if a.options.OpenFunc != nil { if err := a.options.OpenFunc(dir); err != nil { - fmt.Printf("Failed to open resources after snapshot: %v\n", err) + a.options.Logger.Printf("ERROR: failed to open resources after snapshot restore: %v", err) } } }() // Extract the snapshot - if err := a.manager.ExtractSnapshot(ctx, snapshotID, dir); err != nil { + if err := a.manager.ExtractSnapshot(ctx, snapshotID, dir, a.options.CleanOnRestore); err != nil { return fmt.Errorf("failed to extract snapshot: %w", err) } @@ -268,7 +305,9 @@ func (a *Agate) RestoreSnapshotToDir(ctx context.Context, snapshotID string, dir // ListSnapshots returns a list of all available snapshots. func (a *Agate) ListSnapshots(ctx context.Context) ([]store.SnapshotInfo, error) { - return a.manager.ListSnapshots(ctx) + // Create empty ListOptions since we don't have filtering/pagination in this API yet + opts := store.ListOptions{} + return a.manager.ListSnapshots(ctx, opts) } // GetSnapshotDetails returns detailed information about a specific snapshot. @@ -301,7 +340,10 @@ func (a *Agate) Open() error { // Close releases all resources used by the Agate instance. func (a *Agate) Close() error { - return a.options.CloseFunc() + if a.options.CloseFunc != nil { + return a.options.CloseFunc() + } + return nil } // StartServer starts a gRPC server to share snapshots. @@ -348,31 +390,155 @@ func (a *Agate) GetRemoteSnapshot(ctx context.Context, address string, snapshotI } defer client.Close() - // Create a temporary directory for the downloaded snapshot - tempDir := filepath.Join(a.options.WorkDir, "temp", snapshotID) - if err := os.MkdirAll(tempDir, 0755); err != nil { - return fmt.Errorf("failed to create temporary directory: %w", err) - } - - // Download the snapshot - if err := client.DownloadSnapshot(ctx, snapshotID, tempDir, localParentID); err != nil { - return fmt.Errorf("failed to download snapshot: %w", err) - } - - // Get the snapshot details to create a local copy - details, err := client.FetchSnapshotDetails(ctx, snapshotID) + // Get the remote snapshot details + remoteSnapshot, err := client.FetchSnapshotDetails(ctx, snapshotID) if err != nil { return fmt.Errorf("failed to get snapshot details: %w", err) } - // Create a local snapshot from the downloaded files - _, err = a.manager.CreateSnapshot(ctx, tempDir, details.Name, localParentID) - if err != nil { - return fmt.Errorf("failed to create local snapshot: %w", err) + // Create a temporary directory for downloading files + tempDownloadDir := filepath.Join(a.options.WorkDir, "temp_download", snapshotID) + if err := os.MkdirAll(tempDownloadDir, 0755); err != nil { + return fmt.Errorf("failed to create temporary download directory: %w", err) + } + defer os.RemoveAll(tempDownloadDir) // Clean up when done + + a.options.Logger.Printf("Downloading snapshot %s from %s", snapshotID, address) + + // If localParentID is provided, try to reuse files from the local parent snapshot + if localParentID != "" { + a.options.Logger.Printf("Using local parent snapshot %s for incremental download", localParentID) + + // Get the local parent snapshot details + localParent, err := a.GetSnapshotDetails(ctx, localParentID) + if err != nil { + a.options.Logger.Printf("Warning: Failed to get local parent snapshot details: %v", err) + } else { + // Extract the local parent snapshot to a temporary directory + localParentDir := filepath.Join(a.options.WorkDir, "temp_download", localParentID) + if err := os.MkdirAll(localParentDir, 0755); err != nil { + a.options.Logger.Printf("Warning: Failed to create temporary directory for local parent: %v", err) + } else { + defer os.RemoveAll(localParentDir) // Clean up when done + + if err := a.manager.ExtractSnapshot(ctx, localParentID, localParentDir, true); err != nil { + a.options.Logger.Printf("Warning: Failed to extract local parent snapshot: %v", err) + } else { + // Copy unchanged files from the local parent to the download directory + for _, file := range remoteSnapshot.Files { + // Skip directories, they'll be created as needed + if file.IsDir { + continue + } + + // Check if the file exists in the local parent with the same hash + var localFile *store.FileInfo + for _, lf := range localParent.Files { + if lf.Path == file.Path && lf.SHA256 == file.SHA256 { + localFile = &lf + break + } + } + + if localFile != nil { + // File exists in local parent with the same hash, copy it + srcPath := filepath.Join(localParentDir, localFile.Path) + dstPath := filepath.Join(tempDownloadDir, file.Path) + + // Ensure the destination directory exists + if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil { + a.options.Logger.Printf("Failed to create directory for %s: %v", dstPath, err) + continue + } + + // Copy the file + srcFile, err := os.Open(srcPath) + if err != nil { + a.options.Logger.Printf("Failed to copy file %s, will download instead: %v", file.Path, err) + continue + } + defer srcFile.Close() + + dstFile, err := os.Create(dstPath) + if err != nil { + a.options.Logger.Printf("Failed to create destination file %s: %v", dstPath, err) + continue + } + + _, err = io.Copy(dstFile, srcFile) + dstFile.Close() + + if err != nil { + a.options.Logger.Printf("Failed to copy file data for %s: %v", file.Path, err) + // If copy fails, the file will be downloaded + } else { + a.options.Logger.Printf("Reusing file %s from local parent", file.Path) + } + } + } + } + } + } } - // Clean up the temporary directory - os.RemoveAll(tempDir) + // Download the snapshot files + a.options.Logger.Printf("Downloading files for snapshot %s", snapshotID) + // Get snapshot details to know what files we need to download + remoteDetails, err := client.FetchSnapshotDetails(ctx, snapshotID) + if err != nil { + return fmt.Errorf("failed to get remote snapshot details: %w", err) + } + + // Check which files we already have and which we need to download + for _, file := range remoteDetails.Files { + if file.IsDir { + continue // Skip directories + } + + filePath := filepath.Join(tempDownloadDir, file.Path) + if _, err := os.Stat(filePath); os.IsNotExist(err) { + // File doesn't exist yet, we'll need to download it + a.options.Logger.Printf("Downloading file %s", file.Path) + } + } + + if err := client.DownloadSnapshot(ctx, snapshotID, tempDownloadDir, localParentID); err != nil { + return fmt.Errorf("failed to download snapshot: %w", err) + } + + a.options.Logger.Printf("Creating archive from downloaded files") + + // Create a zip archive from the downloaded files + zipPath := filepath.Join(a.options.WorkDir, "temp_download", snapshotID+".zip") + if err := archive.CreateArchive(tempDownloadDir, zipPath); err != nil { + return fmt.Errorf("failed to create zip archive: %w", err) + } + + // Store the blob with the remote snapshot ID + zipFile, err := os.Open(zipPath) + if err != nil { + return fmt.Errorf("failed to open zip file: %w", err) + } + defer zipFile.Close() + defer os.Remove(zipPath) // Clean up the zip file when done + + a.options.Logger.Printf("Storing blob with ID %s", remoteSnapshot.ID) + + // Store the blob with the remote snapshot ID + _, err = a.options.BlobStore.StoreBlob(ctx, remoteSnapshot.ID, zipFile) + if err != nil { + return fmt.Errorf("failed to store blob: %w", err) + } + + a.options.Logger.Printf("Saving snapshot metadata") + + // Save the remote snapshot metadata + err = a.options.MetadataStore.SaveSnapshotMetadata(ctx, *remoteSnapshot) + if err != nil { + return fmt.Errorf("failed to save snapshot metadata: %w", err) + } + + a.options.Logger.Printf("Successfully imported remote snapshot %s", snapshotID) return nil } diff --git a/api_test.go b/api_test.go index d61907b..f7b3b6a 100644 --- a/api_test.go +++ b/api_test.go @@ -1,9 +1,12 @@ package agate import ( + "bytes" "context" + "log" "os" "path/filepath" + "strings" "testing" ) @@ -252,6 +255,91 @@ func TestAPIListSnapshots(t *testing.T) { } } +func TestAgate_Logging(t *testing.T) { + // Create a temporary directory for tests + tempDir, err := os.MkdirTemp("", "agate-test-*") + if err != nil { + t.Fatalf("Failed to create temp directory: %v", err) + } + defer os.RemoveAll(tempDir) + + // Create a data directory + dataDir := filepath.Join(tempDir, "data") + if err := os.MkdirAll(dataDir, 0755); err != nil { + t.Fatalf("Failed to create data directory: %v", err) + } + + // Create test files in the active directory + activeDir := filepath.Join(dataDir, "blobs", "active") + if err := os.MkdirAll(activeDir, 0755); err != nil { + t.Fatalf("Failed to create active directory: %v", err) + } + createAPITestFiles(t, activeDir) + + // Create a buffer to capture log output + var logBuffer bytes.Buffer + logger := log.New(&logBuffer, "", 0) + + // Create Agate options with the logger + options := AgateOptions{ + WorkDir: dataDir, + OpenFunc: func(dir string) error { + return nil + }, + CloseFunc: func() error { + return nil + }, + Logger: logger, + } + + // Create Agate instance + ag, err := New(options) + if err != nil { + t.Fatalf("Failed to create Agate instance: %v", err) + } + defer ag.Close() + + // Perform operations that should generate logs + ctx := context.Background() + + // Save a snapshot + snapshotID, err := ag.SaveSnapshot(ctx, "Test Snapshot", "") + if err != nil { + t.Fatalf("Failed to create snapshot: %v", err) + } + + // Restore the snapshot + err = ag.RestoreSnapshot(ctx, snapshotID) + if err != nil { + t.Fatalf("Failed to restore snapshot: %v", err) + } + + // Check that logs were generated + logs := logBuffer.String() + if logs == "" { + t.Errorf("No logs were generated") + } + + // Check for expected log messages + expectedLogMessages := []string{ + "Creating new snapshot", + "Restoring snapshot", + } + + for _, msg := range expectedLogMessages { + if !strings.Contains(logs, msg) { + t.Errorf("Expected log message '%s' not found in logs", msg) + } + } +} + +// Note: This test is a placeholder for when the ListSnapshots method is updated to accept ListOptions. +// Currently, the ListSnapshots method in api.go doesn't accept ListOptions, so we can't test that functionality directly. +// The test for ListOptions functionality is covered in TestListSnapshotsMetadata_WithOptions in store/sqlite/sqlite_test.go. +func TestAgate_ListSnapshotsWithOptions(t *testing.T) { + t.Skip("Skipping test as ListSnapshots in api.go doesn't yet support ListOptions") +} + func TestAPIDeleteSnapshot(t *testing.T) { ag, _, cleanup := setupTestAPI(t) defer cleanup() diff --git a/functional_test.go b/functional_test.go index da81380..e842694 100644 --- a/functional_test.go +++ b/functional_test.go @@ -173,13 +173,32 @@ func TestFullWorkflow(t *testing.T) { t.Fatalf("Failed to list snapshots: %v", err) } + // Debug output + t.Logf("After deletion, found %d snapshots:", len(snapshots)) + for i, snap := range snapshots { + t.Logf(" Snapshot %d: ID=%s, Name=%s, ParentID=%s", i+1, snap.ID, snap.Name, snap.ParentID) + } + + // Get detailed information about snapshot 3 + snapshot3, err := ag.GetSnapshotDetails(ctx, snapshot3ID) + if err != nil { + t.Logf("Failed to get snapshot 3 details: %v", err) + } else { + t.Logf("Snapshot 3 details: ID=%s, Name=%s, ParentID=%s", snapshot3.ID, snapshot3.Name, snapshot3.ParentID) + } + + // Verify that snapshot 3's parent ID has been updated to point to snapshot 1 + if snapshot3 != nil && snapshot3.ParentID != snapshot1ID { + t.Errorf("Snapshot 3's parent ID should be updated to point to Snapshot 1 after Snapshot 2 is deleted. Got ParentID=%s, want ParentID=%s", snapshot3.ParentID, snapshot1ID) + } + if len(snapshots) != 2 { t.Errorf("Expected 2 snapshots after deletion, got %d", len(snapshots)) } for _, snap := range snapshots { if snap.ID == snapshot2ID { - t.Errorf("Snapshot 2 should have been deleted") + t.Errorf("Snapshot 2 (ID=%s) should have been deleted", snapshot2ID) } } } diff --git a/grpc/server.go b/grpc/server.go index 9c0464e..07b8bd0 100644 --- a/grpc/server.go +++ b/grpc/server.go @@ -58,7 +58,11 @@ func (s *SnapshotServer) Stop(ctx context.Context) error { // ListSnapshots implements the gRPC ListSnapshots method func (s *SnapshotServer) ListSnapshots(ctx context.Context, req *ListSnapshotsRequest) (*ListSnapshotsResponse, error) { - snapshots, err := s.manager.ListSnapshots(ctx) + // Create empty ListOptions since the proto doesn't have active filter/pagination fields yet + opts := store.ListOptions{} + + // Call manager with the required ListOptions parameter + snapshots, err := s.manager.ListSnapshots(ctx, opts) if err != nil { return nil, fmt.Errorf("failed to list snapshots: %w", err) } diff --git a/grpc_test.go b/grpc_test.go index c477c7f..19ef8c6 100644 --- a/grpc_test.go +++ b/grpc_test.go @@ -1,9 +1,12 @@ package agate import ( + "bytes" "context" + "log" "os" "path/filepath" + "strings" "testing" "time" @@ -202,3 +205,185 @@ func TestGRPCServerClient(t *testing.T) { t.Errorf("file2.txt should not exist in the downloaded snapshot") } } + +// TestGRPC_GetRemoteSnapshot_Incremental tests the incremental download functionality +// of GetRemoteSnapshot, verifying that it reuses files from a local parent snapshot +// instead of downloading them again. +func TestGRPC_GetRemoteSnapshot_Incremental(t *testing.T) { + // Skip this test in short mode + if testing.Short() { + t.Skip("Skipping incremental GetRemoteSnapshot test in short mode") + } + + // Create a temporary directory for the server + serverDir, err := os.MkdirTemp("", "agate-server-*") + if err != nil { + t.Fatalf("Failed to create server temp directory: %v", err) + } + defer os.RemoveAll(serverDir) + + // Create a temporary directory for the client + clientDir, err := os.MkdirTemp("", "agate-client-*") + if err != nil { + t.Fatalf("Failed to create client temp directory: %v", err) + } + defer os.RemoveAll(clientDir) + + // Create a buffer to capture client logs + var clientLogBuffer bytes.Buffer + clientLogger := log.New(&clientLogBuffer, "", 0) + + // Create Agate options for the server + serverOptions := AgateOptions{ + WorkDir: serverDir, + } + + // Create Agate options for the client with logger + clientOptions := AgateOptions{ + WorkDir: clientDir, + Logger: clientLogger, + } + + // Create Agate instances for server and client + serverAgate, err := New(serverOptions) + if err != nil { + t.Fatalf("Failed to create server Agate instance: %v", err) + } + defer serverAgate.Close() + + clientAgate, err := New(clientOptions) + if err != nil { + t.Fatalf("Failed to create client Agate instance: %v", err) + } + defer clientAgate.Close() + + // Create a data directory on the server + serverDataDir := serverAgate.options.BlobStore.GetActiveDir() + if err := os.MkdirAll(serverDataDir, 0755); err != nil { + t.Fatalf("Failed to create server data directory: %v", err) + } + + // Create test files for snapshot A on the server + if err := os.MkdirAll(filepath.Join(serverDataDir, "subdir"), 0755); err != nil { + t.Fatalf("Failed to create subdirectory: %v", err) + } + + snapshotAFiles := map[string]string{ + filepath.Join(serverDataDir, "file1.txt"): "Content of file 1", + filepath.Join(serverDataDir, "file2.txt"): "Content of file 2", + filepath.Join(serverDataDir, "subdir/file3.txt"): "Content of file 3", + } + + for path, content := range snapshotAFiles { + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatalf("Failed to create test file %s: %v", path, err) + } + } + + // Create snapshot A on the server + ctx := context.Background() + snapshotAID, err := serverAgate.SaveSnapshot(ctx, "Snapshot A", "") + if err != nil { + t.Fatalf("Failed to create snapshot A: %v", err) + } + t.Logf("Created snapshot A with ID: %s", snapshotAID) + + // Modify some files and add a new file for snapshot B + snapshotBChanges := map[string]string{ + filepath.Join(serverDataDir, "file1.txt"): "Modified content of file 1", // Modified file + filepath.Join(serverDataDir, "file4.txt"): "Content of new file 4", // New file + filepath.Join(serverDataDir, "subdir/file5.txt"): "Content of new file 5", // New file in subdir + } + + for path, content := range snapshotBChanges { + // Ensure parent directory exists + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatalf("Failed to create directory for %s: %v", path, err) + } + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatalf("Failed to create/modify test file %s: %v", path, err) + } + } + + // Create snapshot B on the server (with A as parent) + snapshotBID, err := serverAgate.SaveSnapshot(ctx, "Snapshot B", snapshotAID) + if err != nil { + t.Fatalf("Failed to create snapshot B: %v", err) + } + t.Logf("Created snapshot B with ID: %s", snapshotBID) + + // Start the gRPC server + serverAddress := "localhost:50052" // Use a different port than the other test + server, err := remote.RunServer(ctx, serverAgate.manager, serverAddress) + if err != nil { + t.Fatalf("Failed to start gRPC server: %v", err) + } + defer server.Stop(ctx) + + // Give the server a moment to start + time.Sleep(100 * time.Millisecond) + + // Step 1: Client downloads snapshot A + err = clientAgate.GetRemoteSnapshot(ctx, serverAddress, snapshotAID, "") + if err != nil { + t.Fatalf("Failed to download snapshot A: %v", err) + } + t.Log("Client successfully downloaded snapshot A") + + // Clear the log buffer to capture only logs from the incremental download + clientLogBuffer.Reset() + + // Step 2: Client downloads snapshot B, specifying A as the local parent + err = clientAgate.GetRemoteSnapshot(ctx, serverAddress, snapshotBID, snapshotAID) + if err != nil { + t.Fatalf("Failed to download snapshot B: %v", err) + } + t.Log("Client successfully downloaded snapshot B") + + // Step 3: Verify that snapshot B was correctly imported + // Restore snapshot B to a directory + restoreDir := filepath.Join(clientDir, "restore") + if err := os.MkdirAll(restoreDir, 0755); err != nil { + t.Fatalf("Failed to create restore directory: %v", err) + } + + err = clientAgate.RestoreSnapshotToDir(ctx, snapshotBID, restoreDir) + if err != nil { + t.Fatalf("Failed to restore snapshot B: %v", err) + } + + // Verify the restored files match the expected content + expectedFiles := map[string]string{ + filepath.Join(restoreDir, "file1.txt"): "Modified content of file 1", // Modified file + filepath.Join(restoreDir, "file2.txt"): "Content of file 2", // Unchanged file + filepath.Join(restoreDir, "file4.txt"): "Content of new file 4", // New file + filepath.Join(restoreDir, "subdir/file3.txt"): "Content of file 3", // Unchanged file + filepath.Join(restoreDir, "subdir/file5.txt"): "Content of new file 5", // New file + } + + for path, expectedContent := range expectedFiles { + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("Failed to read restored file %s: %v", path, err) + } + if string(content) != expectedContent { + t.Errorf("Restored file %s has wrong content: got %s, want %s", path, string(content), expectedContent) + } + } + + // Step 4: Analyze logs to verify incremental download behavior + logs := clientLogBuffer.String() + + // Check for evidence of file reuse + if !strings.Contains(logs, "Reusing file") { + t.Errorf("No evidence of file reuse in logs") + } + + // Check for evidence of downloading only new/changed files + if !strings.Contains(logs, "Downloading file") { + t.Errorf("No evidence of downloading new files in logs") + } + + // Log the relevant parts for debugging + t.Logf("Log evidence of incremental download:\n%s", logs) +} diff --git a/interfaces/snapshot.go b/interfaces/snapshot.go index 941ad2c..13ff365 100644 --- a/interfaces/snapshot.go +++ b/interfaces/snapshot.go @@ -7,27 +7,40 @@ import ( "gitea.unprism.ru/KRBL/Agate/store" ) -// SnapshotManager defines the interface that the server needs to interact with snapshots +// SnapshotManager is an interface that defines operations for managing and interacting with snapshots. type SnapshotManager interface { - // GetSnapshotDetails retrieves detailed metadata for a specific snapshot + // CreateSnapshot creates a new snapshot from the specified source directory, associating it with a given name and parent ID. + // Returns the created Snapshot with its metadata or an error if the process fails. + CreateSnapshot(ctx context.Context, sourceDir string, name string, parentID string) (*store.Snapshot, error) + + // GetSnapshotDetails retrieves detailed metadata for a specific snapshot identified by its unique snapshotID. + // Returns a Snapshot object containing metadata GetSnapshotDetails(ctx context.Context, snapshotID string) (*store.Snapshot, error) - // ListSnapshots retrieves a list of all available snapshots - ListSnapshots(ctx context.Context) ([]store.SnapshotInfo, error) + // ListSnapshots retrieves a list of available snapshots with filtering and pagination options. + ListSnapshots(ctx context.Context, opts store.ListOptions) ([]store.SnapshotInfo, error) - // OpenFile retrieves and opens a file from the specified snapshot + // DeleteSnapshot removes a snapshot identified by snapshotID. Returns an error if the snapshot does not exist or cannot be deleted. + DeleteSnapshot(ctx context.Context, snapshotID string) error + + // OpenFile retrieves and opens a file from the specified snapshot, returning a readable stream and an error, if any. OpenFile(ctx context.Context, snapshotID string, filePath string) (io.ReadCloser, error) - // CreateSnapshot creates a new snapshot from the specified source directory - CreateSnapshot(ctx context.Context, sourceDir string, name string, parentID string) (*store.Snapshot, error) + // ExtractSnapshot extracts the contents of a specified snapshot to a target directory at the given path. + // If cleanTarget is true, the target directory will be cleaned before extraction. + // Returns an error if the snapshot ID is invalid or the extraction fails. + ExtractSnapshot(ctx context.Context, snapshotID string, path string, cleanTarget bool) error + + // UpdateSnapshotMetadata updates the metadata of an existing snapshot, allowing changes to its name. + UpdateSnapshotMetadata(ctx context.Context, snapshotID string, newName string) error } // SnapshotServer defines the interface for a server that can share snapshots type SnapshotServer interface { - // Start initializes and begins the server's operation + // Start initializes and begins the server's operation, handling incoming requests or processes within the provided context. Start(ctx context.Context) error - // Stop gracefully shuts down the server + // Stop gracefully shuts down the server, releasing any allocated resources and ensuring all operations are completed. Stop(ctx context.Context) error } diff --git a/manager.go b/manager.go index c3e8154..fc8df9a 100644 --- a/manager.go +++ b/manager.go @@ -6,9 +6,9 @@ import ( "errors" "fmt" "io" + "log" "os" "path/filepath" - "sort" "strings" "time" @@ -16,20 +16,26 @@ import ( "gitea.unprism.ru/KRBL/Agate/archive" "gitea.unprism.ru/KRBL/Agate/hash" + "gitea.unprism.ru/KRBL/Agate/interfaces" "gitea.unprism.ru/KRBL/Agate/store" ) type SnapshotManagerData struct { metadataStore store.MetadataStore blobStore store.BlobStore + logger *log.Logger } -func CreateSnapshotManager(metadataStore store.MetadataStore, blobStore store.BlobStore) (SnapshotManager, error) { +func CreateSnapshotManager(metadataStore store.MetadataStore, blobStore store.BlobStore, logger *log.Logger) (interfaces.SnapshotManager, error) { if metadataStore == nil || blobStore == nil { return nil, errors.New("parameters can't be nil") } - return &SnapshotManagerData{metadataStore, blobStore}, nil + return &SnapshotManagerData{ + metadataStore: metadataStore, + blobStore: blobStore, + logger: logger, + }, nil } func (data *SnapshotManagerData) CreateSnapshot(ctx context.Context, sourceDir string, name string, parentID string) (*store.Snapshot, error) { @@ -162,9 +168,9 @@ func (data *SnapshotManagerData) GetSnapshotDetails(ctx context.Context, snapsho return snapshot, nil } -func (data *SnapshotManagerData) ListSnapshots(ctx context.Context) ([]store.SnapshotInfo, error) { - // Retrieve list of snapshots from the metadata store - snapshots, err := data.metadataStore.ListSnapshotsMetadata(ctx) +func (data *SnapshotManagerData) ListSnapshots(ctx context.Context, opts store.ListOptions) ([]store.SnapshotInfo, error) { + // Retrieve list of snapshots from the metadata store with the provided options + snapshots, err := data.metadataStore.ListSnapshotsMetadata(ctx, opts) if err != nil { return nil, fmt.Errorf("failed to list snapshots: %w", err) } @@ -177,8 +183,8 @@ func (data *SnapshotManagerData) DeleteSnapshot(ctx context.Context, snapshotID return errors.New("snapshot ID cannot be empty") } - // First check if the snapshot exists - _, err := data.metadataStore.GetSnapshotMetadata(ctx, snapshotID) + // First check if the snapshot exists and get its details + snapshot, err := data.metadataStore.GetSnapshotMetadata(ctx, snapshotID) if err != nil { if errors.Is(err, ErrNotFound) { // If snapshot doesn't exist, return success (idempotent operation) @@ -187,6 +193,39 @@ func (data *SnapshotManagerData) DeleteSnapshot(ctx context.Context, snapshotID return fmt.Errorf("failed to check if snapshot exists: %w", err) } + // Get the parent ID of the snapshot being deleted + parentID := snapshot.ParentID + + // Find all snapshots that have the deleted snapshot as their parent + // We need to update their parent ID to maintain the chain + opts := store.ListOptions{} + allSnapshots, err := data.metadataStore.ListSnapshotsMetadata(ctx, opts) + if err != nil { + return fmt.Errorf("failed to list snapshots: %w", err) + } + + // Update parent references for any snapshots that have this one as a parent + for _, info := range allSnapshots { + if info.ParentID == snapshotID { + // Get the full snapshot details + childSnapshot, err := data.metadataStore.GetSnapshotMetadata(ctx, info.ID) + if err != nil { + data.logger.Printf("WARNING: failed to get child snapshot %s details: %v", info.ID, err) + continue + } + + // Update the parent ID to point to the deleted snapshot's parent + childSnapshot.ParentID = parentID + + // Save the updated snapshot + if err := data.metadataStore.SaveSnapshotMetadata(ctx, *childSnapshot); err != nil { + data.logger.Printf("WARNING: failed to update parent reference for snapshot %s: %v", info.ID, err) + } else { + data.logger.Printf("Updated parent reference for snapshot %s from %s to %s", info.ID, snapshotID, parentID) + } + } + } + // Delete the metadata first if err := data.metadataStore.DeleteSnapshotMetadata(ctx, snapshotID); err != nil { return fmt.Errorf("failed to delete snapshot metadata: %w", err) @@ -197,7 +236,7 @@ func (data *SnapshotManagerData) DeleteSnapshot(ctx context.Context, snapshotID // Note: We don't return here because we've already deleted the metadata // and the blob store should handle the case where the blob doesn't exist // Log the error instead - fmt.Printf("Warning: failed to delete snapshot blob: %v\n", err) + data.logger.Printf("WARNING: failed to delete snapshot blob: %v", err) } return nil @@ -248,7 +287,7 @@ func (data *SnapshotManagerData) OpenFile(ctx context.Context, snapshotID string return pr, nil } -func (data *SnapshotManagerData) ExtractSnapshot(ctx context.Context, snapshotID string, path string) error { +func (data *SnapshotManagerData) ExtractSnapshot(ctx context.Context, snapshotID string, path string, cleanTarget bool) error { if snapshotID == "" { return errors.New("snapshot ID cannot be empty") } @@ -258,8 +297,8 @@ func (data *SnapshotManagerData) ExtractSnapshot(ctx context.Context, snapshotID path = data.blobStore.GetActiveDir() } - // First check if the snapshot exists and get its metadata - snapshot, err := data.metadataStore.GetSnapshotMetadata(ctx, snapshotID) + // First check if the snapshot exists + _, err := data.metadataStore.GetSnapshotMetadata(ctx, snapshotID) if err != nil { if errors.Is(err, ErrNotFound) { return ErrNotFound @@ -273,9 +312,20 @@ func (data *SnapshotManagerData) ExtractSnapshot(ctx context.Context, snapshotID return fmt.Errorf("failed to get blob path: %w", err) } - // Ensure the target directory exists - if err := os.MkdirAll(path, 0755); err != nil { - return fmt.Errorf("failed to create target directory: %w", err) + // If cleanTarget is true, clean the target directory before extraction + if cleanTarget { + // Remove the directory and recreate it + if err := os.RemoveAll(path); err != nil { + return fmt.Errorf("failed to clean target directory: %w", err) + } + if err := os.MkdirAll(path, 0755); err != nil { + return fmt.Errorf("failed to create target directory: %w", err) + } + } else { + // Just ensure the target directory exists + if err := os.MkdirAll(path, 0755); err != nil { + return fmt.Errorf("failed to create target directory: %w", err) + } } // Extract the archive to the target directory @@ -283,94 +333,6 @@ func (data *SnapshotManagerData) ExtractSnapshot(ctx context.Context, snapshotID return fmt.Errorf("failed to extract snapshot: %w", err) } - // Create maps for files and directories in the snapshot for quick lookup - snapshotFiles := make(map[string]bool) - snapshotDirs := make(map[string]bool) - for _, file := range snapshot.Files { - if file.IsDir { - snapshotDirs[file.Path] = true - } else { - snapshotFiles[file.Path] = true - } - } - - // First pass: Collect all files and directories in the target - var allPaths []string - err = filepath.Walk(path, func(filePath string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - // Skip the root directory itself - if filePath == path { - return nil - } - - // Create relative path - relPath, err := filepath.Rel(path, filePath) - if err != nil { - return fmt.Errorf("failed to get relative path: %w", err) - } - relPath = filepath.ToSlash(relPath) - - allPaths = append(allPaths, filePath) - return nil - }) - - if err != nil { - return fmt.Errorf("failed to scan target directory: %w", err) - } - - // Sort paths by length in descending order to process deepest paths first - // This ensures we process files before their parent directories - sort.Slice(allPaths, func(i, j int) bool { - return len(allPaths[i]) > len(allPaths[j]) - }) - - // Second pass: Remove files and directories that aren't in the snapshot - for _, filePath := range allPaths { - info, err := os.Stat(filePath) - if err != nil { - // Skip if file no longer exists (might have been in a directory we already removed) - if os.IsNotExist(err) { - continue - } - return fmt.Errorf("failed to stat file %s: %w", filePath, err) - } - - // Create relative path - relPath, err := filepath.Rel(path, filePath) - if err != nil { - return fmt.Errorf("failed to get relative path: %w", err) - } - relPath = filepath.ToSlash(relPath) - - if info.IsDir() { - // For directories, check if it's in the snapshot or if it's empty - if !snapshotDirs[relPath] { - // Check if directory is empty - entries, err := os.ReadDir(filePath) - if err != nil { - return fmt.Errorf("failed to read directory %s: %w", filePath, err) - } - - // If directory is empty, remove it - if len(entries) == 0 { - if err := os.Remove(filePath); err != nil { - return fmt.Errorf("failed to remove directory %s: %w", filePath, err) - } - } - } - } else { - // For files, remove if not in the snapshot - if !snapshotFiles[relPath] { - if err := os.Remove(filePath); err != nil { - return fmt.Errorf("failed to remove file %s: %w", filePath, err) - } - } - } - } - return nil } diff --git a/manager_test.go b/manager_test.go index 9d3e38d..c1c86a4 100644 --- a/manager_test.go +++ b/manager_test.go @@ -90,8 +90,8 @@ func TestCreateAndGetSnapshot(t *testing.T) { } createTestFiles(t, sourceDir) - // Create a snapshot manager - manager, err := CreateSnapshotManager(metadataStore, blobStore) + // Create a snapshot manager with nil logger + manager, err := CreateSnapshotManager(metadataStore, blobStore, nil) if err != nil { t.Fatalf("Failed to create snapshot manager: %v", err) } @@ -142,8 +142,8 @@ func TestListSnapshots(t *testing.T) { } createTestFiles(t, sourceDir) - // Create a snapshot manager - manager, err := CreateSnapshotManager(metadataStore, blobStore) + // Create a snapshot manager with nil logger + manager, err := CreateSnapshotManager(metadataStore, blobStore, nil) if err != nil { t.Fatalf("Failed to create snapshot manager: %v", err) } @@ -165,8 +165,8 @@ func TestListSnapshots(t *testing.T) { t.Fatalf("Failed to create snapshot: %v", err) } - // List the snapshots - snapshots, err := manager.ListSnapshots(ctx) + // List the snapshots with empty options + snapshots, err := manager.ListSnapshots(ctx, store.ListOptions{}) if err != nil { t.Fatalf("Failed to list snapshots: %v", err) } @@ -209,8 +209,8 @@ func TestDeleteSnapshot(t *testing.T) { } createTestFiles(t, sourceDir) - // Create a snapshot manager - manager, err := CreateSnapshotManager(metadataStore, blobStore) + // Create a snapshot manager with nil logger + manager, err := CreateSnapshotManager(metadataStore, blobStore, nil) if err != nil { t.Fatalf("Failed to create snapshot manager: %v", err) } @@ -235,7 +235,7 @@ func TestDeleteSnapshot(t *testing.T) { } // List snapshots to confirm it's gone - snapshots, err := manager.ListSnapshots(ctx) + snapshots, err := manager.ListSnapshots(ctx, store.ListOptions{}) if err != nil { t.Fatalf("Failed to list snapshots: %v", err) } @@ -255,8 +255,8 @@ func TestOpenFile(t *testing.T) { } createTestFiles(t, sourceDir) - // Create a snapshot manager - manager, err := CreateSnapshotManager(metadataStore, blobStore) + // Create a snapshot manager with nil logger + manager, err := CreateSnapshotManager(metadataStore, blobStore, nil) if err != nil { t.Fatalf("Failed to create snapshot manager: %v", err) } @@ -309,8 +309,8 @@ func TestExtractSnapshot(t *testing.T) { } createTestFiles(t, sourceDir) - // Create a snapshot manager - manager, err := CreateSnapshotManager(metadataStore, blobStore) + // Create a snapshot manager with nil logger + manager, err := CreateSnapshotManager(metadataStore, blobStore, nil) if err != nil { t.Fatalf("Failed to create snapshot manager: %v", err) } @@ -328,8 +328,8 @@ func TestExtractSnapshot(t *testing.T) { t.Fatalf("Failed to create target directory: %v", err) } - // Extract the snapshot - err = manager.ExtractSnapshot(ctx, snapshot.ID, targetDir) + // Extract the snapshot with default behavior (cleanTarget=false) + err = manager.ExtractSnapshot(ctx, snapshot.ID, targetDir, false) if err != nil { t.Fatalf("Failed to extract snapshot: %v", err) } @@ -353,12 +353,159 @@ func TestExtractSnapshot(t *testing.T) { } // Try to extract a non-existent snapshot - err = manager.ExtractSnapshot(ctx, "nonexistent-id", targetDir) + err = manager.ExtractSnapshot(ctx, "nonexistent-id", targetDir, false) if err == nil { t.Fatalf("Expected error when extracting non-existent snapshot, got nil") } } +// TestExtractSnapshot_SafeRestore tests that ExtractSnapshot with cleanTarget=false +// does not remove extra files in the target directory +func TestExtractSnapshot_SafeRestore(t *testing.T) { + tempDir, metadataStore, blobStore, cleanup := setupTestEnvironment(t) + defer cleanup() + + // Create a source directory with test files + sourceDir := filepath.Join(tempDir, "source") + if err := os.MkdirAll(sourceDir, 0755); err != nil { + t.Fatalf("Failed to create source directory: %v", err) + } + createTestFiles(t, sourceDir) + + // Create a snapshot manager with nil logger + manager, err := CreateSnapshotManager(metadataStore, blobStore, nil) + if err != nil { + t.Fatalf("Failed to create snapshot manager: %v", err) + } + + // Create a snapshot (snapshot A) + ctx := context.Background() + snapshot, err := manager.CreateSnapshot(ctx, sourceDir, "Snapshot A", "") + if err != nil { + t.Fatalf("Failed to create snapshot: %v", err) + } + + // Create a target directory and place an "extra" file in it + targetDir := filepath.Join(tempDir, "target") + if err := os.MkdirAll(targetDir, 0755); err != nil { + t.Fatalf("Failed to create target directory: %v", err) + } + extraFilePath := filepath.Join(targetDir, "extra.txt") + if err := os.WriteFile(extraFilePath, []byte("This is an extra file"), 0644); err != nil { + t.Fatalf("Failed to create extra file: %v", err) + } + + // Extract the snapshot with cleanTarget=false + err = manager.ExtractSnapshot(ctx, snapshot.ID, targetDir, false) + if err != nil { + t.Fatalf("Failed to extract snapshot: %v", err) + } + + // Check that all files from the snapshot were restored + testFiles := map[string]string{ + filepath.Join(targetDir, "file1.txt"): "This is file 1", + filepath.Join(targetDir, "file2.txt"): "This is file 2", + filepath.Join(targetDir, "subdir/subfile1.txt"): "This is subfile 1", + filepath.Join(targetDir, "subdir/subfile2.txt"): "This is subfile 2", + } + + for path, expectedContent := range testFiles { + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("Failed to read extracted file %s: %v", path, err) + } + if string(content) != expectedContent { + t.Errorf("Extracted file %s has wrong content: got %s, want %s", path, string(content), expectedContent) + } + } + + // Check that the extra file was NOT deleted + if _, err := os.Stat(extraFilePath); os.IsNotExist(err) { + t.Errorf("Extra file was deleted, but should have been preserved with cleanTarget=false") + } else if err != nil { + t.Fatalf("Failed to check if extra file exists: %v", err) + } else { + // Read the content to make sure it wasn't modified + content, err := os.ReadFile(extraFilePath) + if err != nil { + t.Fatalf("Failed to read extra file: %v", err) + } + if string(content) != "This is an extra file" { + t.Errorf("Extra file content was modified: got %s, want %s", string(content), "This is an extra file") + } + } +} + +// TestExtractSnapshot_CleanRestore tests that ExtractSnapshot with cleanTarget=true +// completely cleans the target directory before restoration +func TestExtractSnapshot_CleanRestore(t *testing.T) { + tempDir, metadataStore, blobStore, cleanup := setupTestEnvironment(t) + defer cleanup() + + // Create a source directory with test files + sourceDir := filepath.Join(tempDir, "source") + if err := os.MkdirAll(sourceDir, 0755); err != nil { + t.Fatalf("Failed to create source directory: %v", err) + } + createTestFiles(t, sourceDir) + + // Create a snapshot manager with nil logger + manager, err := CreateSnapshotManager(metadataStore, blobStore, nil) + if err != nil { + t.Fatalf("Failed to create snapshot manager: %v", err) + } + + // Create a snapshot (snapshot A) + ctx := context.Background() + snapshot, err := manager.CreateSnapshot(ctx, sourceDir, "Snapshot A", "") + if err != nil { + t.Fatalf("Failed to create snapshot: %v", err) + } + + // Create a target directory and place an "extra" file in it + targetDir := filepath.Join(tempDir, "target") + if err := os.MkdirAll(targetDir, 0755); err != nil { + t.Fatalf("Failed to create target directory: %v", err) + } + extraFilePath := filepath.Join(targetDir, "extra.txt") + if err := os.WriteFile(extraFilePath, []byte("This is an extra file"), 0644); err != nil { + t.Fatalf("Failed to create extra file: %v", err) + } + + // Extract the snapshot with cleanTarget=true + err = manager.ExtractSnapshot(ctx, snapshot.ID, targetDir, true) + if err != nil { + t.Fatalf("Failed to extract snapshot: %v", err) + } + + // Check that all files from the snapshot were restored + testFiles := map[string]string{ + filepath.Join(targetDir, "file1.txt"): "This is file 1", + filepath.Join(targetDir, "file2.txt"): "This is file 2", + filepath.Join(targetDir, "subdir/subfile1.txt"): "This is subfile 1", + filepath.Join(targetDir, "subdir/subfile2.txt"): "This is subfile 2", + } + + for path, expectedContent := range testFiles { + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("Failed to read extracted file %s: %v", path, err) + } + if string(content) != expectedContent { + t.Errorf("Extracted file %s has wrong content: got %s, want %s", path, string(content), expectedContent) + } + } + + // Check that the extra file WAS deleted + if _, err := os.Stat(extraFilePath); os.IsNotExist(err) { + // This is the expected behavior + } else if err != nil { + t.Fatalf("Failed to check if extra file exists: %v", err) + } else { + t.Errorf("Extra file was not deleted, but should have been removed with cleanTarget=true") + } +} + func TestUpdateSnapshotMetadata(t *testing.T) { tempDir, metadataStore, blobStore, cleanup := setupTestEnvironment(t) defer cleanup() @@ -370,8 +517,8 @@ func TestUpdateSnapshotMetadata(t *testing.T) { } createTestFiles(t, sourceDir) - // Create a snapshot manager - manager, err := CreateSnapshotManager(metadataStore, blobStore) + // Create a snapshot manager with nil logger + manager, err := CreateSnapshotManager(metadataStore, blobStore, nil) if err != nil { t.Fatalf("Failed to create snapshot manager: %v", err) } diff --git a/remote/server.go b/remote/server.go index 5c02ba1..dffed9c 100644 --- a/remote/server.go +++ b/remote/server.go @@ -59,7 +59,10 @@ func (s *Server) Stop(ctx context.Context) error { // ListSnapshots implements the gRPC ListSnapshots method func (s *Server) ListSnapshots(ctx context.Context, req *agateGrpc.ListSnapshotsRequest) (*agateGrpc.ListSnapshotsResponse, error) { - snapshots, err := s.manager.ListSnapshots(ctx) + // Create empty ListOptions since the proto doesn't have active filter/pagination fields yet + opts := store.ListOptions{} + + snapshots, err := s.manager.ListSnapshots(ctx, opts) if err != nil { return nil, fmt.Errorf("failed to list snapshots: %w", err) } diff --git a/snapshot.go b/snapshot.go deleted file mode 100644 index 86e5bfb..0000000 --- a/snapshot.go +++ /dev/null @@ -1,53 +0,0 @@ -package agate - -import ( - "context" - "gitea.unprism.ru/KRBL/Agate/store" - "io" -) - -// SnapshotManager is an interface that defines operations for managing and interacting with snapshots. -type SnapshotManager interface { - // CreateSnapshot creates a new snapshot from the specified source directory, associating it with a given name and parent ID. - // Returns the created Snapshot with its metadata or an error if the process fails. - CreateSnapshot(ctx context.Context, sourceDir string, name string, parentID string) (*store.Snapshot, error) - - // GetSnapshotDetails retrieves detailed metadata for a specific snapshot identified by its unique snapshotID. - // Returns a Snapshot object containing metadata - GetSnapshotDetails(ctx context.Context, snapshotID string) (*store.Snapshot, error) - - // ListSnapshots retrieves a list of all available snapshots, returning their basic information as SnapshotInfo. - ListSnapshots(ctx context.Context) ([]store.SnapshotInfo, error) - - // DeleteSnapshot removes a snapshot identified by snapshotID. Returns an error if the snapshot does not exist or cannot be deleted. - DeleteSnapshot(ctx context.Context, snapshotID string) error - - // OpenFile retrieves and opens a file from the specified snapshot, returning a readable stream and an error, if any. - OpenFile(ctx context.Context, snapshotID string, filePath string) (io.ReadCloser, error) - - // ExtractSnapshot extracts the contents of a specified snapshot to a target directory at the given path. - // Returns an error if the snapshot ID is invalid or the extraction fails. - ExtractSnapshot(ctx context.Context, snapshotID string, path string) error - - // UpdateSnapshotMetadata updates the metadata of an existing snapshot, allowing changes to its name. - UpdateSnapshotMetadata(ctx context.Context, snapshotID string, newName string) error -} - -type SnapshotServer interface { - // Start initializes and begins the server's operation, handling incoming requests or processes within the provided context. - Start(ctx context.Context) error - - // Stop gracefully shuts down the server, releasing any allocated resources and ensuring all operations are completed. - Stop(ctx context.Context) error -} - -type SnapshotClient interface { - // ListSnapshots retrieves a list of snapshots containing basic metadata, such as ID, name, parent ID, and creation time. - ListSnapshots(ctx context.Context) ([]store.SnapshotInfo, error) - - // FetchSnapshotDetails retrieves detailed metadata about a specific snapshot identified by snapshotID. - FetchSnapshotDetails(ctx context.Context, snapshotID string) (*store.Snapshot, error) - - // DownloadSnapshot retrieves the snapshot content for the given snapshotID and returns it as an io.ReadCloser. - DownloadSnapshot(ctx context.Context, snapshotID string) (io.ReadCloser, error) -} diff --git a/store/sqlite/sqlite.go b/store/sqlite/sqlite.go index 88cbfa8..bd20a00 100644 --- a/store/sqlite/sqlite.go +++ b/store/sqlite/sqlite.go @@ -167,58 +167,83 @@ func (s *sqliteStore) GetSnapshotMetadata(ctx context.Context, snapshotID string return &snap, nil } -// ListSnapshotsMetadata извлекает краткую информацию обо всех снапшотах. -func (s *sqliteStore) ListSnapshotsMetadata(ctx context.Context) ([]store.SnapshotInfo, error) { - // Simplified implementation to debug the issue - fmt.Println("ListSnapshotsMetadata called") +// ListSnapshotsMetadata retrieves basic information about snapshots with filtering and pagination. +func (s *sqliteStore) ListSnapshotsMetadata(ctx context.Context, opts store.ListOptions) ([]store.SnapshotInfo, error) { + // Build the query with optional filtering + var query string + var args []interface{} - // Get all snapshot IDs first - query := `SELECT id FROM snapshots ORDER BY creation_time DESC;` - fmt.Println("Executing query:", query) + if opts.FilterByName != "" { + query = `SELECT id, name, parent_id, creation_time FROM snapshots WHERE name LIKE ? ORDER BY creation_time DESC` + args = append(args, "%"+opts.FilterByName+"%") + } else { + query = `SELECT id, name, parent_id, creation_time FROM snapshots ORDER BY creation_time DESC` + } - rows, err := s.db.QueryContext(ctx, query) + // Add pagination if specified + if opts.Limit > 0 { + query += " LIMIT ?" + args = append(args, opts.Limit) + + if opts.Offset > 0 { + query += " OFFSET ?" + args = append(args, opts.Offset) + } + } + + // Execute the query + rows, err := s.db.QueryContext(ctx, query, args...) if err != nil { - return nil, fmt.Errorf("failed to query snapshot IDs: %w", err) + return nil, fmt.Errorf("failed to query snapshots: %w", err) } defer rows.Close() var snapshots []store.SnapshotInfo - // For each ID, get the full snapshot details + // Iterate through the results for rows.Next() { - var id string - if err := rows.Scan(&id); err != nil { - return nil, fmt.Errorf("failed to scan snapshot ID: %w", err) + var info store.SnapshotInfo + var parentID sql.NullString + var creationTimeStr string + + if err := rows.Scan(&info.ID, &info.Name, &parentID, &creationTimeStr); err != nil { + return nil, fmt.Errorf("failed to scan snapshot row: %w", err) } - // Get the full snapshot details - snapshot, err := s.GetSnapshotMetadata(ctx, id) - if err != nil { - return nil, fmt.Errorf("failed to get snapshot details for ID %s: %w", id, err) + // Set parent ID if not NULL + if parentID.Valid { + info.ParentID = parentID.String } - // Convert to SnapshotInfo - info := store.SnapshotInfo{ - ID: snapshot.ID, - Name: snapshot.Name, - ParentID: snapshot.ParentID, - CreationTime: snapshot.CreationTime, + // Parse creation time + const sqliteLayout = "2006-01-02 15:04:05" // Standard SQLite DATETIME format without timezone + t, parseErr := time.Parse(sqliteLayout, creationTimeStr) + if parseErr != nil { + // Try format with milliseconds if the first one didn't work + const sqliteLayoutWithMs = "2006-01-02 15:04:05.999999999" + t, parseErr = time.Parse(sqliteLayoutWithMs, creationTimeStr) + if parseErr != nil { + // Try RFC3339 if saved as UTC().Format(time.RFC3339) + t, parseErr = time.Parse(time.RFC3339, creationTimeStr) + if parseErr != nil { + return nil, fmt.Errorf("failed to parse creation time '%s' for snapshot %s: %w", creationTimeStr, info.ID, parseErr) + } + } } + info.CreationTime = t.UTC() // Store as UTC snapshots = append(snapshots, info) } if err := rows.Err(); err != nil { - return nil, fmt.Errorf("error iterating snapshot IDs: %w", err) + return nil, fmt.Errorf("error iterating snapshot rows: %w", err) } // If no snapshots found, return an empty slice if len(snapshots) == 0 { - fmt.Println("No snapshots found") return []store.SnapshotInfo{}, nil } - fmt.Printf("Found %d snapshots\n", len(snapshots)) return snapshots, nil } diff --git a/store/sqlite/sqlite_test.go b/store/sqlite/sqlite_test.go index b3c1656..4ea3c8c 100644 --- a/store/sqlite/sqlite_test.go +++ b/store/sqlite/sqlite_test.go @@ -121,7 +121,7 @@ func TestListSnapshotsMetadata(t *testing.T) { // Create test snapshots ctx := context.Background() now := time.Now().UTC().Truncate(time.Second) - + testSnapshots := []store.Snapshot{ { ID: "snapshot-1", @@ -154,8 +154,8 @@ func TestListSnapshotsMetadata(t *testing.T) { } } - // List the snapshots - snapshots, err := s.ListSnapshotsMetadata(ctx) + // List the snapshots with empty options + snapshots, err := s.ListSnapshotsMetadata(ctx, store.ListOptions{}) if err != nil { t.Fatalf("Failed to list snapshots: %v", err) } @@ -189,6 +189,164 @@ func TestListSnapshotsMetadata(t *testing.T) { } } +func TestListSnapshotsMetadata_WithOptions(t *testing.T) { + // Create a temporary directory for tests + tempDir, err := os.MkdirTemp("", "agate-test-*") + if err != nil { + t.Fatalf("Failed to create temp directory: %v", err) + } + defer os.RemoveAll(tempDir) // Clean up after test + + // Create a new store + dbPath := filepath.Join(tempDir, "test.db") + s, err := NewSQLiteStore(dbPath) + if err != nil { + t.Fatalf("Failed to create SQLite store: %v", err) + } + defer s.Close() + + // Create test snapshots with different names + ctx := context.Background() + now := time.Now().UTC().Truncate(time.Second) + + testSnapshots := []store.Snapshot{ + { + ID: "alpha-1", + Name: "alpha-1", + ParentID: "", + CreationTime: now.Add(-3 * time.Hour), + Files: []store.FileInfo{}, + }, + { + ID: "alpha-2", + Name: "alpha-2", + ParentID: "alpha-1", + CreationTime: now.Add(-2 * time.Hour), + Files: []store.FileInfo{}, + }, + { + ID: "beta-1", + Name: "beta-1", + ParentID: "", + CreationTime: now.Add(-1 * time.Hour), + Files: []store.FileInfo{}, + }, + } + + // Save the snapshots + for _, snap := range testSnapshots { + err = s.SaveSnapshotMetadata(ctx, snap) + if err != nil { + t.Fatalf("Failed to save snapshot metadata: %v", err) + } + } + + // Test different ListOptions scenarios + t.Run("FilterByName", func(t *testing.T) { + // Filter snapshots by name "alpha" + opts := store.ListOptions{ + FilterByName: "alpha", + } + snapshots, err := s.ListSnapshotsMetadata(ctx, opts) + if err != nil { + t.Fatalf("Failed to list snapshots with filter: %v", err) + } + + // Should return 2 snapshots (alpha-1 and alpha-2) + if len(snapshots) != 2 { + t.Errorf("Wrong number of snapshots returned: got %d, want %d", len(snapshots), 2) + } + + // Check that only alpha snapshots are returned + for _, snap := range snapshots { + if snap.ID != "alpha-1" && snap.ID != "alpha-2" { + t.Errorf("Unexpected snapshot ID in filtered results: %s", snap.ID) + } + } + }) + + t.Run("Limit", func(t *testing.T) { + // Limit to 1 snapshot (should return the newest one) + opts := store.ListOptions{ + Limit: 1, + } + snapshots, err := s.ListSnapshotsMetadata(ctx, opts) + if err != nil { + t.Fatalf("Failed to list snapshots with limit: %v", err) + } + + // Should return 1 snapshot + if len(snapshots) != 1 { + t.Errorf("Wrong number of snapshots returned: got %d, want %d", len(snapshots), 1) + } + + // The newest snapshot should be beta-1 + if snapshots[0].ID != "beta-1" { + t.Errorf("Wrong snapshot returned with limit: got %s, want %s", snapshots[0].ID, "beta-1") + } + }) + + t.Run("Offset", func(t *testing.T) { + // Limit to 1 snapshot with offset 1 (should return the second newest) + opts := store.ListOptions{ + Limit: 1, + Offset: 1, + } + snapshots, err := s.ListSnapshotsMetadata(ctx, opts) + if err != nil { + t.Fatalf("Failed to list snapshots with offset: %v", err) + } + + // Should return 1 snapshot + if len(snapshots) != 1 { + t.Errorf("Wrong number of snapshots returned: got %d, want %d", len(snapshots), 1) + } + + // The second newest snapshot should be alpha-2 + if snapshots[0].ID != "alpha-2" { + t.Errorf("Wrong snapshot returned with offset: got %s, want %s", snapshots[0].ID, "alpha-2") + } + }) + + t.Run("FilterAndPagination", func(t *testing.T) { + // Filter by "alpha" with limit 1 + opts := store.ListOptions{ + FilterByName: "alpha", + Limit: 1, + } + snapshots, err := s.ListSnapshotsMetadata(ctx, opts) + if err != nil { + t.Fatalf("Failed to list snapshots with filter and pagination: %v", err) + } + + // Should return 1 snapshot + if len(snapshots) != 1 { + t.Errorf("Wrong number of snapshots returned: got %d, want %d", len(snapshots), 1) + } + + // The newest alpha snapshot should be alpha-2 + if snapshots[0].ID != "alpha-2" { + t.Errorf("Wrong snapshot returned with filter and limit: got %s, want %s", snapshots[0].ID, "alpha-2") + } + }) + + t.Run("NoResults", func(t *testing.T) { + // Filter by a name that doesn't exist + opts := store.ListOptions{ + FilterByName: "gamma", + } + snapshots, err := s.ListSnapshotsMetadata(ctx, opts) + if err != nil { + t.Fatalf("Failed to list snapshots with non-matching filter: %v", err) + } + + // Should return 0 snapshots + if len(snapshots) != 0 { + t.Errorf("Expected 0 snapshots, got %d", len(snapshots)) + } + }) +} + func TestDeleteSnapshotMetadata(t *testing.T) { // Create a temporary directory for tests tempDir, err := os.MkdirTemp("", "agate-test-*") @@ -238,4 +396,4 @@ func TestDeleteSnapshotMetadata(t *testing.T) { if err != nil { t.Fatalf("DeleteSnapshotMetadata returned an error for non-existent snapshot: %v", err) } -} \ No newline at end of file +} diff --git a/store/store.go b/store/store.go index 6599c65..d181b33 100644 --- a/store/store.go +++ b/store/store.go @@ -31,6 +31,13 @@ type SnapshotInfo struct { CreationTime time.Time // Время создания } +// ListOptions provides options for filtering and paginating snapshot lists +type ListOptions struct { + FilterByName string // Filter snapshots by name (substring match) + Limit int // Maximum number of snapshots to return + Offset int // Number of snapshots to skip +} + // MetadataStore определяет интерфейс для хранения и извлечения метаданных снапшотов. type MetadataStore interface { // SaveSnapshotMetadata сохраняет полные метаданные снапшота, включая список файлов. @@ -41,8 +48,8 @@ type MetadataStore interface { // Возвращает agate.ErrNotFound, если снапшот не найден. GetSnapshotMetadata(ctx context.Context, snapshotID string) (*Snapshot, error) - // ListSnapshotsMetadata извлекает краткую информацию обо всех снапшотах. - ListSnapshotsMetadata(ctx context.Context) ([]SnapshotInfo, error) + // ListSnapshotsMetadata извлекает краткую информацию о снапшотах с фильтрацией и пагинацией. + ListSnapshotsMetadata(ctx context.Context, opts ListOptions) ([]SnapshotInfo, error) // DeleteSnapshotMetadata удаляет метаданные снапшота по его ID. // Не должен возвращать ошибку, если снапшот не найден.