From b05058b5cd83a0a8764fc53396aeb29a99d8ef9d Mon Sep 17 00:00:00 2001 From: Alexander Lazarenko Date: Sat, 10 May 2025 00:57:34 +0300 Subject: [PATCH] Add active directory management for snapshot operations Introduced `GetActiveDir` and `CleanActiveDir` methods in the blob store to manage a dedicated directory for active snapshot operations. This ensures a clean working state before starting new operations and prevents conflicts. Updated related logic in snapshot creation and restoration to utilize the active directory. --- README.md | 310 +++++++++++++++++++++++++++++++++ manager.go | 27 ++- store/filesystem/filesystem.go | 41 ++++- store/store.go | 7 + 4 files changed, 376 insertions(+), 9 deletions(-) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..ed7fdab --- /dev/null +++ b/README.md @@ -0,0 +1,310 @@ +# Agate + +Agate is a Go library for creating, managing, and sharing snapshots of directories. It provides functionality for creating incremental snapshots, storing them efficiently, and sharing them over a network using gRPC. + +## Installation + +```bash +go get gitea.unprism.ru/KRBL/Agate +``` + +## Features + +- Create snapshots of directories +- Incremental snapshots (only store changes) +- Restore snapshots +- List and manage snapshots +- Share snapshots over a network using gRPC +- Connect to remote snapshot repositories + +## Basic Usage + +### Creating a Snapshot Repository + +To create a snapshot repository, you need to initialize the Agate library with the appropriate options: + +```go +package main + +import ( + "context" + "fmt" + "log" + "os" + "path/filepath" + + "gitea.unprism.ru/KRBL/Agate" + "gitea.unprism.ru/KRBL/Agate/stores" +) + +func main() { + // Create directories for your repository + workDir := "/path/to/your/repository" + if err := os.MkdirAll(workDir, 0755); err != nil { + log.Fatalf("Failed to create work directory: %v", err) + } + + // Initialize the default stores + metadataStore, blobStore, err := stores.InitDefaultStores(workDir) + if err != nil { + log.Fatalf("Failed to initialize stores: %v", err) + } + defer metadataStore.Close() + + // Initialize Agate + agateOptions := agate.AgateOptions{ + WorkDir: workDir, + MetadataStore: metadataStore, + BlobStore: blobStore, + } + + ag, err := agate.New(agateOptions) + if err != nil { + log.Fatalf("Failed to initialize Agate: %v", err) + } + defer ag.Close() + + // Create a snapshot + ctx := context.Background() + snapshotID, err := ag.SaveSnapshot(ctx, "My First Snapshot", "") + if err != nil { + log.Fatalf("Failed to create snapshot: %v", err) + } + fmt.Printf("Created snapshot with ID: %s\n", snapshotID) + + // List snapshots + snapshots, err := ag.ListSnapshots(ctx) + if err != nil { + log.Fatalf("Failed to list snapshots: %v", err) + } + fmt.Printf("Found %d snapshots:\n", len(snapshots)) + for _, s := range snapshots { + fmt.Printf(" - %s: %s (created at %s)\n", s.ID, s.Name, s.CreationTime.Format("2006-01-02 15:04:05")) + } +} +``` + +### Hosting a Snapshot Repository + +To host a snapshot repository and make it available over the network, you can use the `StartServer` method: + +```go +package main + +import ( + "context" + "log" + "os" + "os/signal" + "syscall" + + "gitea.unprism.ru/KRBL/Agate" + "gitea.unprism.ru/KRBL/Agate/stores" +) + +func main() { + // Create directories for your repository + workDir := "/path/to/your/repository" + if err := os.MkdirAll(workDir, 0755); err != nil { + log.Fatalf("Failed to create work directory: %v", err) + } + + // Initialize the default stores + metadataStore, blobStore, err := stores.InitDefaultStores(workDir) + if err != nil { + log.Fatalf("Failed to initialize stores: %v", err) + } + defer metadataStore.Close() + + // Initialize Agate + agateOptions := agate.AgateOptions{ + WorkDir: workDir, + MetadataStore: metadataStore, + BlobStore: blobStore, + } + + ag, err := agate.New(agateOptions) + if err != nil { + log.Fatalf("Failed to initialize Agate: %v", err) + } + defer ag.Close() + + // Start the gRPC server + ctx := context.Background() + address := "0.0.0.0:50051" // Listen on all interfaces, port 50051 + if err := ag.StartServer(ctx, address); err != nil { + log.Fatalf("Failed to start server: %v", err) + } + + log.Printf("Server started on %s", address) + + // Wait for termination signal + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + <-sigCh + + log.Println("Shutting down...") +} +``` + +### Connecting to a Hosted Snapshot Repository + +To connect to a hosted snapshot repository and retrieve snapshots: + +```go +package main + +import ( + "context" + "fmt" + "log" + "os" + + "gitea.unprism.ru/KRBL/Agate" + "gitea.unprism.ru/KRBL/Agate/stores" +) + +func main() { + // Create directories for your local repository + workDir := "/path/to/your/local/repository" + if err := os.MkdirAll(workDir, 0755); err != nil { + log.Fatalf("Failed to create work directory: %v", err) + } + + // Initialize the default stores + metadataStore, blobStore, err := stores.InitDefaultStores(workDir) + if err != nil { + log.Fatalf("Failed to initialize stores: %v", err) + } + defer metadataStore.Close() + + // Initialize Agate + agateOptions := agate.AgateOptions{ + WorkDir: workDir, + MetadataStore: metadataStore, + BlobStore: blobStore, + } + + ag, err := agate.New(agateOptions) + if err != nil { + log.Fatalf("Failed to initialize Agate: %v", err) + } + defer ag.Close() + + // Connect to a remote server + ctx := context.Background() + remoteAddress := "remote-server:50051" + + // List snapshots from the remote server + snapshots, err := ag.GetRemoteSnapshotList(ctx, remoteAddress) + if err != nil { + log.Fatalf("Failed to list remote snapshots: %v", err) + } + + fmt.Printf("Found %d remote snapshots:\n", len(snapshots)) + for _, s := range snapshots { + fmt.Printf(" - %s: %s (created at %s)\n", s.ID, s.Name, s.CreationTime.Format("2006-01-02 15:04:05")) + } + + // Download a specific snapshot + if len(snapshots) > 0 { + snapshotID := snapshots[0].ID + fmt.Printf("Downloading snapshot %s...\n", snapshotID) + + // Download the snapshot (pass empty string as localParentID if this is the first download) + if err := ag.GetRemoteSnapshot(ctx, remoteAddress, snapshotID, ""); err != nil { + log.Fatalf("Failed to download snapshot: %v", err) + } + + fmt.Printf("Successfully downloaded snapshot %s\n", snapshotID) + } +} +``` + +## Advanced Usage + +### Creating Incremental Snapshots + +You can create incremental snapshots by specifying a parent snapshot ID: + +```go +// Create a first snapshot +snapshotID1, err := ag.SaveSnapshot(ctx, "First Snapshot", "") +if err != nil { + log.Fatalf("Failed to create first snapshot: %v", err) +} + +// Make some changes to your files... + +// Create a second snapshot with the first one as parent +snapshotID2, err := ag.SaveSnapshot(ctx, "Second Snapshot", snapshotID1) +if err != nil { + log.Fatalf("Failed to create second snapshot: %v", err) +} +``` + +### Restoring a Snapshot + +To restore a snapshot: + +```go +if err := ag.RestoreSnapshot(ctx, snapshotID); err != nil { + log.Fatalf("Failed to restore snapshot: %v", err) +} +``` + +### Getting Snapshot Details + +To get detailed information about a snapshot: + +```go +snapshot, err := ag.GetSnapshotDetails(ctx, snapshotID) +if err != nil { + log.Fatalf("Failed to get snapshot details: %v", err) +} + +fmt.Printf("Snapshot: %s\n", snapshot.Name) +fmt.Printf("Created: %s\n", snapshot.CreationTime.Format("2006-01-02 15:04:05")) +fmt.Printf("Files: %d\n", len(snapshot.Files)) +``` + +### Deleting a Snapshot + +To delete a snapshot: + +```go +if err := ag.DeleteSnapshot(ctx, snapshotID); err != nil { + log.Fatalf("Failed to delete snapshot: %v", err) +} +``` + +## API Reference + +### Agate + +The main entry point for the library. + +- `New(options AgateOptions) (*Agate, error)` - Create a new Agate instance +- `SaveSnapshot(ctx context.Context, name string, parentID string) (string, error)` - Create a new snapshot +- `RestoreSnapshot(ctx context.Context, snapshotID string) error` - Restore a snapshot +- `ListSnapshots(ctx context.Context) ([]store.SnapshotInfo, error)` - List all snapshots +- `GetSnapshotDetails(ctx context.Context, snapshotID string) (*store.Snapshot, error)` - Get details of a snapshot +- `DeleteSnapshot(ctx context.Context, snapshotID string) error` - Delete a snapshot +- `StartServer(ctx context.Context, address string) error` - Start a gRPC server to share snapshots +- `ConnectRemote(address string) (*grpc.SnapshotClient, error)` - Connect to a remote server +- `GetRemoteSnapshotList(ctx context.Context, address string) ([]store.SnapshotInfo, error)` - List snapshots from a remote server +- `GetRemoteSnapshot(ctx context.Context, address string, snapshotID string, localParentID string) error` - Download a snapshot from a remote server + +### AgateOptions + +Configuration options for the Agate library. + +- `WorkDir string` - Directory where snapshots will be stored and managed +- `OpenFunc func(dir string) error` - Called after a snapshot is restored +- `CloseFunc func() error` - Called before a snapshot is created or restored +- `MetadataStore store.MetadataStore` - Implementation of the metadata store +- `BlobStore store.BlobStore` - Implementation of the blob store + +## License + +[Add your license information here] \ No newline at end of file diff --git a/manager.go b/manager.go index 5688f0c..b1a3c0b 100644 --- a/manager.go +++ b/manager.go @@ -59,12 +59,20 @@ func (data *SnapshotManagerData) CreateSnapshot(ctx context.Context, sourceDir s // Generate a unique ID for the snapshot snapshotID := uuid.New().String() - // Create a temporary file for the archive - tempFile, err := os.CreateTemp("", "agate-snapshot-*.zip") - if err != nil { - return nil, fmt.Errorf("failed to create temporary file: %w", err) + // Clean the active directory to avoid conflicts + if err := data.blobStore.CleanActiveDir(ctx); err != nil { + return nil, fmt.Errorf("failed to clean active directory: %w", err) + } + + // Get the active directory for operations + activeDir := data.blobStore.GetActiveDir() + + // Create a temporary file for the archive in the active directory + tempFilePath := filepath.Join(activeDir, "temp-"+snapshotID+".zip") + tempFile, err := os.Create(tempFilePath) + if err != nil { + return nil, fmt.Errorf("failed to create temporary file in active directory: %w", err) } - tempFilePath := tempFile.Name() tempFile.Close() // Close it as CreateArchive will reopen it defer os.Remove(tempFilePath) // Clean up temp file after we're done @@ -253,8 +261,15 @@ func (data *SnapshotManagerData) ExtractSnapshot(ctx context.Context, snapshotID if snapshotID == "" { return errors.New("snapshot ID cannot be empty") } + + // If no specific path is provided, use the active directory if path == "" { - return errors.New("target path cannot be empty") + // Clean the active directory to avoid conflicts + if err := data.blobStore.CleanActiveDir(ctx); err != nil { + return fmt.Errorf("failed to clean active directory: %w", err) + } + + path = filepath.Join(data.blobStore.GetActiveDir(), snapshotID) } // First check if the snapshot exists diff --git a/store/filesystem/filesystem.go b/store/filesystem/filesystem.go index ae805f4..2cc1cb9 100644 --- a/store/filesystem/filesystem.go +++ b/store/filesystem/filesystem.go @@ -14,16 +14,27 @@ const blobExtension = ".zip" // fileSystemStore реализует интерфейс store.BlobStore с использованием локальной файловой системы. type fileSystemStore struct { - baseDir string // Директория для хранения блобов (архивов) + baseDir string // Директория для хранения блобов (архивов) + activeDir string // Директория для активных операций (создание и восстановление) } // NewFileSystemStore создает новое хранилище блобов в указанной директории. func NewFileSystemStore(baseDir string) (store.BlobStore, error) { - // Убедимся, что директория существует + // Убедимся, что базовая директория существует if err := os.MkdirAll(baseDir, 0755); err != nil { return nil, fmt.Errorf("failed to create base directory %s for filesystem blob store: %w", baseDir, err) } - return &fileSystemStore{baseDir: baseDir}, nil + + // Создаем директорию для активных операций внутри базовой директории + activeDir := filepath.Join(baseDir, "active") + if err := os.MkdirAll(activeDir, 0755); err != nil { + return nil, fmt.Errorf("failed to create active directory %s for filesystem blob store: %w", activeDir, err) + } + + return &fileSystemStore{ + baseDir: baseDir, + activeDir: activeDir, + }, nil } // getBlobPath формирует полный путь к файлу блоба. @@ -106,3 +117,27 @@ func (fs *fileSystemStore) GetBlobPath(ctx context.Context, snapshotID string) ( // Файл существует, возвращаем путь return blobPath, nil } + +// GetActiveDir возвращает путь к директории для активных операций. +func (fs *fileSystemStore) GetActiveDir() string { + return fs.activeDir +} + +// CleanActiveDir очищает директорию для активных операций. +// Это полезно перед началом новых операций, чтобы избежать конфликтов. +func (fs *fileSystemStore) CleanActiveDir(ctx context.Context) error { + // Удаляем все файлы в активной директории, но сохраняем саму директорию + entries, err := os.ReadDir(fs.activeDir) + if err != nil { + return fmt.Errorf("failed to read active directory: %w", err) + } + + for _, entry := range entries { + path := filepath.Join(fs.activeDir, entry.Name()) + if err := os.RemoveAll(path); err != nil { + return fmt.Errorf("failed to remove %s from active directory: %w", path, err) + } + } + + return nil +} diff --git a/store/store.go b/store/store.go index c407cd8..ebda68d 100644 --- a/store/store.go +++ b/store/store.go @@ -71,4 +71,11 @@ type BlobStore interface { // Это может быть полезно для функций пакета archive, которые работают с путями. // Возвращает agate.ErrNotFound, если блоб не найден. GetBlobPath(ctx context.Context, snapshotID string) (string, error) + + // GetActiveDir возвращает путь к директории для активных операций (создание и восстановление). + GetActiveDir() string + + // CleanActiveDir очищает директорию для активных операций. + // Это полезно перед началом новых операций, чтобы избежать конфликтов. + CleanActiveDir(ctx context.Context) error }