Compare commits
1 Commits
v0.2.0-alp
...
v0.2.1-alp
Author | SHA1 | Date | |
---|---|---|---|
efa2bec38b
|
41
README.md
41
README.md
@ -223,6 +223,41 @@ func main() {
|
|||||||
|
|
||||||
## Advanced Usage
|
## Advanced Usage
|
||||||
|
|
||||||
|
### Registering a Local Snapshot
|
||||||
|
|
||||||
|
You can register a local snapshot from an existing archive file with a specified UUID:
|
||||||
|
|
||||||
|
```go
|
||||||
|
// Register a local snapshot from an archive file
|
||||||
|
archivePath := "/path/to/your/archive.zip"
|
||||||
|
snapshotID := "custom-uuid-for-snapshot"
|
||||||
|
snapshotName := "My Local Snapshot"
|
||||||
|
|
||||||
|
if err := ag.RegisterLocalSnapshot(ctx, archivePath, snapshotID, snapshotName); err != nil {
|
||||||
|
log.Fatalf("Failed to register local snapshot: %v", err)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Downloading Only Snapshot Metadata
|
||||||
|
|
||||||
|
You can download only the metadata of a snapshot from a remote server without downloading the actual files:
|
||||||
|
|
||||||
|
```go
|
||||||
|
// Download only the metadata of a snapshot from a remote server
|
||||||
|
remoteAddress := "remote-server:50051"
|
||||||
|
snapshotID := "snapshot-id-to-download"
|
||||||
|
|
||||||
|
if err := ag.GetRemoteSnapshotMetadata(ctx, remoteAddress, snapshotID); err != nil {
|
||||||
|
log.Fatalf("Failed to download snapshot metadata: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// If you have a local blob but missing metadata, you can restore the metadata
|
||||||
|
// by passing an empty address
|
||||||
|
if err := ag.GetRemoteSnapshotMetadata(ctx, "", snapshotID); err != nil {
|
||||||
|
log.Fatalf("Failed to restore snapshot metadata: %v", err)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
### Creating Incremental Snapshots
|
### Creating Incremental Snapshots
|
||||||
|
|
||||||
You can create incremental snapshots by specifying a parent snapshot ID:
|
You can create incremental snapshots by specifying a parent snapshot ID:
|
||||||
@ -294,6 +329,8 @@ The main entry point for the library.
|
|||||||
- `ConnectRemote(address string) (*grpc.SnapshotClient, error)` - Connect to a remote server
|
- `ConnectRemote(address string) (*grpc.SnapshotClient, error)` - Connect to a remote server
|
||||||
- `GetRemoteSnapshotList(ctx context.Context, address string) ([]store.SnapshotInfo, error)` - List snapshots from a remote server
|
- `GetRemoteSnapshotList(ctx context.Context, address string) ([]store.SnapshotInfo, error)` - List snapshots from a remote server
|
||||||
- `GetRemoteSnapshot(ctx context.Context, address string, snapshotID string, localParentID string) error` - Download a snapshot from a remote server
|
- `GetRemoteSnapshot(ctx context.Context, address string, snapshotID string, localParentID string) error` - Download a snapshot from a remote server
|
||||||
|
- `RegisterLocalSnapshot(ctx context.Context, archivePath string, snapshotID string, name string) error` - Register a local snapshot from an archive path with a specified UUID
|
||||||
|
- `GetRemoteSnapshotMetadata(ctx context.Context, address string, snapshotID string) error` - Download only the metadata of a snapshot from a remote server
|
||||||
|
|
||||||
### AgateOptions
|
### AgateOptions
|
||||||
|
|
||||||
@ -304,7 +341,3 @@ Configuration options for the Agate library.
|
|||||||
- `CloseFunc func() error` - Called before a snapshot is created or restored
|
- `CloseFunc func() error` - Called before a snapshot is created or restored
|
||||||
- `MetadataStore store.MetadataStore` - Implementation of the metadata store
|
- `MetadataStore store.MetadataStore` - Implementation of the metadata store
|
||||||
- `BlobStore store.BlobStore` - Implementation of the blob store
|
- `BlobStore store.BlobStore` - Implementation of the blob store
|
||||||
|
|
||||||
## License
|
|
||||||
|
|
||||||
[Add your license information here]
|
|
237
api.go
237
api.go
@ -6,12 +6,14 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"gitea.unprism.ru/KRBL/Agate/archive"
|
"gitea.unprism.ru/KRBL/Agate/archive"
|
||||||
"gitea.unprism.ru/KRBL/Agate/grpc"
|
"gitea.unprism.ru/KRBL/Agate/grpc"
|
||||||
|
"gitea.unprism.ru/KRBL/Agate/hash"
|
||||||
"gitea.unprism.ru/KRBL/Agate/interfaces"
|
"gitea.unprism.ru/KRBL/Agate/interfaces"
|
||||||
"io"
|
"io"
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sync"
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
"gitea.unprism.ru/KRBL/Agate/store"
|
"gitea.unprism.ru/KRBL/Agate/store"
|
||||||
"gitea.unprism.ru/KRBL/Agate/stores"
|
"gitea.unprism.ru/KRBL/Agate/stores"
|
||||||
@ -381,6 +383,241 @@ func (a *Agate) GetRemoteSnapshotList(ctx context.Context, address string) ([]st
|
|||||||
return client.ListSnapshots(ctx)
|
return client.ListSnapshots(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RegisterLocalSnapshot registers a local snapshot from an archive path with a specified UUID.
|
||||||
|
// The archive must be a valid ZIP file containing the snapshot files.
|
||||||
|
// If the UUID already exists, an error will be returned.
|
||||||
|
func (a *Agate) RegisterLocalSnapshot(ctx context.Context, archivePath string, snapshotID string, name string) error {
|
||||||
|
a.mutex.Lock()
|
||||||
|
defer a.mutex.Unlock()
|
||||||
|
|
||||||
|
a.options.Logger.Printf("Registering local snapshot from archive %s with ID %s", archivePath, snapshotID)
|
||||||
|
|
||||||
|
// Check if the archive file exists
|
||||||
|
if _, err := os.Stat(archivePath); os.IsNotExist(err) {
|
||||||
|
return fmt.Errorf("archive file does not exist: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if a snapshot with this ID already exists
|
||||||
|
_, err := a.options.MetadataStore.GetSnapshotMetadata(ctx, snapshotID)
|
||||||
|
if err == nil {
|
||||||
|
return fmt.Errorf("snapshot with ID %s already exists", snapshotID)
|
||||||
|
} else if !errors.Is(err, store.ErrNotFound) {
|
||||||
|
return fmt.Errorf("failed to check if snapshot exists: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a temporary directory for extracting the archive
|
||||||
|
tempDir := filepath.Join(a.options.WorkDir, "temp_extract", snapshotID)
|
||||||
|
if err := os.MkdirAll(tempDir, 0755); err != nil {
|
||||||
|
return fmt.Errorf("failed to create temporary directory: %w", err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(tempDir) // Clean up when done
|
||||||
|
|
||||||
|
// Extract the archive to the temporary directory to analyze its contents
|
||||||
|
if err := extractArchive(archivePath, tempDir); err != nil {
|
||||||
|
return fmt.Errorf("failed to extract archive: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the list of files in the archive
|
||||||
|
var files []store.FileInfo
|
||||||
|
err = filepath.Walk(tempDir, func(path string, info os.FileInfo, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip the root directory
|
||||||
|
if path == tempDir {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the relative path
|
||||||
|
relPath, err := filepath.Rel(tempDir, path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to get relative path: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate SHA256 for files (not directories)
|
||||||
|
var sha256 string
|
||||||
|
if !info.IsDir() {
|
||||||
|
// Calculate the hash directly from the file path
|
||||||
|
sha256, err = hash.CalculateFileHash(path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to calculate file hash: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add file info to the list
|
||||||
|
files = append(files, store.FileInfo{
|
||||||
|
Path: relPath,
|
||||||
|
Size: info.Size(),
|
||||||
|
IsDir: info.IsDir(),
|
||||||
|
SHA256: sha256,
|
||||||
|
})
|
||||||
|
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to analyze archive contents: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Copy the archive to the blob store
|
||||||
|
archiveFile, err := os.Open(archivePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open archive file: %w", err)
|
||||||
|
}
|
||||||
|
defer archiveFile.Close()
|
||||||
|
|
||||||
|
// Store the blob with the specified snapshot ID
|
||||||
|
_, err = a.options.BlobStore.StoreBlob(ctx, snapshotID, archiveFile)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to store blob: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create and save the snapshot metadata
|
||||||
|
snapshot := store.Snapshot{
|
||||||
|
ID: snapshotID,
|
||||||
|
Name: name,
|
||||||
|
ParentID: "",
|
||||||
|
CreationTime: time.Now(),
|
||||||
|
Files: files,
|
||||||
|
}
|
||||||
|
|
||||||
|
err = a.options.MetadataStore.SaveSnapshotMetadata(ctx, snapshot)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to save snapshot metadata: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
a.options.Logger.Printf("Successfully registered local snapshot with ID %s", snapshotID)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetRemoteSnapshotMetadata downloads only the metadata of a snapshot from a remote server.
|
||||||
|
// If address is empty, it will try to restore the metadata from the local blob.
|
||||||
|
func (a *Agate) GetRemoteSnapshotMetadata(ctx context.Context, address string, snapshotID string) error {
|
||||||
|
a.mutex.Lock()
|
||||||
|
defer a.mutex.Unlock()
|
||||||
|
|
||||||
|
// Check if the snapshot already exists locally
|
||||||
|
_, err := a.options.MetadataStore.GetSnapshotMetadata(ctx, snapshotID)
|
||||||
|
if err == nil {
|
||||||
|
a.options.Logger.Printf("Snapshot %s already exists locally", snapshotID)
|
||||||
|
return nil
|
||||||
|
} else if !errors.Is(err, store.ErrNotFound) {
|
||||||
|
return fmt.Errorf("failed to check if snapshot exists: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// If address is provided, download metadata from remote server
|
||||||
|
if address != "" {
|
||||||
|
a.options.Logger.Printf("Downloading metadata for snapshot %s from %s", snapshotID, address)
|
||||||
|
|
||||||
|
client, err := a.ConnectRemote(address)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to connect to remote server: %w", err)
|
||||||
|
}
|
||||||
|
defer client.Close()
|
||||||
|
|
||||||
|
// Get the remote snapshot details
|
||||||
|
remoteSnapshot, err := client.FetchSnapshotDetails(ctx, snapshotID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to get snapshot details: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save the remote snapshot metadata
|
||||||
|
err = a.options.MetadataStore.SaveSnapshotMetadata(ctx, *remoteSnapshot)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to save snapshot metadata: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
a.options.Logger.Printf("Successfully downloaded metadata for snapshot %s", snapshotID)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// If no address is provided, try to restore metadata from the local blob
|
||||||
|
a.options.Logger.Printf("Trying to restore metadata for snapshot %s from local blob", snapshotID)
|
||||||
|
|
||||||
|
// Check if the blob exists
|
||||||
|
blobPath, err := a.options.BlobStore.GetBlobPath(ctx, snapshotID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to get blob path: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := os.Stat(blobPath); os.IsNotExist(err) {
|
||||||
|
return fmt.Errorf("blob for snapshot %s does not exist", snapshotID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a temporary directory for extracting the archive
|
||||||
|
tempDir := filepath.Join(a.options.WorkDir, "temp_extract", snapshotID)
|
||||||
|
if err := os.MkdirAll(tempDir, 0755); err != nil {
|
||||||
|
return fmt.Errorf("failed to create temporary directory: %w", err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(tempDir) // Clean up when done
|
||||||
|
|
||||||
|
// Extract the archive to the temporary directory to analyze its contents
|
||||||
|
if err := extractArchive(blobPath, tempDir); err != nil {
|
||||||
|
return fmt.Errorf("failed to extract archive: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the list of files in the archive
|
||||||
|
var files []store.FileInfo
|
||||||
|
err = filepath.Walk(tempDir, func(path string, info os.FileInfo, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip the root directory
|
||||||
|
if path == tempDir {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the relative path
|
||||||
|
relPath, err := filepath.Rel(tempDir, path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to get relative path: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate SHA256 for files (not directories)
|
||||||
|
var sha256 string
|
||||||
|
if !info.IsDir() {
|
||||||
|
// Calculate the hash directly from the file path
|
||||||
|
sha256, err = hash.CalculateFileHash(path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to calculate file hash: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add file info to the list
|
||||||
|
files = append(files, store.FileInfo{
|
||||||
|
Path: relPath,
|
||||||
|
Size: info.Size(),
|
||||||
|
IsDir: info.IsDir(),
|
||||||
|
SHA256: sha256,
|
||||||
|
})
|
||||||
|
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to analyze archive contents: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create and save the snapshot metadata
|
||||||
|
snapshot := store.Snapshot{
|
||||||
|
ID: snapshotID,
|
||||||
|
Name: snapshotID, // Use the ID as the name since we don't have a better name
|
||||||
|
ParentID: "",
|
||||||
|
CreationTime: time.Now(),
|
||||||
|
Files: files,
|
||||||
|
}
|
||||||
|
|
||||||
|
err = a.options.MetadataStore.SaveSnapshotMetadata(ctx, snapshot)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to save snapshot metadata: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
a.options.Logger.Printf("Successfully restored metadata for snapshot %s from local blob", snapshotID)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// GetRemoteSnapshot downloads a snapshot from a remote server.
|
// GetRemoteSnapshot downloads a snapshot from a remote server.
|
||||||
// If localParentID is provided, it will be used to optimize the download by skipping files that already exist locally.
|
// If localParentID is provided, it will be used to optimize the download by skipping files that already exist locally.
|
||||||
func (a *Agate) GetRemoteSnapshot(ctx context.Context, address string, snapshotID string, localParentID string) error {
|
func (a *Agate) GetRemoteSnapshot(ctx context.Context, address string, snapshotID string, localParentID string) error {
|
||||||
|
Reference in New Issue
Block a user