david/apocapoc-api
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add automated backup system with SQLite VACUUM

Browse Source 
- Implement backup package with SQLite VACUUM INTO for safe backups
- Add scheduler with configurable interval (default: 24h)
- Add retention policy with automatic cleanup (default: 7 days)
- Add optional gzip compression (~10x size reduction)
- Add comprehensive tests for backup creation and cleanup
- Integrate backup scheduler in main.go with graceful shutdown
- Add backup configuration variables (BACKUP_ENABLED, BACKUP_INTERVAL, BACKUP_RETENTION_DAYS, BACKUP_PATH, BACKUP_COMPRESS)
- Backups run automatically in background goroutine
- Coverage: backup package 52.7%
This commit is contained in:
David Folch Agulles
2025-12-05 00:17:37 +01:00
parent 88b5d11113
commit e9e7e9dbac
7 changed files with 494 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/infrastructure/backup/backup.go
+95
View File
@@ -0,0 +1,95 @@
package backup
import (
"compress/gzip"
"database/sql"
"fmt"
"io"
"os"
"path/filepath"
"time"
"apocapoc-api/internal/infrastructure/logger"
)
type Config struct {
Enabled bool
Interval time.Duration
RetentionDays int
Path string
Compress bool
DatabasePath string
}
func CreateBackup(db *sql.DB, config Config) error {
if !config.Enabled {
return nil
}
if err := os.MkdirAll(config.Path, 0755); err != nil {
return fmt.Errorf("failed to create backup directory: %w", err)
}
timestamp := time.Now().Format("20060102_150405")
filename := fmt.Sprintf("apocapoc_%s.db", timestamp)
backupPath := filepath.Join(config.Path, filename)
logger.Info().
Str("backup_path", backupPath).
Msg("Starting database backup")
if err := backupDatabase(db, backupPath); err != nil {
logger.Error().
Err(err).
Str("backup_path", backupPath).
Msg("Backup failed")
return fmt.Errorf("backup failed: %w", err)
}
if config.Compress {
compressedPath := backupPath + ".gz"
if err := compressFile(backupPath, compressedPath); err != nil {
logger.Warn().
Err(err).
Str("backup_path", backupPath).
Msg("Compression failed, keeping uncompressed backup")
} else {
os.Remove(backupPath)
backupPath = compressedPath
}
}
logger.Info().
Str("backup_path", backupPath).
Msg("Backup completed successfully")
return nil
}
func backupDatabase(db *sql.DB, destPath string) error {
_, err := db.Exec(fmt.Sprintf("VACUUM INTO '%s'", destPath))
if err != nil {
return fmt.Errorf("vacuum into failed: %w", err)
}
return nil
}
func compressFile(srcPath, destPath string) error {
srcFile, err := os.Open(srcPath)
if err != nil {
return err
}
defer srcFile.Close()
destFile, err := os.Create(destPath)
if err != nil {
return err
}
defer destFile.Close()
gzipWriter := gzip.NewWriter(destFile)
defer gzipWriter.Close()
_, err = io.Copy(gzipWriter, srcFile)
return err
}
internal/infrastructure/backup/backup_test.go
+169
View File
@@ -0,0 +1,169 @@
package backup
import (
"database/sql"
"os"
"path/filepath"
"testing"
"time"
_ "modernc.org/sqlite"
)
func TestCreateBackup(t *testing.T) {
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "test.db")
db, err := sql.Open("sqlite", dbPath)
if err != nil {
t.Fatalf("Failed to create test database: %v", err)
}
defer db.Close()
_, err = db.Exec("CREATE TABLE test (id INTEGER PRIMARY KEY, name TEXT)")
if err != nil {
t.Fatalf("Failed to create test table: %v", err)
}
_, err = db.Exec("INSERT INTO test (name) VALUES ('test1'), ('test2')")
if err != nil {
t.Fatalf("Failed to insert test data: %v", err)
}
backupPath := filepath.Join(tempDir, "backups")
config := Config{
Enabled: true,
Interval: 24 * time.Hour,
RetentionDays: 7,
Path: backupPath,
Compress: false,
DatabasePath: dbPath,
}
err = CreateBackup(db, config)
if err != nil {
t.Fatalf("CreateBackup failed: %v", err)
}
files, err := os.ReadDir(backupPath)
if err != nil {
t.Fatalf("Failed to read backup directory: %v", err)
}
if len(files) != 1 {
t.Errorf("Expected 1 backup file, got %d", len(files))
}
if len(files) > 0 && filepath.Ext(files[0].Name()) != ".db" {
t.Errorf("Expected backup file to have .db extension, got %s", files[0].Name())
}
}
func TestCreateBackupWithCompression(t *testing.T) {
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "test.db")
db, err := sql.Open("sqlite", dbPath)
if err != nil {
t.Fatalf("Failed to create test database: %v", err)
}
defer db.Close()
_, err = db.Exec("CREATE TABLE test (id INTEGER PRIMARY KEY, name TEXT)")
if err != nil {
t.Fatalf("Failed to create test table: %v", err)
}
backupPath := filepath.Join(tempDir, "backups")
config := Config{
Enabled: true,
Interval: 24 * time.Hour,
RetentionDays: 7,
Path: backupPath,
Compress: true,
DatabasePath: dbPath,
}
err = CreateBackup(db, config)
if err != nil {
t.Fatalf("CreateBackup failed: %v", err)
}
files, err := os.ReadDir(backupPath)
if err != nil {
t.Fatalf("Failed to read backup directory: %v", err)
}
if len(files) != 1 {
t.Errorf("Expected 1 backup file, got %d", len(files))
}
if len(files) > 0 && filepath.Ext(files[0].Name()) != ".gz" {
t.Errorf("Expected backup file to have .gz extension, got %s", files[0].Name())
}
}
func TestCreateBackupDisabled(t *testing.T) {
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "test.db")
db, err := sql.Open("sqlite", dbPath)
if err != nil {
t.Fatalf("Failed to create test database: %v", err)
}
defer db.Close()
backupPath := filepath.Join(tempDir, "backups")
config := Config{
Enabled: false,
Interval: 24 * time.Hour,
RetentionDays: 7,
Path: backupPath,
Compress: false,
DatabasePath: dbPath,
}
err = CreateBackup(db, config)
if err != nil {
t.Fatalf("CreateBackup failed: %v", err)
}
_, err = os.Stat(backupPath)
if !os.IsNotExist(err) {
t.Error("Backup directory should not exist when backup is disabled")
}
}
func TestCleanOldBackups(t *testing.T) {
tempDir := t.TempDir()
backupPath := filepath.Join(tempDir, "backups")
os.MkdirAll(backupPath, 0755)
oldFile := filepath.Join(backupPath, "apocapoc_20200101_120000.db")
recentFile := filepath.Join(backupPath, "apocapoc_"+time.Now().Format("20060102_150405")+".db")
os.WriteFile(oldFile, []byte("old"), 0644)
os.WriteFile(recentFile, []byte("recent"), 0644)
oldTime := time.Now().AddDate(0, 0, -10)
os.Chtimes(oldFile, oldTime, oldTime)
config := Config{
Enabled: true,
RetentionDays: 7,
Path: backupPath,
}
err := CleanOldBackups(config)
if err != nil {
t.Fatalf("CleanOldBackups failed: %v", err)
}
if _, err := os.Stat(oldFile); !os.IsNotExist(err) {
t.Error("Old backup file should have been deleted")
}
if _, err := os.Stat(recentFile); err != nil {
t.Error("Recent backup file should still exist")
}
}
internal/infrastructure/backup/retention.go
+80
View File
@@ -0,0 +1,80 @@
package backup
import (
"os"
"path/filepath"
"strings"
"time"
"apocapoc-api/internal/infrastructure/logger"
)
func CleanOldBackups(config Config) error {
if !config.Enabled {
return nil
}
if config.RetentionDays <= 0 {
return nil
}
cutoffTime := time.Now().AddDate(0, 0, -config.RetentionDays)
files, err := os.ReadDir(config.Path)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
deletedCount := 0
for _, file := range files {
if file.IsDir() {
continue
}
if !strings.HasPrefix(file.Name(), "apocapoc_") {
continue
}
if !strings.HasSuffix(file.Name(), ".db") && !strings.HasSuffix(file.Name(), ".db.gz") {
continue
}
filePath := filepath.Join(config.Path, file.Name())
info, err := os.Stat(filePath)
if err != nil {
logger.Warn().
Err(err).
Str("file", filePath).
Msg("Failed to stat backup file")
continue
}
if info.ModTime().Before(cutoffTime) {
if err := os.Remove(filePath); err != nil {
logger.Warn().
Err(err).
Str("file", filePath).
Msg("Failed to delete old backup")
continue
}
logger.Info().
Str("file", file.Name()).
Time("mod_time", info.ModTime()).
Msg("Deleted old backup")
deletedCount++
}
}
if deletedCount > 0 {
logger.Info().
Int("deleted_count", deletedCount).
Int("retention_days", config.RetentionDays).
Msg("Backup cleanup completed")
}
return nil
}
internal/infrastructure/backup/scheduler.go
+77
View File
@@ -0,0 +1,77 @@
package backup
import (
"database/sql"
"time"
"apocapoc-api/internal/infrastructure/logger"
)
type Scheduler struct {
db *sql.DB
config Config
stopCh chan struct{}
}
func NewScheduler(db *sql.DB, config Config) *Scheduler {
return &Scheduler{
db: db,
config: config,
stopCh: make(chan struct{}),
}
}
func (s *Scheduler) Start() {
if !s.config.Enabled {
logger.Info().Msg("Backup scheduler is disabled")
return
}
logger.Info().
Dur("interval", s.config.Interval).
Int("retention_days", s.config.RetentionDays).
Str("path", s.config.Path).
Bool("compress", s.config.Compress).
Msg("Starting backup scheduler")
go s.run()
}
func (s *Scheduler) run() {
if err := CreateBackup(s.db, s.config); err != nil {
logger.Error().Err(err).Msg("Initial backup failed")
}
if err := CleanOldBackups(s.config); err != nil {
logger.Error().Err(err).Msg("Initial cleanup failed")
}
ticker := time.NewTicker(s.config.Interval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
logger.Debug().Msg("Running scheduled backup")
if err := CreateBackup(s.db, s.config); err != nil {
logger.Error().Err(err).Msg("Scheduled backup failed")
continue
}
if err := CleanOldBackups(s.config); err != nil {
logger.Error().Err(err).Msg("Backup cleanup failed")
}
case <-s.stopCh:
logger.Info().Msg("Backup scheduler stopped")
return
}
}
}
func (s *Scheduler) Stop() {
if s.config.Enabled {
close(s.stopCh)
}
}
internal/infrastructure/config/config.go
+44 -34
View File
@@ -8,46 +8,56 @@ import (
)
type Config struct {
DBPath string
Port string
AppURL string
JWTSecret string
JWTExpiry string
RefreshTokenExpiry string
DefaultTimezone string
SMTPHost string
SMTPPort string
SMTPUser string
SMTPPassword string
SMTPFrom string
SupportEmail string
SendWelcomeEmail string
RegistrationMode string
LogLevel string
Environment string
DBPath string
Port string
AppURL string
JWTSecret string
JWTExpiry string
RefreshTokenExpiry string
DefaultTimezone string
SMTPHost string
SMTPPort string
SMTPUser string
SMTPPassword string
SMTPFrom string
SupportEmail string
SendWelcomeEmail string
RegistrationMode string
LogLevel string
Environment string
BackupEnabled string
BackupInterval string
BackupRetentionDays string
BackupPath string
BackupCompress string
}
func Load() (*Config, error) {
godotenv.Load()
cfg := &Config{
DBPath: os.Getenv("DB_PATH"),
Port: getEnvOrDefault("PORT", "8080"),
AppURL: getEnvOrDefault("APP_URL", "http://localhost:8080"),
JWTSecret: os.Getenv("JWT_SECRET"),
JWTExpiry: os.Getenv("JWT_EXPIRY"),
RefreshTokenExpiry: os.Getenv("REFRESH_TOKEN_EXPIRY"),
DefaultTimezone: os.Getenv("DEFAULT_TIMEZONE"),
SMTPHost: os.Getenv("SMTP_HOST"),
SMTPPort: getEnvOrDefault("SMTP_PORT", "587"),
SMTPUser: os.Getenv("SMTP_USER"),
SMTPPassword: os.Getenv("SMTP_PASSWORD"),
SMTPFrom: os.Getenv("SMTP_FROM"),
SupportEmail: getEnvOrDefault("SUPPORT_EMAIL", "contact@apocapoc.app"),
SendWelcomeEmail: getEnvOrDefault("SEND_WELCOME_EMAIL", "false"),
RegistrationMode: getEnvOrDefault("REGISTRATION_MODE", "open"),
LogLevel: getEnvOrDefault("LOG_LEVEL", "info"),
Environment: getEnvOrDefault("ENVIRONMENT", "production"),
DBPath: os.Getenv("DB_PATH"),
Port: getEnvOrDefault("PORT", "8080"),
AppURL: getEnvOrDefault("APP_URL", "http://localhost:8080"),
JWTSecret: os.Getenv("JWT_SECRET"),
JWTExpiry: os.Getenv("JWT_EXPIRY"),
RefreshTokenExpiry: os.Getenv("REFRESH_TOKEN_EXPIRY"),
DefaultTimezone: os.Getenv("DEFAULT_TIMEZONE"),
SMTPHost: os.Getenv("SMTP_HOST"),
SMTPPort: getEnvOrDefault("SMTP_PORT", "587"),
SMTPUser: os.Getenv("SMTP_USER"),
SMTPPassword: os.Getenv("SMTP_PASSWORD"),
SMTPFrom: os.Getenv("SMTP_FROM"),
SupportEmail: getEnvOrDefault("SUPPORT_EMAIL", "contact@apocapoc.app"),
SendWelcomeEmail: getEnvOrDefault("SEND_WELCOME_EMAIL", "false"),
RegistrationMode: getEnvOrDefault("REGISTRATION_MODE", "open"),
LogLevel: getEnvOrDefault("LOG_LEVEL", "info"),
Environment: getEnvOrDefault("ENVIRONMENT", "production"),
BackupEnabled: getEnvOrDefault("BACKUP_ENABLED", "false"),
BackupInterval: getEnvOrDefault("BACKUP_INTERVAL", "24h"),
BackupRetentionDays: getEnvOrDefault("BACKUP_RETENTION_DAYS", "7"),
BackupPath: getEnvOrDefault("BACKUP_PATH", "./data/backups"),
BackupCompress: getEnvOrDefault("BACKUP_COMPRESS", "true"),
}
if cfg.DBPath == "" {