diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/InfrastructureBackupTask.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/InfrastructureBackupTask.java new file mode 100644 index 000000000000..25c086430fb0 --- /dev/null +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/InfrastructureBackupTask.java @@ -0,0 +1,299 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup; + +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.poll.BackgroundPollTask; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +/** + * Scheduled task that backs up CloudStack infrastructure to NAS storage: + * + * + * Database credentials are read from /etc/cloudstack/management/db.properties. + * Backups are stored under {nasBackupPath}/infra-backup/{timestamp}/ with + * automatic retention management. + */ +public class InfrastructureBackupTask extends ManagedContextRunnable implements BackgroundPollTask { + + private static final Logger LOG = LogManager.getLogger(InfrastructureBackupTask.class); + + private static final String DB_PROPERTIES_PATH = "/etc/cloudstack/management/db.properties"; + private static final String MANAGEMENT_CONFIG_PATH = "/etc/cloudstack/management"; + private static final String AGENT_CONFIG_PATH = "/etc/cloudstack/agent"; + private static final String SSL_CERT_PATH = "/etc/cloudstack/management/cert"; + + /** 24 hours in milliseconds */ + private static final long DAILY_INTERVAL_MS = 86400L * 1000L; + + private final NASBackupProvider provider; + + public InfrastructureBackupTask(NASBackupProvider provider) { + this.provider = provider; + } + + @Override + public Long getDelay() { + return DAILY_INTERVAL_MS; + } + + /** Indirection so tests can override without standing up the ConfigDepot. */ + protected boolean isEnabled() { + return Boolean.TRUE.equals(NASBackupProvider.NASInfraBackupEnabled.value()); + } + + protected String getBackupLocation() { + return NASBackupProvider.NASInfraBackupLocation.value(); + } + + protected int getRetentionCount() { + return NASBackupProvider.NASInfraBackupRetention.value(); + } + + protected boolean isDatabaseIncluded() { + return Boolean.TRUE.equals(NASBackupProvider.NASInfraBackupIncludeDatabase.value()); + } + + protected boolean isUsageDbIncluded() { + return Boolean.TRUE.equals(NASBackupProvider.NASInfraBackupUsageDb.value()); + } + + @Override + protected void runInContext() { + if (!isEnabled()) { + LOG.debug("Infrastructure backup is disabled (nas.infra.backup.enabled=false)"); + return; + } + + String nasBackupPath = getBackupLocation(); + if (nasBackupPath == null || nasBackupPath.isEmpty()) { + LOG.error("Infrastructure backup location not configured (nas.infra.backup.location is empty)"); + return; + } + + int retentionCount = getRetentionCount(); + boolean includeDatabase = isDatabaseIncluded(); + boolean includeUsageDb = isUsageDbIncluded(); + + String timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss")); + String backupDir = nasBackupPath + "/infra-backup/" + timestamp; + + LOG.info("Starting infrastructure backup to {} (database included: {})", backupDir, includeDatabase); + + try { + File dir = new File(backupDir); + if (!dir.mkdirs()) { + LOG.error("Failed to create backup directory: {}", backupDir); + return; + } + + // 1 & 2. Database backup — opt-in via nas.infra.backup.include.database. + // Production deployments typically run their own mysqldump cron jobs and disable this; + // it exists for small/edge deployments wanting unified DR on the same NAS as VM backups. + if (includeDatabase) { + Properties dbProps = loadDbProperties(); + if (dbProps == null) { + LOG.error("Database backup requested but failed to load properties from {} — skipping DB component", DB_PROPERTIES_PATH); + } else { + String dbHost = dbProps.getProperty("db.cloud.host", "localhost"); + String dbUser = dbProps.getProperty("db.cloud.username", "cloud"); + String dbPassword = dbProps.getProperty("db.cloud.password", ""); + + backupDatabase("cloud", backupDir, timestamp, dbHost, dbUser, dbPassword); + + if (includeUsageDb) { + String usageHost = dbProps.getProperty("db.usage.host", dbHost); + String usageUser = dbProps.getProperty("db.usage.username", dbUser); + String usagePassword = dbProps.getProperty("db.usage.password", dbPassword); + backupDatabase("cloud_usage", backupDir, timestamp, usageHost, usageUser, usagePassword); + } + } + } else { + LOG.debug("Database backup skipped (nas.infra.backup.include.database=false). " + + "Manage DB backups externally for production deployments."); + } + + // 3. Backup management server configs + backupDirectory(MANAGEMENT_CONFIG_PATH, backupDir, "management-config"); + + // 4. Backup agent configs (if present on this host) + File agentDir = new File(AGENT_CONFIG_PATH); + if (agentDir.exists()) { + backupDirectory(AGENT_CONFIG_PATH, backupDir, "agent-config"); + } + + // 5. Backup SSL certificates + File sslDir = new File(SSL_CERT_PATH); + if (sslDir.exists()) { + backupDirectory(SSL_CERT_PATH, backupDir, "ssl-certs"); + } + + // 6. Cleanup old backups based on retention policy + cleanupOldBackups(nasBackupPath, retentionCount); + + LOG.info("Infrastructure backup completed successfully: {}", backupDir); + + } catch (Exception e) { + LOG.error("Infrastructure backup failed: {}", e.getMessage(), e); + } + } + + protected Properties loadDbProperties() { + File propsFile = new File(DB_PROPERTIES_PATH); + if (!propsFile.exists()) { + LOG.warn("Database properties file not found: {}", DB_PROPERTIES_PATH); + return null; + } + + Properties props = new Properties(); + try (BufferedReader reader = new BufferedReader(new FileReader(propsFile))) { + props.load(reader); + return props; + } catch (IOException e) { + LOG.error("Failed to read database properties: {}", e.getMessage()); + return null; + } + } + + protected void backupDatabase(String dbName, String backupDir, String timestamp, + String dbHost, String dbUser, String dbPassword) { + String dumpFile = backupDir + "/" + dbName + "-" + timestamp + ".sql.gz"; + + // Use --single-transaction for InnoDB hot backup (no table locks, consistent snapshot) + String[] cmd = {"/bin/bash", "-c", + String.format("mysqldump --single-transaction --routines --triggers --events " + + "-h '%s' -u '%s' -p'%s' '%s' | gzip > '%s'", + dbHost, dbUser, dbPassword, dbName, dumpFile)}; + + try { + ProcessBuilder pb = new ProcessBuilder(cmd); + pb.redirectErrorStream(true); + Process process = pb.start(); + boolean completed = process.waitFor(300, TimeUnit.SECONDS); + + if (!completed) { + process.destroyForcibly(); + LOG.error("Database backup timed out for {}", dbName); + return; + } + + if (process.exitValue() != 0) { + LOG.error("Database backup failed for {} with exit code {}", dbName, process.exitValue()); + return; + } + + File dump = new File(dumpFile); + LOG.info("Database {} backed up: {} ({} bytes)", dbName, dumpFile, dump.length()); + + } catch (IOException | InterruptedException e) { + LOG.error("Failed to backup database {}: {}", dbName, e.getMessage()); + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + } + } + + protected void backupDirectory(String sourcePath, String backupDir, String archiveName) { + File source = new File(sourcePath); + if (!source.exists() || !source.isDirectory()) { + LOG.debug("Directory {} does not exist, skipping", sourcePath); + return; + } + + String tarFile = backupDir + "/" + archiveName + ".tar.gz"; + String[] cmd = {"/bin/bash", "-c", + String.format("tar czf '%s' -C '%s' .", tarFile, sourcePath)}; + + try { + ProcessBuilder pb = new ProcessBuilder(cmd); + pb.redirectErrorStream(true); + Process process = pb.start(); + boolean completed = process.waitFor(60, TimeUnit.SECONDS); + + if (completed && process.exitValue() == 0) { + LOG.info("Directory {} backed up to {}", sourcePath, tarFile); + } else { + if (!completed) { + process.destroyForcibly(); + } + LOG.warn("Directory backup failed for {} (exit code: {})", + sourcePath, completed ? process.exitValue() : "timeout"); + } + } catch (IOException | InterruptedException e) { + LOG.error("Failed to backup directory {}: {}", sourcePath, e.getMessage()); + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + } + } + + protected void cleanupOldBackups(String nasBackupPath, int retentionCount) { + File infraDir = new File(nasBackupPath + "/infra-backup"); + if (!infraDir.exists()) { + return; + } + + File[] backups = infraDir.listFiles(File::isDirectory); + if (backups == null || backups.length <= retentionCount) { + return; + } + + // Sort by name (timestamp-based), oldest first + Arrays.sort(backups, Comparator.comparing(File::getName)); + + int toDelete = backups.length - retentionCount; + for (int i = 0; i < toDelete; i++) { + LOG.info("Removing old infrastructure backup: {}", backups[i].getName()); + deleteDirectory(backups[i]); + } + } + + private void deleteDirectory(File dir) { + File[] files = dir.listFiles(); + if (files != null) { + for (File file : files) { + if (file.isDirectory()) { + deleteDirectory(file); + } else { + if (!file.delete()) { + LOG.warn("Failed to delete file: {}", file.getAbsolutePath()); + } + } + } + } + if (!dir.delete()) { + LOG.warn("Failed to delete directory: {}", dir.getAbsolutePath()); + } + } +} diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index f2ea8ac71c91..5e1d20324835 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -53,6 +53,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.poll.BackgroundPollManager; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; @@ -60,6 +61,7 @@ import org.apache.logging.log4j.LogManager; import javax.inject.Inject; +import javax.naming.ConfigurationException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; @@ -84,6 +86,59 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co true, BackupFrameworkEnabled.key()); + static final ConfigKey NASInfraBackupEnabled = new ConfigKey<>("Advanced", Boolean.class, + "nas.infra.backup.enabled", + "false", + "Enable automated infrastructure backup to NAS storage. When enabled, the management " + + "server will perform a daily backup of CloudStack configuration files and SSL " + + "certificates to the configured NAS location. The CloudStack database is NOT included " + + "by default — for production deployments, manage database backups externally (e.g. via " + + "a cron job running mysqldump). To opt in to bundling the database with this backup " + + "(useful only for small / edge / single-MS deployments without separate ops tooling), " + + "also set nas.infra.backup.include.database=true.", + true, + ConfigKey.Scope.Global, + BackupFrameworkEnabled.key()); + + static final ConfigKey NASInfraBackupIncludeDatabase = new ConfigKey<>("Advanced", Boolean.class, + "nas.infra.backup.include.database", + "false", + "Include the CloudStack database in the daily infrastructure backup. Defaults to false " + + "because production deployments typically manage DB backups via external tooling (e.g. " + + "cron + mysqldump, replication, dedicated backup appliance) and are better served " + + "doing so. Only set true when you want one-knob disaster recovery for a small/edge " + + "deployment and the same NAS that already holds your VM backups is an acceptable " + + "target. Has no effect unless nas.infra.backup.enabled is also true.", + true, + ConfigKey.Scope.Global, + BackupFrameworkEnabled.key()); + + static final ConfigKey NASInfraBackupLocation = new ConfigKey<>("Advanced", String.class, + "nas.infra.backup.location", + "", + "NAS mount path where infrastructure backups are stored (e.g. /mnt/nas-backup). " + + "Backups will be written to {location}/infra-backup/{timestamp}/.", + true, + ConfigKey.Scope.Global, + BackupFrameworkEnabled.key()); + + static final ConfigKey NASInfraBackupRetention = new ConfigKey<>("Advanced", Integer.class, + "nas.infra.backup.retention", + "7", + "Number of infrastructure backup sets to retain. Older backups are automatically removed.", + true, + ConfigKey.Scope.Global, + BackupFrameworkEnabled.key()); + + static final ConfigKey NASInfraBackupUsageDb = new ConfigKey<>("Advanced", Boolean.class, + "nas.infra.backup.include.usage.db", + "true", + "Also include the cloud_usage database when the CloudStack database is being backed " + + "up. Has no effect unless nas.infra.backup.include.database is also true.", + true, + ConfigKey.Scope.Global, + BackupFrameworkEnabled.key()); + @Inject private BackupDao backupDao; @@ -129,6 +184,16 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Inject private DiskOfferingDao diskOfferingDao; + @Inject + private BackgroundPollManager backgroundPollManager; + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + super.configure(name, params); + backgroundPollManager.submitTask(new InfrastructureBackupTask(this)); + return true; + } + private Long getClusterIdFromRootVolume(VirtualMachine vm) { VolumeVO rootVolume = volumeDao.getInstanceRootVolume(vm.getId()); StoragePoolVO rootDiskPool = primaryDataStoreDao.findById(rootVolume.getPoolId()); @@ -594,7 +659,12 @@ public Boolean crossZoneInstanceCreationEnabled(BackupOffering backupOffering) { @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[]{ - NASBackupRestoreMountTimeout + NASBackupRestoreMountTimeout, + NASInfraBackupEnabled, + NASInfraBackupIncludeDatabase, + NASInfraBackupLocation, + NASInfraBackupRetention, + NASInfraBackupUsageDb }; } diff --git a/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/InfrastructureBackupTaskTest.java b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/InfrastructureBackupTaskTest.java new file mode 100644 index 000000000000..7c8e3b02b2f3 --- /dev/null +++ b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/InfrastructureBackupTaskTest.java @@ -0,0 +1,236 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class InfrastructureBackupTaskTest { + + private Path tmpRoot; + + @Before + public void setUp() throws IOException { + tmpRoot = Files.createTempDirectory("cs-infra-backup-test-"); + } + + @After + public void tearDown() { + if (tmpRoot != null) { + deleteRecursively(tmpRoot.toFile()); + } + } + + /** Captures backupDatabase/backupDirectory invocations instead of shelling out. */ + private static class RecordingTask extends InfrastructureBackupTask { + boolean enabled = true; + String location; + int retention = 7; + boolean databaseIncluded = false; + boolean usageDbIncluded = true; + Properties dbProps; + final List databaseBackupNames = new ArrayList<>(); + final List directoryBackupNames = new ArrayList<>(); + final AtomicInteger retentionCalls = new AtomicInteger(0); + + RecordingTask() { + super(null); + } + + @Override + protected boolean isEnabled() { return enabled; } + @Override + protected String getBackupLocation() { return location; } + @Override + protected int getRetentionCount() { return retention; } + @Override + protected boolean isDatabaseIncluded() { return databaseIncluded; } + @Override + protected boolean isUsageDbIncluded() { return usageDbIncluded; } + + @Override + protected Properties loadDbProperties() { + return dbProps; + } + + @Override + protected void backupDatabase(String dbName, String backupDir, String timestamp, + String dbHost, String dbUser, String dbPassword) { + databaseBackupNames.add(dbName); + } + + @Override + protected void backupDirectory(String sourcePath, String backupDir, String archiveName) { + directoryBackupNames.add(archiveName); + } + + @Override + protected void cleanupOldBackups(String nasBackupPath, int retentionCount) { + retentionCalls.incrementAndGet(); + } + } + + private static Properties stubDbProps() { + Properties props = new Properties(); + props.setProperty("db.cloud.host", "localhost"); + props.setProperty("db.cloud.username", "cloud"); + props.setProperty("db.cloud.password", "secret"); + return props; + } + + @Test + public void doesNothingWhenMasterSwitchDisabled() { + RecordingTask task = new RecordingTask(); + task.enabled = false; + task.location = tmpRoot.toString(); + + task.runInContext(); + + Assert.assertTrue("no DB backups when feature disabled", task.databaseBackupNames.isEmpty()); + Assert.assertTrue("no dir backups when feature disabled", task.directoryBackupNames.isEmpty()); + Assert.assertEquals("no retention when feature disabled", 0, task.retentionCalls.get()); + } + + @Test + public void doesNothingWhenLocationEmpty() { + RecordingTask task = new RecordingTask(); + task.enabled = true; + task.location = ""; + + task.runInContext(); + + Assert.assertTrue(task.databaseBackupNames.isEmpty()); + Assert.assertTrue(task.directoryBackupNames.isEmpty()); + Assert.assertEquals(0, task.retentionCalls.get()); + } + + @Test + public void skipsDatabaseByDefault() { + RecordingTask task = new RecordingTask(); + task.enabled = true; + task.location = tmpRoot.toString(); + task.databaseIncluded = false; // default + task.dbProps = stubDbProps(); + + task.runInContext(); + + Assert.assertTrue("DB component is opt-in: no DB backups by default", + task.databaseBackupNames.isEmpty()); + // Configs+certs still attempted (they're skipped silently if dirs absent) + Assert.assertEquals("retention still runs", 1, task.retentionCalls.get()); + } + + @Test + public void backsUpCloudDbWhenIncludeDatabaseTrue() { + RecordingTask task = new RecordingTask(); + task.enabled = true; + task.location = tmpRoot.toString(); + task.databaseIncluded = true; + task.usageDbIncluded = false; // suppress cloud_usage + task.dbProps = stubDbProps(); + + task.runInContext(); + + Assert.assertEquals("only cloud DB backed up", 1, task.databaseBackupNames.size()); + Assert.assertEquals("cloud", task.databaseBackupNames.get(0)); + } + + @Test + public void backsUpBothDbsWhenUsageEnabled() { + RecordingTask task = new RecordingTask(); + task.enabled = true; + task.location = tmpRoot.toString(); + task.databaseIncluded = true; + task.usageDbIncluded = true; + task.dbProps = stubDbProps(); + + task.runInContext(); + + Assert.assertEquals("both DBs backed up", 2, task.databaseBackupNames.size()); + Assert.assertEquals("cloud", task.databaseBackupNames.get(0)); + Assert.assertEquals("cloud_usage", task.databaseBackupNames.get(1)); + } + + @Test + public void usageDbFlagIgnoredWhenDatabaseExcluded() { + RecordingTask task = new RecordingTask(); + task.enabled = true; + task.location = tmpRoot.toString(); + task.databaseIncluded = false; // master DB gate off + task.usageDbIncluded = true; // should be ignored + task.dbProps = stubDbProps(); + + task.runInContext(); + + Assert.assertTrue("usage DB requires include.database=true", + task.databaseBackupNames.isEmpty()); + } + + @Test + public void skipsDbBackupWhenPropertiesUnreadable() { + RecordingTask task = new RecordingTask(); + task.enabled = true; + task.location = tmpRoot.toString(); + task.databaseIncluded = true; + task.dbProps = null; // simulate missing/unreadable db.properties + + task.runInContext(); + + Assert.assertTrue("no DB backups attempted when props can't be loaded", + task.databaseBackupNames.isEmpty()); + Assert.assertEquals("retention still runs (configs+certs path unaffected)", + 1, task.retentionCalls.get()); + } + + @Test + public void retentionRunsOnEverySuccessfulPass() { + RecordingTask task = new RecordingTask(); + task.enabled = true; + task.location = tmpRoot.toString(); + task.retention = 3; + task.databaseIncluded = false; + + task.runInContext(); + + Assert.assertEquals(1, task.retentionCalls.get()); + } + + @Test + public void dailyIntervalIs24Hours() { + InfrastructureBackupTask task = new RecordingTask(); + Assert.assertEquals(Long.valueOf(86_400_000L), task.getDelay()); + } + + private void deleteRecursively(File f) { + File[] kids = f.listFiles(); + if (kids != null) { + for (File k : kids) deleteRecursively(k); + } + f.delete(); + } +}