Skip to content

Instantly share code, notes, and snippets.

@ericmerrill

ericmerrill/backup_courses.php

Created April 16, 2021 15:37
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save ericmerrill/0ea565d21481e7c1eab2b80dbcfa13c3 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save ericmerrill/0ea565d21481e7c1eab2b80dbcfa13c3 to your computer and use it in GitHub Desktop.
Download ZIP
A file for creating bulk backups of Moodle courses
Raw
backup_courses.php
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
/**
* This script allows to do bulk backups.
*
* @copyright 2021 Oakland University
* @author Eric Merrill ([email protected])
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
define('CLI_SCRIPT', true);
// Update this line based on where you place this file.
require(dirname(dirname(dirname(__FILE__))).'/config.php');
require_once($CFG->libdir.'/clilib.php');
list($options, $unrecognized) = cli_get_params(array('help' => false,
'course' => false,
'cat' => false,
'check-only' => false,
'check-checksum' => false,
'start' => 0,
'limit' => 0,
'no-users' => false,
'force' => false,
'directory' => false),
array('h' => 'help',
'C' => 'course',
'c' => 'cat',
's' => 'start',
'l' => 'limit',
'n' => 'no-users',
'f' => 'force'));
if ($unrecognized) {
$unrecognized = implode("\n ", $unrecognized);
cli_error(get_string('cliunknowoption', 'admin', $unrecognized));
}
if (!$options['course'] && !$options['cat']) {
echo "Need to specify a category or course.\n\n";
$options['help'] = true;
}
if ($options['help']) {
$help =
"Execute archive backups.
You can run this file multiple times in parallel - it uses Moodle's locking system to make sure any one course is
only being worked on by one instance at a time.
Once a full pass is complete, you will want to run it again. Typically we will run it multiple times, until it
runs cleanly throught with processing any courses or files.
On the first pass throught, it will make a backup file in the specified destination directory. Then on a second
pass, it will read in the existing mbz, and confirm it contains key files that indicate it is good. If it is,
then it will create a sha1 file of the same name, that contains the sha1 hash of the mbz. On future runs, then
presence of the sha1 causes the script to continue on. The --check-checksum will make it confirm the sha1 sums
match if desired.
Backup settings can be reviewed/changed at the bottom of this script file, there are two sets, one with user
data, and one without. Force without user data using the --no-users flag.
Options:
-C, --course Course ID
-c, --cat Category ID
-s, --start Limit start
-l, --limit Limit count
-n, --no-users Create backups without user data
-f, --force Force backup, even if file already exists
--check-only Perform the existing file check only. Don't move or recreate.
--check-checksum When using a .sha1 checkfile to confirm backup has been validated, confirm the checksum value.
--directory Set the absolute directory to place backups in.
If not specified, the automated backup location is used.
-h, --help Print out this help
Example:
\$sudo -u www-data /usr/bin/php admin/cli/backup_courses.php
";
echo $help;
die;
}
if (CLI_MAINTENANCE) {
echo "CLI maintenance mode active, backup execution suspended.\n";
exit(1);
}
if (moodle_needs_upgrading()) {
echo "Moodle upgrade pending, backup execution suspended.\n";
exit(1);
}
require_once($CFG->libdir.'/adminlib.php');
require_once($CFG->libdir.'/gradelib.php');
require_once($CFG->dirroot.'/backup/util/includes/backup_includes.php');
require_once($CFG->dirroot.'/backup/util/helper/backup_cron_helper.class.php');
$DIRECTORY = null;
$starttime = time();
$thisweek = date('Y-m-d');
if ($options['course']) {
if (!is_numeric($options['course'])) {
echo "ERROR: Course ID needs to be a number.\n";
exit(1);
}
$courses = $DB->get_records('course', array('id' => $options['course']));
} else if ($options['cat']) {
if (!is_numeric($options['cat'])) {
echo "ERROR: Category ID needs to be a number.\n";
exit(1);
}
if (!is_numeric($options['start'])) {
echo "ERROR: Limit start needs to be a number.\n";
exit(1);
}
if (!is_numeric($options['limit'])) {
echo "ERROR: Limit count needs to be a number.\n";
exit(1);
}
$courses = $DB->get_records('course', array('category' => $options['cat']), 'id ASC', '*', $options['start'], $options['limit']);
}
if (!$courses) {
echo "ERROR: No courses found.\n";
exit(1);
}
$users = true;
if ($options['no-users']) {
$users = false;
}
setup_config();
$basepath = get_config('backup', 'backup_auto_destination');
$num = 1;
$count = count($courses);
$lf = \core\lock\lock_config::get_lock_factory('local_elis_course_backup');
if ($options['check-only']) {
echo "Performing a check of {$basepath}\n";
foreach ($courses as $course) {
$key = 'course:'.$course->id;
$lock = $lf->get_lock($key, 0);
echo "$num of $count, course ".$course->id.". ";
if ($lock === false) {
echo "Lock held elsewhere.";
} else {
$existing = find_existing_filename($course);
if ($existing !== false) {
echo "{$existing} found. ";
if (check_file_exists($existing)) {
echo "Check file already exists.";
} else {
$good = check_backup_good($existing);
if ($good) {
echo "File is good.";
create_check_file($existing);
} else {
echo "ERROR: Backup file not good.";
}
}
} else {
echo "ERROR: No backup found. ";
}
$lock->release();
}
echo "\n";
$num++;
}
exit();
}
echo "Saving to $basepath\n";
foreach ($courses as $course) {
echo "$num of $count, course ".$course->id.". ";
$key = 'course:'.$course->id;
$lock = $lf->get_lock($key, 0);
if ($lock === false) {
echo "Course lock in place, being backed up elsewhere.";
} else if (($filename = course_exists($course)) !== false) {
echo "Course already exists as ".$filename.".";
$lock->release();
} else {
echo "Backing up ";
if ($status = backup_cron_automated_helper::launch_automated_backup($course, $starttime, 2)) {
$time = time();
$suffix = '-nu';
if ($users) {
$suffix = '';
}
for ($i = 0; $i <= 1200; $i = $i + 60) {
$orgfile = $basepath.'/backup-moodle2-course-'.$course->id.'-'.
date('Ymd', $time-$i).'-'.date('Hi', $time-$i).$suffix.'.mbz';
if (file_exists($orgfile)) {
break;
}
}
// Final filename is specified here.
$basename = get_filename_base($course);
$name = $basename."-".$thisweek.".mbz";
$newfile = $basepath.'/'.$name;
if (rename($orgfile, $newfile)) {
echo $name.".";
if (!check_backup_good($name)) {
// This shouldn't happen. The file we just made looks like it's bad.
echo "ERROR: The file created seems like it is bad!";
$badfile = $basepath.'/bad-created-'.$name;
rename($newfile, $badfile);
}
} else {
echo "Error renaming.";
}
} else {
echo "Error creating backup.";
}
// Just in case there was some old check file. Remove it.
$checkfile = $basename."-".$thisweek.".sha1";
if (file_exists($basepath.'/'.$checkfile)) {
unlink($basepath.'/'.$checkfile);
}
$lock->release();
}
print "\n";
$num++;
}
function course_exists($course) {
global $basepath, $options;
if ($options['force']) {
return false;
}
$filename = find_existing_filename($course);
if (empty($filename)) {
return false;
}
if (check_file_exists($filename)) {
// The file exists and was already checked.
echo "Checkfile used. ";
return $filename;
} else if (check_backup_good($filename)) {
// Good file, we can use it.
// This is more than the first check, so we can mark it as checked.
echo "File is good. ";
create_check_file($filename);
return $filename;
} else {
// This means we suspect the existing file is bad. Move it, and we will try again.
echo "Moving existing file. ";
$orgfile = $basepath.'/'.$filename;
$newfile = $basepath.'/bad-'.$filename;
rename($orgfile, $newfile);
return false;
}
return false;
}
function find_existing_filename($course) {
global $basepath, $options, $DIRECTORY;
$name = get_filename_base($course);
if (is_null($DIRECTORY)) {
reload_directory();
}
$filename = search_directory_for_file($name);
return $filename;
}
function search_directory_for_file($name, $extension = 'mbz') {
$result = search_directory_for_file_inner($name, $extension);
if ($result !== false) {
return $result;
}
reload_directory();
return search_directory_for_file_inner($name, $extension);
}
function search_directory_for_file_inner($name, $extension = 'mbz') {
global $DIRECTORY;
foreach ($DIRECTORY as $file) {
$ext = $file->getExtension();
if (strcmp($ext, $extension) !== 0) {
continue;
}
$filename = $file->getFilename();
if (strncmp($filename, $name, strlen($name)) === 0) {
return $filename;
}
}
return false;
}
function reload_directory() {
global $basepath, $DIRECTORY;
$DIRECTORY = new DirectoryIterator($basepath);
if (empty($DIRECTORY)) {
echo "ERROR: Cannot open {$basepath}.";
exit(1);
}
}
function find_existing_filename_old($course) {
global $basepath, $options;
$existing = get_directory_list($basepath, '', false, false, true);
if (!($idnumber = $course->idnumber)) {
$idnumber = "noidnum";
}
$name = $idnumber."-".$course->id;
foreach ($existing as $filename) {
if (strncmp($filename, $name, strlen($name)) === 0 && strncmp('mbz', substr($filename, -3), 3) === 0) {
return $filename;
}
}
}
function check_backup_good($backupfile) {
global $basepath, $options, $users;
// A collection of files we expect to be in the complete archive.
// 1 means any filetype, 2 means only in files with user data.
$expected = ['.ARCHIVE_INDEX' => 1,
'badges.xml' => 2,
'completion.xml' => 1,
'course/calendar.xml' => 1,
'course/course.xml' => 1,
'course/comments.xml' => 2,
'course/enrolments.xml' => 1,
'course/filters.xml' => 1,
'course/inforef.xml' => 1,
'course/logs.xml' => 2,
'course/logstores.xml' => 2,
'course/roles.xml' => 1,
'files.xml' => 1,
'grade_history.xml' => 1,
'gradebook.xml' => 1,
'groups.xml' => 1,
'moodle_backup.xml' => 1,
'outcomes.xml' => 1,
'questions.xml' => 1,
'roles.xml' => 1,
'scales.xml' => 1,
'sections/' => 1,
'users.xml' => 2,
'moodle_backup.log' => 1];
if (!$users) {
foreach ($expected as $key => $value) {
if ($value == 2) {
unset($expected[$key]);
}
}
}
$command = 'tar -tf '.escapeshellarg($basepath.'/'.$backupfile).' 2>&1';
exec($command, $output, $returncode);
if ($returncode !== 0) {
echo "ERROR: Bad return code reading file at ".$basepath.'/'.$backupfile.". ";
return false;
}
if (empty($output) || count($output) < 10) {
// BAD!
echo "ERROR: Bad existing file found at ".$basepath.'/'.$backupfile.". ";
return false;
}
// We process through each line of the result, removing it from the expected array.
foreach ($output as $file) {
unset($expected[$file]);
}
// If this array is empty when we are done, then we know we got all the files we expected.
if (empty($expected)) {
return true;
} else {
// There were expected files not found, so we are going to flag this file.
echo "ERROR: Bad existing file found at ".$basepath.'/'.$backupfile.". Missing files: \n "
.implode(array_keys($expected), "\n ");
return false;
}
}
function create_check_file($filename) {
global $basepath;
$sha = sha1_file($basepath.'/'.$filename);
$newname = substr($filename, 0, -3).'sha1';
file_put_contents($basepath.'/'.$newname, $sha);
//touch($basepath.'/'.$newname);
}
function check_file_exists($filename) {
global $basepath, $options;
$checkname = substr($filename, 0, -3).'sha1';
$checkpath = $basepath.'/'.$checkname;
$exists = search_directory_for_file($checkname, 'sha1');
if (!$exists) {
return false;
}
if (empty($options['check-checksum'])) {
return true;
}
// Check the contents of the file.
$fh = fopen($checkpath, 'r');
if (empty($fh)) {
return false;
}
$checksha = trim(fread($fh, 45));
fclose($fp);
if (empty($checksha) || strlen($checksha) !== 40) {
unlink($checkpath);
return false;
}
$newsha = sha1_file($basepath.'/'.$filename);
if (strcmp($checksha, $newsha) === 0) {
// This means the file was good with a matching sha.
return true;
}
// This means the sha checksums didn't match. We should delete it.
unlink($checkpath);
return false;
}
function get_filename_base($course) {
global $users;
if (!($idnumber = $course->idnumber)) {
$idnumber = "noidnum";
}
$name = $idnumber."-".$course->id;
if ($users) {
$name .= '-userdata';
} else {
$name .= '-nousers';
}
return $name;
}
// Force the config for what we are doing.
function setup_config() {
global $CFG, $users, $options;
if ($users) {
$settings = ['backup_auto_storage' => 1,
'backup_auto_users' => true,
'backup_auto_role_assignments' => true,
'backup_auto_activities' => true,
'backup_auto_blocks' => true,
'backup_auto_filters' => true,
'backup_auto_comments' => true,
'backup_auto_badges' => true,
'backup_auto_calendarevents' => true,
'backup_auto_userscompletion' => true,
'backup_auto_logs' => true,
'backup_auto_histories' => true,
'backup_auto_questionbank' => true,
'backup_auto_groups' => true,
'backup_auto_competencies' => true];
} else {
$settings = ['backup_auto_storage' => 1,
'backup_auto_users' => false,
'backup_auto_role_assignments' => false,
'backup_auto_activities' => true,
'backup_auto_blocks' => true,
'backup_auto_filters' => true,
'backup_auto_comments' => false,
'backup_auto_badges' => false,
'backup_auto_calendarevents' => true,
'backup_auto_userscompletion' => false,
'backup_auto_logs' => false,
'backup_auto_histories' => false,
'backup_auto_questionbank' => true,
'backup_auto_groups' => true,
'backup_auto_competencies' => true];
}
if ($options['directory']) {
$dir = $options['directory'];
if (strpos($dir, '/') !== 0) {
echo "ERROR: Directory {$dir} is not absolue - must start with a slash.\n";
exit(1);
}
if (strrpos($dir, '/') === strlen($dir)-1) {
$dir = substr($dir, 0, strlen($dir)-1);
}
if (!is_dir($dir) || !is_writable($dir)) {
echo "ERROR: Directory {$dir} does not exist or is not writable.\n";
exit(1);
}
$settings['backup_auto_destination'] = $dir;
}
if (isset($CFG->forced_plugin_settings['backup'])) {
$CFG->forced_plugin_settings['backup'] = array_merge($CFG->forced_plugin_settings['backup'], $settings);
} else {
$CFG->forced_plugin_settings['backup'] = $settings;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment