Sichern von MySQL-Datenbanken im Objektspeicher mit Percona unter Ubuntu 16.04

sudo nano /usr/local/bin/object_storage.py

#!/usr/bin/env python3

import argparse
import os
import sys
from datetime import datetime, timedelta

import boto3
import pytz
from botocore.client import ClientError, Config
from dateutil.parser import parse

# "backup_bucket" must be a universally unique name, so choose something
# specific to your setup.
# The bucket will be created in your account if it does not already exist
backup_bucket = os.environ['MYBUCKETNAME']
access_key = os.environ['MYACCESSKEY']
secret_key = os.environ['MYSECRETKEY']
endpoint_url = os.environ['MYENDPOINTURL']
region_name = os.environ['MYREGIONNAME']


class Space():
   def __init__(self, bucket):
       self.session = boto3.session.Session()
       self.client = self.session.client('s3',
                                         region_name=region_name,
                                         endpoint_url=endpoint_url,
                                         aws_access_key_id=access_key,
                                         aws_secret_access_key=secret_key,
                                         config=Config(signature_version='s3')
                                         )
       self.bucket = bucket
       self.paginator = self.client.get_paginator('list_objects')

   def create_bucket(self):
       try:
           self.client.head_bucket(Bucket=self.bucket)
       except ClientError as e:
           if e.response['Error']['Code'] == '404':
               self.client.create_bucket(Bucket=self.bucket)
           elif e.response['Error']['Code'] == '403':
               print("The bucket name \"{}\" is already being used by "
                     "someone.  Please try using a different bucket "
                     "name.".format(self.bucket))
               sys.exit(1)
           else:
               print("Unexpected error: {}".format(e))
               sys.exit(1)

   def upload_files(self, files):
       for filename in files:
           self.client.upload_file(Filename=filename, Bucket=self.bucket,
                                   Key=os.path.basename(filename))
           print("Uploaded {} to \"{}\"".format(filename, self.bucket))

   def remove_file(self, filename):
       self.client.delete_object(Bucket=self.bucket,
                                 Key=os.path.basename(filename))

   def prune_backups(self, days_to_keep):
       oldest_day = datetime.now(pytz.utc) - timedelta(days=int(days_to_keep))
       try:
           # Create an iterator to page through results
           page_iterator = self.paginator.paginate(Bucket=self.bucket)
           # Collect objects older than the specified date
           objects_to_prune = [filename['Key'] for page in page_iterator
                               for filename in page['Contents']
                               if filename['LastModified'] < oldest_day]
       except KeyError:
           # If the bucket is empty
           sys.exit()
       for object in objects_to_prune:
           print("Removing \"{}\" from {}".format(object, self.bucket))
           self.remove_file(object)

   def download_file(self, filename):
       self.client.download_file(Bucket=self.bucket,
                                 Key=filename, Filename=filename)

   def get_day(self, day_to_get):
       try:
           # Attempt to parse the date format the user provided
           input_date = parse(day_to_get)
       except ValueError:
           print("Cannot parse the provided date: {}".format(day_to_get))
           sys.exit(1)
       day_string = input_date.strftime("-%m-%d-%Y_")
       print_date = input_date.strftime("%A, %b. %d %Y")
       print("Looking for objects from {}".format(print_date))
       try:
           # create an iterator to page through results
           page_iterator = self.paginator.paginate(Bucket=self.bucket)
           objects_to_grab = [filename['Key'] for page in page_iterator
                              for filename in page['Contents']
                              if day_string in filename['Key']]
       except KeyError:
           print("No objects currently in bucket")
           sys.exit()
       if objects_to_grab:
           for object in objects_to_grab:
               print("Downloading \"{}\" from {}".format(object, self.bucket))
               self.download_file(object)
       else:
           print("No objects found from: {}".format(print_date))
           sys.exit()


def is_valid_file(filename):
   if os.path.isfile(filename):
       return filename
   else:
       raise argparse.ArgumentTypeError("File \"{}\" does not exist."
                                        .format(filename))


def parse_arguments():
   parser = argparse.ArgumentParser(
       description='''Client to perform backup-related tasks with
                    object storage.''')
   subparsers = parser.add_subparsers()

   # parse arguments for the "upload" command
   parser_upload = subparsers.add_parser('upload')
   parser_upload.add_argument('files', type=is_valid_file, nargs='+')
   parser_upload.set_defaults(func=upload)

   # parse arguments for the "prune" command
   parser_prune = subparsers.add_parser('prune')
   parser_prune.add_argument('--days-to-keep', default=30)
   parser_prune.set_defaults(func=prune)

   # parse arguments for the "download" command
   parser_download = subparsers.add_parser('download')
   parser_download.add_argument('filename')
   parser_download.set_defaults(func=download)

   # parse arguments for the "get_day" command
   parser_get_day = subparsers.add_parser('get_day')
   parser_get_day.add_argument('day')
   parser_get_day.set_defaults(func=get_day)

   return parser.parse_args()


def upload(space, args):
   space.upload_files(args.files)


def prune(space, args):
   space.prune_backups(args.days_to_keep)


def download(space, args):
   space.download_file(args.filename)


def get_day(space, args):
   space.get_day(args.day)


def main():
   args = parse_arguments()
   space = Space(bucket=backup_bucket)
   space.create_bucket()
   args.func(space, args)


if __name__ == '__main__':
   main()

sudo chmod +x /usr/local/bin/object_storage.py

sudo nano /usr/local/bin/remote-backup-mysql.sh

#!/bin/bash

export LC_ALL=C

days_to_keep=30
backup_owner="backup"
parent_dir="/backups/mysql"
defaults_file="/etc/mysql/backup.cnf"
working_dir="${parent_dir}/working"
log_file="${working_dir}/backup-progress.log"
encryption_key_file="${parent_dir}/encryption_key"
storage_configuration_file="${parent_dir}/object_storage_config.sh"
now="$(date)"
now_string="$(date -d"${now}" +%m-%d-%Y_%H-%M-%S)"
processors="$(nproc --all)"

# Use this to echo to standard error
error () {
   printf "%s: %s\n" "$(basename "${BASH_SOURCE}")" "${1}" >&2
   exit 1
}

trap 'error "An unexpected error occurred."' ERR

sanity_check () {
   # Check user running the script
   if [ "$(id --user --name)" != "$backup_owner" ]; then
       error "Script can only be run as the \"$backup_owner\" user"
   fi

   # Check whether the encryption key file is available
   if [ ! -r "${encryption_key_file}" ]; then
       error "Cannot read encryption key at ${encryption_key_file}"
   fi

   # Check whether the object storage configuration file is available
   if [ ! -r "${storage_configuration_file}" ]; then
       error "Cannot read object storage configuration from ${storage_configuration_file}"
   fi

   # Check whether the object storage configuration is set in the file
   source "${storage_configuration_file}"
   if [ -z "${MYACCESSKEY}" ] || [ -z "${MYSECRETKEY}" ] || [ -z "${MYBUCKETNAME}" ]; then
       error "Object storage configuration are not set properly in ${storage_configuration_file}"
   fi
}

set_backup_type () {
   backup_type="full"


   # Grab date of the last backup if available
   if [ -r "${working_dir}/xtrabackup_info" ]; then
       last_backup_date="$(date -d"$(grep start_time "${working_dir}/xtrabackup_info" | cut -d' ' -f3)" +%s)"
   else
           last_backup_date=0
   fi

   # Grab today's date, in the same format
   todays_date="$(date -d "$(date -d "${now}" "+%D")" +%s)"

   # Compare the two dates
   (( $last_backup_date == $todays_date ))
   same_day="${?}"

   # The first backup each new day will be a full backup
   # If today's date is the same as the last backup, take an incremental backup instead
   if [ "$same_day" -eq "0" ]; then
       backup_type="incremental"
   fi
}

set_options () {
   # List the xtrabackup arguments
   xtrabackup_args=(
       "--defaults-file=${defaults_file}"
       "--backup"
       "--extra-lsndir=${working_dir}"
       "--compress"
       "--stream=xbstream"
       "--encrypt=AES256"
       "--encrypt-key-file=${encryption_key_file}"
       "--parallel=${processors}"
       "--compress-threads=${processors}"
       "--encrypt-threads=${processors}"
       "--slave-info"
   )

   set_backup_type

   # Add option to read LSN (log sequence number) if taking an incremental backup
   if [ "$backup_type" == "incremental" ]; then
       lsn=$(awk '/to_lsn/ {print $3;}' "${working_dir}/xtrabackup_checkpoints")
       xtrabackup_args+=( "--incremental-lsn=${lsn}" )
   fi
}

rotate_old () {
   # Remove previous backup artifacts
   find "${working_dir}" -name "*.xbstream" -type f -delete

   # Remove any backups from object storage older than 30 days
   /usr/local/bin/object_storage.py prune --days-to-keep "${days_to_keep}"
}

take_backup () {
   find "${working_dir}" -type f -name "*.incomplete" -delete
   xtrabackup "${xtrabackup_args[@]}" --target-dir="${working_dir}" > "${working_dir}/${backup_type}-${now_string}.xbstream.incomplete" 2> "${log_file}"

   mv "${working_dir}/${backup_type}-${now_string}.xbstream.incomplete" "${working_dir}/${backup_type}-${now_string}.xbstream"
}

upload_backup () {
   /usr/local/bin/object_storage.py upload "${working_dir}/${backup_type}-${now_string}.xbstream"
}

main () {
   mkdir -p "${working_dir}"
   sanity_check && set_options && rotate_old && take_backup && upload_backup

   # Check success and print message
   if tail -1 "${log_file}" | grep -q "completed OK"; then
       printf "Backup successful!\n"
       printf "Backup created at %s/%s-%s.xbstream\n" "${working_dir}" "${backup_type}" "${now_string}"
   else
       error "Backup failure! If available, check ${log_file} for more information"
   fi
}

main

sudo chmod +x /usr/local/bin/remote-backup-mysql.sh

sudo nano /usr/local/bin/download-day.sh

#!/bin/bash

export LC_ALL=C

backup_owner="backup"
storage_configuration_file="/backups/mysql/object_storage_config.sh"
day_to_download="${1}"

# Use this to echo to standard error
error () {
   printf "%s: %s\n" "$(basename "${BASH_SOURCE}")" "${1}" >&2
   exit 1
}

trap 'error "An unexpected error occurred."' ERR

sanity_check () {
   # Check user running the script
   if [ "$(id --user --name)" != "$backup_owner" ]; then
       error "Script can only be run as the \"$backup_owner\" user"
   fi

   # Check whether the object storage configuration file is available
   if [ ! -r "${storage_configuration_file}" ]; then
       error "Cannot read object storage configuration from ${storage_configuration_file}"
   fi

   # Check whether the object storage configuration is set in the file
   source "${storage_configuration_file}"
   if [ -z "${MYACCESSKEY}" ] || [ -z "${MYSECRETKEY}" ] || [ -z "${MYBUCKETNAME}" ]; then
       error "Object storage configuration are not set properly in ${storage_configuration_file}"
   fi
}

main () {
   sanity_check
   /usr/local/bin/object_storage.py get_day "${day_to_download}"
}

main

sudo chmod +x /usr/local/bin/download-day.sh

sudo -u backup remote-backup-mysql.sh

OutputUploaded /backups/mysql/working/full-10-17-2017_19-09-30.xbstream to ""
Backup successful!
Backup created at /backups/mysql/working/full-10-17-2017_19-09-30.xbstream

ls /backups/mysql/working

Outputbackup-progress.log  full-10-17-2017_19-09-30.xbstream  xtrabackup_checkpoints  xtrabackup_info

mysql -u root -p -e 'INSERT INTO playground.equipment (type, quant, color) VALUES ("sandbox", 4, "brown");'

sudo -u backup remote-backup-mysql.sh

OutputUploaded /backups/mysql/working/incremental-10-17-2017_19-19-20.xbstream to ""
Backup successful!
Backup created at /backups/mysql/working/incremental-10-17-2017_19-19-20.xbstream

ls /backups/mysql/working

Outputbackup-progress.log  incremental-10-17-2017_19-19-20.xbstream  xtrabackup_checkpoints  xtrabackup_info

sudo -u backup mkdir /tmp/backup_archives
cd /tmp/backup_archives

sudo -u backup download-day.sh ""

OutputLooking for objects from Tuesday, Oct. 17 2017
Downloading "full-10-17-2017_19-09-30.xbstream" from
Downloading "incremental-10-17-2017_19-19-20.xbstream" from

ls

Outputfull-10-17-2017_19-09-30.xbstream  incremental-10-17-2017_19-19-20.xbstream

sudo -u backup extract-mysql.sh *.xbstream

cd restore
sudo -u backup prepare-mysql.sh

OutputBackup looks to be fully prepared.  Please check the "prepare-progress.log" file
to verify before continuing.

If everything looks correct, you can apply the restored files.

First, stop MySQL and move or remove the contents of the MySQL data directory:

       sudo systemctl stop mysql
       sudo mv /var/lib/mysql/ /tmp/

Then, recreate the data directory and  copy the backup files:

       sudo mkdir /var/lib/mysql
       sudo xtrabackup --copy-back --target-dir=/tmp/backup_archives/restore/full-10-17-2017_19-09-30

Afterward the files are copied, adjust the permissions and restart the service:

       sudo chown -R mysql:mysql /var/lib/mysql
       sudo find /var/lib/mysql -type d -exec chmod 750 {} \;
       sudo systemctl start mysql

sudo systemctl stop mysql

sudo mv /var/lib/mysql/ /tmp/mysql-remote

sudo mkdir /var/lib/mysql

sudo xtrabackup --copy-back --target-dir=

sudo chown -R mysql:mysql /var/lib/mysql
sudo find /var/lib/mysql -type d -exec chmod 750 {} \;

sudo systemctl start mysql
mysql -u root -p -e 'SELECT * FROM playground.equipment;'

Output+----+---------+-------+--------+
| id | type    | quant | color  |
+----+---------+-------+--------+
|  1 | slide   |     2 | blue   |
|  2 | swing   |    10 | yellow |
|  3 | sandbox |     4 | brown  |
+----+---------+-------+--------+

cd ~
sudo rm -rf /tmp/backup_archives/restore