--- xtrabackup-1.2/innobackupex-1.5.1 Fri Dec 10 13:26:33 2010 +++ xtrabackup-1.2-orig/xtrabackup-1.2/innobackupex-1.5.1 Sun Apr 11 10:12:39 2010 @@ -15,7 +15,6 @@ use FileHandle; use File::Basename; use English qw(-no_match_vars); -use Data::Dumper; # version of this script my $innobackup_version = '1.5.1-xtrabackup'; @@ -31,8 +30,6 @@ "; - - # required Perl version (5.005) my @required_perl_version = (5, 0, 5); my $required_perl_version_old_style = 5.005; @@ -70,7 +67,6 @@ my $option_copy_back = ''; my $option_include = ''; my $option_databases = ''; -my $option_exclude = ''; my $option_throttle = ''; my $option_sleep = ''; my $option_compress = 999; @@ -206,8 +202,6 @@ init(); my $ibbackup_exit_code = 0; -my $dbs_not_backed_up_aref = []; -my $db_files_not_backed_up_href = {}; if ($option_copy_back) { # copy files from backup directory back to their original locations @@ -217,100 +211,24 @@ apply_log(); } else { # make a backup of InnoDB and MyISAM tables, indexes and .frm files. - ($ibbackup_exit_code, - $dbs_not_backed_up_aref, - $db_files_not_backed_up_href) = backup(); + $ibbackup_exit_code = backup(); } $now = current_time(); -my $num_dbs_not_backed_up = scalar(@$dbs_not_backed_up_aref); -my $num_dbs_with_missing_files = scalar( keys(%$db_files_not_backed_up_href) ); - - if ($option_stream eq 'tar') { print STDERR "$prefix You must use -i (--ignore-zeros) option for extraction of the tar stream.\n"; } -# nick.pavey@amd.com, 12/10/2010 -# Convert if / elseif structure into SWITCH to improve readability -# Make sure that an appropriate error message is output for each end condition - we -# do not want a user to think their backup is complete when in fact it's possible that it isn't. - -SWITCH : { - - # program has completed successfully with no errors and nothing skipped - if ( ($ibbackup_exit_code == 0 ) && - ($num_dbs_not_backed_up == 0) && - ($num_dbs_with_missing_files == 0) ) { - - print STDERR "\n$now $prefix completed OK!\n\n"; - - last SWITCH; - } - - # The InnoDB portion completed OK, but the part of the backup implemented in this script encountered - # at least some issues. This may not necessarily mean that the backup is invalid, but we have to let - # the user know. - if ( ($ibbackup_exit_code == 0 ) && - ($num_dbs_not_backed_up != 0) || - ($num_dbs_with_missing_files != 0) ) { - - print STDERR "\n$now $prefix InnoDB backup completed OK, but some databases / files were not backed up.\n"; - print STDERR "$now $prefix This may or may not be an issue, depending on your command line switches\n"; - print STDERR "$now $prefix and the way in which your database operates\n\n"; - - display_not_backed_up_info($dbs_not_backed_up_aref, - $db_files_not_backed_up_href); - - last SWITCH; - } - - - - # InnoDB backup didn't work out, but the regular file copy worked OK. - if ( ($ibbackup_exit_code == 1) && - ($num_dbs_not_backed_up == 0) && - ($num_dbs_with_missing_files == 0) ) { - - print STDERR "$now $prefix $option_ibbackup_binary failed! (exit code $ibbackup_exit_code) The backup may not be complete.\n\n"; - last SWITCH; - } - - - # Nothing worked out! - if ( ($ibbackup_exit_code == 1) && - ($num_dbs_not_backed_up != 0) || - ($num_dbs_with_missing_files != 0) ) { - - print STDERR "$now $prefix $option_ibbackup_binary failed and some dbs / files were not backed up! (exit code $ibbackup_exit_code) The backup may not be complete.\n\n"; - - print STDERR "\n$now $prefix Some databases / files were not backed up.\n"; - print STDERR "$now $prefix This may or may not be an issue, depending on your command line switches\n"; - print STDERR "$now $prefix and the way in which your database operates\n\n"; - - display_not_backed_up_info($dbs_not_backed_up_aref, - $db_files_not_backed_up_href); - - display_not_backed_up_info($dbs_not_backed_up_aref, - $db_files_not_backed_up_href); - - last SWITCH; - } - -} # end of SWITCH block - -print STDERR "\n$prefix Backup created in directory '$backup_dir'\n"; -if ($mysql_binlog_position) { - print STDERR "$prefix MySQL binlog position: $mysql_binlog_position\n"; +if ( $ibbackup_exit_code == 0 ) { + # program has completed successfully + print STDERR "$now $prefix completed OK!\n"; } -if ($mysql_slave_position && $option_slave_info) { - print STDERR "$prefix MySQL slave binlog position: $mysql_slave_position\n"; +else { + print STDERR "$now $prefix $option_ibbackup_binary failed! (exit code $ibbackup_exit_code) The backup may not be complete.\n"; } - -# Make sure that the exit code reflects all the possible ways in which a backup can be less than 100% good -exit ($ibbackup_exit_code || $num_dbs_not_backed_up || $num_dbs_with_missing_files); +exit $ibbackup_exit_code; ###################################################################### # end of program execution @@ -595,7 +513,7 @@ } # backup .frm, .MRG, .MYD, .MYI, .TRG, .TRN, .ARM, .ARZ and .opt files - my ($dbs_not_backed_up_aref, $db_files_not_backed_up_href) = backup_files(); + backup_files(); # resume ibbackup and wait till it has finished my $ibbackup_exit_code = resume_ibbackup(); @@ -626,8 +544,15 @@ unlink "$orig_datadir/xtrabackup_checkpoints" || Die "Failed to delete '$orig_datadir/xtrabackup_checkpoints': $!"; } + print STDERR "\n$prefix Backup created in directory '$backup_dir'\n"; + if ($mysql_binlog_position) { + print STDERR "$prefix MySQL binlog position: $mysql_binlog_position\n"; + } + if ($mysql_slave_position && $option_slave_info) { + print STDERR "$prefix MySQL slave binlog position: $mysql_slave_position\n"; + } - return ($ibbackup_exit_code, $dbs_not_backed_up_aref, $db_files_not_backed_up_href); + return $ibbackup_exit_code; } @@ -1110,7 +1035,7 @@ } if ($option_mysql_password) { - $options = "$options --password='$option_mysql_password'"; + $options = "$options --password=$option_mysql_password"; } if ($option_mysql_user) { $options = "$options --user=$option_mysql_user"; @@ -1734,7 +1659,6 @@ 'copy-back' => \$option_copy_back, 'include=s' => \$option_include, 'databases=s' => \$option_databases, - 'exclude=s' => \$option_exclude, 'use-memory=s' => \$option_use_memory, 'uncompress' => \$option_uncompress, 'export' => \$option_export, @@ -1947,89 +1871,39 @@ my $database; my $wildcard = '*.{frm,MYD,MYI,MRG,TRG,TRN,ARM,ARZ,opt,par}'; - # nick.pavey@amd.com, 12/10/2010 - # Modified the setup code for the direct file copy to make it clearer what's going on. - # Added code to show the user which backup operations didn't happen or failed in flight - - # Also added an additional check to avoid race conditions when --no-lock is used: - - # Backup starts - # Generate list of tables to back up - # Start iterating through the list - # Select the next table to backup - # A running MySQL server DROPs this table, asynchronously to this thread - # Try to back up the table but discover it has disappeared - # Fatal error which shuts down the backup - - # If the MySQL server was quiesced with FLUSH TABLES WITH READ LOCK before the - # backup of .frm etc etc tables was started, then this race condition will not be an issue. - - my @dbs_not_backed_up = (); - my $database_files_not_backed_up_href = {}; - - my $uname = `whoami`; - chomp $uname; - - opendir(DIR, $source_dir) + opendir(DIR, $source_dir) || Die "Can't open directory '$source_dir': $!\n"; - $now = current_time(); - print STDERR "\n$now $prefix Starting to backup .frm, .MRG, .MYD, .MYI,\n"; print STDERR "$prefix .TRG, .TRN, .ARM, .ARZ and .opt files in\n"; - print STDERR "$prefix subdirectories of '$source_dir'\n\n"; - - - my @database_dirs_to_backup = grep { $_ ne "." && $_ ne ".."} readdir(DIR); - - foreach $database (@database_dirs_to_backup) { - - my $db_path = "$source_dir/$database"; - - # skip files that are not database directories - next if (! -d $db_path ); - - print STDERR "\nBacking up database : $database\n"; - - if ( (! -r $db_path) && (! -w $db_path) ) { - print STDERR "\nDirectory $db_path cannot be fully accessed by the backup user ($uname) - skipping\n\n"; - - push @dbs_not_backed_up, $database; - next; - } - - my $print_each_file = 0; - my $file_c; - my @scp_files; - - if (check_if_required($database) != 1) { - print STDERR "Database '$database' is not required for this backup - skipping\n"; - - push @dbs_not_backed_up, $database; - next; - } - - if (!$option_remote_host && !$option_stream) { - - if (! -e "$backup_dir/$database") { - # create database directory for the backup - mkdir("$backup_dir/$database", 0777) - || Die "Couldn't create directory '$backup_dir/$database': $!"; - } - - } elsif ($option_remote_host) { - - if (system("ssh $option_remote_host test -e $backup_dir/$database") != 0) { - system("ssh $option_remote_host mkdir $backup_dir/$database"); - } - - } + print STDERR "$prefix subdirectories of '$source_dir'\n"; + # loop through all database directories + while (defined($database = readdir(DIR))) { + my $print_each_file = 0; + my $file_c; + my @scp_files; + # skip files that are not database directories + if ($database eq '.' || $database eq '..') { next; } + next unless -d "$source_dir/$database"; + next unless check_if_required($database); + + if (!$option_remote_host && !$option_stream) { + if (! -e "$backup_dir/$database") { + # create database directory for the backup + mkdir("$backup_dir/$database", 0777) + || Die "Couldn't create directory '$backup_dir/$database': $!"; + } + } elsif ($option_remote_host) { + if (system("ssh $option_remote_host test -e $backup_dir/$database") + != 0) { + system("ssh $option_remote_host mkdir $backup_dir/$database"); + } + } # copy files of this database opendir(DBDIR, "$source_dir/$database"); @list = grep(/\.(frm)|(MYD)|(MYI)|(MRG)|(TRG)|(TRN)|(ARM)|(ARZ)|(opt)|(par)$/, readdir(DBDIR)); closedir DBDIR; - $file_c = @list; if ($file_c <= $backup_file_print_limit) { $print_each_file = 1; @@ -2037,116 +1911,46 @@ print STDERR "$prefix Backing up files " . "'$source_dir/$database/$wildcard' ($file_c files)\n"; } - foreach $file (@list) { - - # copying may take a long time, so we have to prevent - # mysql connection from timing out - mysql_keep_alive(); - next unless check_if_required($database, $file); - - if ($print_each_file) { - print STDERR "$prefix Backing up file '$source_dir/$database/$file'\n"; - } - - # nick.pavey@amd.com, 12/10/2010 - # Convert if / elsif structure into a SWITCH block - easier to understand. - SWITCH : { - - if (!$option_remote_host && !$option_stream) { - $src_name = escape_path("$source_dir/$database/$file"); - $dst_name = escape_path("$backup_dir/$database"); - - my $rtn_code = system("$CP_CMD \"$src_name\" \"$dst_name\""); - if ($rtn_code != 0) { - print STDERR "Failed to copy file '$src_name' - continuing\n"; - - if (! exists $db_files_not_backed_up_href->{$database} ) { - $db_files_not_backed_up_href->{$database} = []; - } - - my $aref = $db_files_not_backed_up_href->{$database}; - push @$aref, $file; - next; - } - - last SWITCH; - } - - if ($option_remote_host) { - - if (! -e $file) { - print STDERR "Could not locate file '$file' - making note and continuing\n"; - - if (! exists $db_files_not_backed_up_href->{$database} ) { - $db_files_not_backed_up_href->{$database} = []; - } - - my $aref = $db_files_not_backed_up_href->{$database}; - push @$aref, $file; - next; - } - - # Queue up files for one single scp per database. - push(@scp_files, "'$file'"); - - last SWITCH; - } - - if($option_stream eq 'tar') { - - if (! -e $file) { - print STDERR "Could not locate file '$file' - making note and continuing\n"; - if (! exists $db_files_not_backed_up_href->{$database} ) { - $db_files_not_backed_up_href->{$database} = []; - } - - my $aref = $db_files_not_backed_up_href->{$database}; - push @$aref, $file; - next; - } - - my $ret = 0; - my $file_name = substr($file, rindex($file, '/') + 1); - $file_name=~s/([\$\\\" ])/\\$1/g; - $ret = system("cd $source_dir; tar cf - $database/$file_name") >> 8; - if ($ret == 1) { - print STDERR "$prefix If you use GNU tar, this warning can be ignored.\n"; - } elsif ($ret != 0) { - print STDERR "$prefix tar returned with exit code $ret.\n"; - print STDERR "Failed to stream '$database/$file_name': $!"; - print STDERR "Making note and continuing... \n"; - - if (! exists $db_files_not_backed_up_href->{$database} ) { - $db_files_not_backed_up_href->{$database} = []; - } - - my $aref = $db_files_not_backed_up_href->{$database}; - push @$aref, $file_name; - next; - } - - last SWITCH; - } - - } # end of SWITCH block - - } # end of inner FOREACH loop - + # copying may take a long time, so we have to prevent + # mysql connection from timing out + mysql_keep_alive(); + next unless check_if_required($database, $file); + + if ($print_each_file) { + print STDERR "$prefix Backing up file '$source_dir/$database/$file'\n"; + } + if (!$option_remote_host && !$option_stream) { + $src_name = escape_path("$source_dir/$database/$file"); + $dst_name = escape_path("$backup_dir/$database"); + system("$CP_CMD \"$src_name\" \"$dst_name\"") + and Die "Failed to copy file '$file': $!"; + } elsif ($option_remote_host) { + # Queue up files for one single scp per database. + push(@scp_files, "'$file'"); + } elsif($option_stream eq 'tar') { + my $ret = 0; + my $file_name = substr($file, rindex($file, '/') + 1); + $file_name=~s/([\$\\\" ])/\\$1/g; + $ret = system("cd $source_dir; tar cf - $database/$file_name") >> 8; + if ($ret == 1) { + print STDERR "$prefix If you use GNU tar, this warning can be ignored.\n"; + } elsif ($ret != 0) { + print STDERR "$prefix tar returned with exit code $ret.\n"; + Die "Failed to stream '$database/$file_name': $!"; + } + } + } if ($option_remote_host and @scp_files) { my $scp_file_list = join(" ", map { "$source_dir/$database/$_" } @scp_files); system("scp $option_scp_opt $scp_file_list '$option_remote_host:$backup_dir/$database/'") and Die "Failed to execute \"scp $option_scp_opt $scp_file_list '$option_remote_host:$backup_dir/$database/'\": $!"; } - - } # end of outer FOREACH loop - + } closedir(DIR); $now = current_time(); print STDERR "$now $prefix Finished backing up .frm, .MRG, .MYD, .MYI, .TRG, .TRN, .ARM, .ARZ and .opt files\n\n"; - - return (\@dbs_not_backed_up, $db_files_not_backed_up_href); } @@ -2359,37 +2163,10 @@ undef $table_path; } - # nick.pavey@amd.com, 12/10/2010 - # Extension to allow databases to be excluded from the backup - - # Search to see if this database has been excluded from the list of - # databases to back up. - - my @excluded_db_array = split ',', $option_exclude; - my $excluded_db = 0; - - my $tgt_db; - foreach $tgt_db (@excluded_db_array) { - if ($tgt_db eq $db) { - $excluded_db = 1; - last; - } - - } # end of FOREACH loop - - if ($excluded_db == 1) { - print STDERR "Database '$db' is excluded from the backup list\n"; - return 0; - } - - # NB - I'm deliberately leaving this check here, rather than moving it to - # the top of the subroutine, because I want to allow users to be able to - # exclude a database without having to also specify a '--database' option as well if ($db_count == 0) { - # no databases defined with --databases option, include all databases - return 1; + # no databases defined with --databases option, include all databases + return 1; } - if (defined $databases_list{$db}) { if (defined $table_path) { # get the last component in the table pathname @@ -2481,48 +2258,5 @@ $str =~ s/\/\//\//g; } return $str; - -} - - -# nick.pavey@amd.com, 12/10/2010 -# Added subroutine to display information about which databases / files were not fully backed up - -sub display_not_backed_up_info { - my ($dbs_not_backed_up_aref, - $db_files_not_backed_up_href) = @_; - - my $num_dbs_not_backed_up = scalar(@$dbs_not_backed_up_aref); - my $num_dbs_with_missing_files = scalar( keys(%$db_files_not_backed_up_href) ); - - if ($num_dbs_not_backed_up != 0) { - print STDERR "\nNumber of databases not backed up : $num_dbs_not_backed_up\n"; - - my $db; - foreach $db (@$dbs_not_backed_up_aref) { - print STDERR " $db\n"; - } - - } - - if ($num_dbs_with_missing_files != 0) { - print STDERR "\nNumber of databases where not all files were copied : $num_dbs_with_missing_files\n"; - - my ($db, $files_aref); - while ( ($db, $files_aref) = each %$db_files_not_backed_up_href) { - - print STDERR "Database : $db\n"; - - my $file; - foreach $file (@$files_aref) { - - print STDERR " $file\n"; - - } # end of FOREACH loop - - } # end of WHILE loop - - } - }