source: MondoRescue/branches/2.2.10/mondo/src/mondorestore/mondo-rstr-compare.c@ 2405

/***************************************************************************
    cvsid                : $Id: mondo-rstr-compare.c 2405 2009-09-17 01:45:24Z bruno $
***************************************************************************/

#include <pthread.h>
#include "my-stuff.h"
#include "mr_mem.h"
#include "../common/mondostructures.h"
#include "../common/libmondo.h"
//#include "../../config.h"
#include "mr-externs.h"
#include "mondo-rstr-compare.h"
#include "mondo-restore-EXT.h"
#include "mondo-rstr-tools-EXT.h"

extern char *MONDO_LOGFILE;

/* Reference to global bkpinfo */
extern struct s_bkpinfo *bkpinfo;

extern char *g_mountlist_fname;

//static char cvsid[] = "$Id: mondo-rstr-compare.c 2405 2009-09-17 01:45:24Z bruno $";

void popup_changelist_from_file(char *);


/**
 * @addtogroup LLcompareGroup
 * @{
 */
/**
 * Compare biggiefile number @p bigfileno with the filesystem mounted on @p MNT_RESTORING.
 * @param bkpinfo The backup information structure. Only used in insist_on_this_cd_number().
 * @param bigfileno The biggiefile number (starting from 0) to compare.
 * @note This function uses an MD5 checksum.
 */
int compare_a_biggiefile(long bigfileno)
{

    FILE *fin;
    FILE *fout;

  /** needs malloc *******/
    char *checksum = NULL;
    char *original_cksum = NULL;
    char *bigfile_fname = NULL;
    char *tmp = NULL;
    char *command = NULL;

    char *p;
    int i;
    int retval = 0;

    struct s_filename_and_lstat_info biggiestruct;

  /*********************************************************************
   * allocate memory clear test                sab 16 feb 2003         *
   *********************************************************************/
    assert(bkpinfo != NULL);
  /** end **/

    if (!does_file_exist(slice_fname(bigfileno, 0, ARCHIVES_PATH, ""))) {
        if (does_file_exist(MNT_CDROM "/archives/NOT-THE-LAST")) {
            insist_on_this_cd_number((++g_current_media_number));
        } else {
            log_msg(2, "No CD's left. No biggiefiles left. No prob, Bob.");
            return (0);
        }
    }
    if (!(fin = fopen(slice_fname(bigfileno, 0, ARCHIVES_PATH, ""), "r"))) {
        log_to_screen("Cannot open bigfile %ld (NULL)'s info file", bigfileno + 1);
        return (1);
    }
    fread((void *) &biggiestruct, 1, sizeof(biggiestruct), fin);
    paranoid_fclose(fin);

    mr_asprintf(checksum, "%s", biggiestruct.checksum);
    mr_asprintf(bigfile_fname, "%s", biggiestruct.filename);

    log_msg(2, "biggiestruct.filename = %s", biggiestruct.filename);
    log_msg(2, "biggiestruct.checksum = %s", biggiestruct.checksum);

    if (!g_text_mode) {
        mr_asprintf(tmp, "Comparing %s", bigfile_fname);
        newtDrawRootText(0, 22, tmp);
        paranoid_free(tmp);
        newtRefresh();
    }
    if (!checksum[0]) {
        log_msg(2, "Warning - %s has no checksum", bigfile_fname);
    }
    if (!strncmp(bigfile_fname, "/dev/", 5)) {
        log_msg(2, "IGNORING %s as begining with /dev", bigfile_fname);
        mr_free(bigfile_fname);
        return (1);
    }

    mr_asprintf(command, "md5sum \"%s%s\" > /tmp/md5sum.txt 2> /tmp/errors", MNT_RESTORING, bigfile_fname);
    log_msg(2, command);
    i =  system(command);
    mr_free(command);

    mr_asprintf(tmp, "cat /tmp/errors >> %s 2> /dev/null", MONDO_LOGFILE);
    paranoid_system(tmp);
    paranoid_free(tmp);

    if (i) {
        log_OS_error("Warning - command failed");
        return (1);
    } else {
        if (!(fin = fopen("/tmp/md5sum.txt", "r"))) {
            log_msg(2,
                    "Unable to open /tmp/md5sum.txt; can't get live checksum");
            mr_free(bigfile_fname);
            return (1);
        } else {
            mr_getline(original_cksum, fin);
            paranoid_fclose(fin);
            p = strchr(original_cksum, ' ');
            if (p) {
                *p = '\0';
            }
        }
    }
    mr_asprintf(tmp, "bigfile #%ld ('%s') ", bigfileno + 1, bigfile_fname);
    if ((original_cksum != NULL) && (strcmp(checksum, original_cksum) == 0)) {
        mr_strcat(tmp, " ... OK");
    } else {
        mr_strcat(tmp, "... changed");
        retval++;
    }
    mr_free(checksum);
    mr_free(original_cksum);

    log_msg(1, tmp);
    paranoid_free(tmp);

    if (retval) {
        if (!(fout = fopen(MONDO_CACHE"/changed.txt", "a"))) {
            fatal_error("Cannot openout changed.txt");
        }
        fprintf(fout, "%s\n", bigfile_fname);
        paranoid_fclose(fout);
    }
    mr_free(bigfile_fname);

    return (retval);
}

/**************************************************************************
 *END_COMPARE_A_BIGGIEFILE                                                *
 **************************************************************************/


/**
 * Compare all biggiefiles in the backup.
 * @return 0 for success, nonzero for failure.
 */
int compare_all_biggiefiles()
{
    int retval = 0;
    int res;
    long noof_biggiefiles, bigfileno = 0;
    char *tmp = NULL;

    log_msg(1, "Comparing biggiefiles");

    if (length_of_file(BIGGIELIST) < 6) {
        log_msg(1,
                "OK, really teeny-tiny biggielist; not comparing biggiefiles");
        return (0);
    }
    noof_biggiefiles = count_lines_in_file(BIGGIELIST);
    if (noof_biggiefiles <= 0) {
        log_msg(1, "OK, no biggiefiles; not comparing biggiefiles");
        return (0);
    }
    mvaddstr_and_log_it(g_currentY, 0,
                        "Comparing large files                                                  ");
    open_progress_form("Comparing large files",
                       "I am now comparing the large files",
                       "against the filesystem. Please wait.", "",
                       noof_biggiefiles);
    for (bigfileno = 0; bigfileno < noof_biggiefiles; bigfileno++) {
        mr_asprintf(tmp, "Comparing big file #%ld", bigfileno + 1);
        log_msg(1, tmp);
        update_progress_form(tmp);
        mr_free(tmp);

        res = compare_a_biggiefile(bigfileno);
        retval += res;
        g_current_progress++;
    }
    close_progress_form();
    return (0);
    if (retval) {
        mvaddstr_and_log_it(g_currentY++, 74, "Errors.");
    } else {
        mvaddstr_and_log_it(g_currentY++, 74, "Done.");
    }
    return (retval);
}

/**************************************************************************
 *END_COMPARE_ALL_BIGGIEFILES                                             *
 **************************************************************************/


/**
 * Compare afioball @p tarball_fname against the filesystem.
 * You must be chdir()ed to the directory where the filesystem is mounted
 * before you call this function.
 * @param tarball_fname The filename of the tarball to compare.
 * @param current_tarball_number The fileset number contained in @p tarball_fname.
 * @return 0 for success, nonzero for failure.
 */
int compare_a_tarball(char *tarball_fname, int current_tarball_number)
{
    int retval = 0;
    int res;
    long noof_lines;
    long archiver_errors;
    bool use_star;

  /***  needs malloc *********/
    char *command = NULL;
    char *tmp = NULL;
    char *archiver_exe = NULL;
    char *tmp1 = NULL;
    char *filelist_name = NULL;
    char *logfile = NULL;
    char *compressor_exe = NULL;

    use_star = (strstr(tarball_fname, ".star")) ? TRUE : FALSE;
    assert_string_is_neither_NULL_nor_zerolength(tarball_fname);
    mr_asprintf(filelist_name, MNT_CDROM "/archives/filelist.%d", current_tarball_number);
    noof_lines = count_lines_in_file(filelist_name);
    mr_free(filelist_name);

    if (strstr(tarball_fname, ".bz2")) {
        mr_asprintf(compressor_exe, "bzip2");
    } else if (strstr(tarball_fname, ".lzma")) {
        mr_asprintf(compressor_exe, "lzma");
    } else if (strstr(tarball_fname, ".gz")) {
        mr_asprintf(compressor_exe, "gzip");
    } else if (strstr(tarball_fname, ".lzo")) {
        mr_asprintf(compressor_exe, "lzop");
    }

    if (use_star) {
        mr_asprintf(archiver_exe, "star");
    } else {
        mr_asprintf(archiver_exe, "afio");
    }

    if (compressor_exe) {
        tmp1 = find_home_of_exe(compressor_exe);
        if (!tmp1) {
            mr_free(tmp1);
            mr_free(compressor_exe);
            mr_free(archiver_exe);
            fatal_error("(compare_a_tarball) Compression program missing");
        }
        mr_free(tmp1);

        if (use_star) {
            if (!strcmp(compressor_exe, "bzip2")) {
                mr_strcat(archiver_exe, " -bz");
            } else {
                mr_free(compressor_exe);
                mr_free(archiver_exe);
                fatal_error("(compare_a_tarball) Please use only bzip2 with star");
            }
        } else {
            // afio
            mr_free(compressor_exe);
            mr_asprintf(tmp, "%s", compressor_exe);
            mr_asprintf(compressor_exe, "-P %s -Z", tmp);
            mr_free(tmp);
        }
    }

#ifdef __FreeBSD__
#define BUFSIZE 512L
#else
#define BUFSIZE (1024L*1024L)/TAPE_BLOCK_SIZE
#endif
    mr_asprintf(logfile, "/tmp/afio.log.%d", current_tarball_number);

    if (use_star) {
        // doesn't use compressor_exe
        mr_asprintf(command, "%s -diff H=star file=%s >> %s 2>> %s", archiver_exe, tarball_fname, logfile, logfile);
    } else {
        mr_asprintf(command, "%s -r -b %ld -M 16m -c %ld %s %s >> %s 2>> %s", archiver_exe, TAPE_BLOCK_SIZE, BUFSIZE, compressor_exe, tarball_fname, logfile, logfile);
    }
    mr_free(compressor_exe);
    paranoid_free(archiver_exe);

#undef BUFSIZE

    res = system(command);
    retval += res;
    if (res) {
        log_OS_error(command);
        log_msg(2, "Warning - afio returned error = %d", res);
    }
    mr_free(command);

    if (length_of_file(logfile) > 5) {
        mr_asprintf(command, "sed s/': \\\"'/\\|/ %s | sed s/'\\\": '/\\|/ | cut -d'|' -f2 | sort -u | grep -vE \"^dev/.*\" >> "MONDO_CACHE"/changed.txt", logfile);
        system(command);
        mr_free(command);

        archiver_errors = count_lines_in_file(logfile);
    } else {
        archiver_errors = 0;
    }
    if (archiver_errors) {
        log_msg(1, "%ld difference%c in fileset #%d          ", archiver_errors, (archiver_errors != 1) ? 's' : ' ', current_tarball_number);
    }
    unlink(logfile);
    mr_free(logfile);

    return (retval);
}

/**************************************************************************
 *END_COMPARE_A_TARBALL                                                   *
 **************************************************************************/


/**
 * Compare all afioballs in this backup.
 * @param bkpinfo The backup media structure. Passed to other functions.
 * @return 0 for success, nonzero for failure.
 */
int compare_all_tarballs()
{
    int retval = 0;
    int res;
    int current_tarball_number = 0;

  /**  needs malloc **********/

    char *tarball_fname = NULL;
    char *progress_str = NULL; 
    char *tmp = NULL;
    char *mds = NULL;
    long max_val;

    assert(bkpinfo != NULL);
    mvaddstr_and_log_it(g_currentY, 0, "Comparing archives");
    tmp = read_cfg_var(g_mondo_cfg_file, "last-filelist-number");
    max_val = atol(tmp);
    mr_free(tmp);

    mds = media_descriptor_string(bkpinfo->backup_media_type);
    mr_asprintf(progress_str, "Comparing with %s #%d ", mds, g_current_media_number);

    open_progress_form("Comparing files",
                       "Comparing tarballs against filesystem.",
                       "Please wait. This may take some time.",
                       progress_str, max_val);

    log_to_screen(progress_str);

    for (;;) {
        insist_on_this_cd_number(g_current_media_number);
        update_progress_form(progress_str);
        mr_asprintf(tarball_fname, MNT_CDROM "/archives/%d.afio.bz2", current_tarball_number);

        if (!does_file_exist(tarball_fname)) {
            mr_free(tarball_fname);
            mr_asprintf(tarball_fname, MNT_CDROM "/archives/%d.afio.lzo", current_tarball_number);
        }
        if (!does_file_exist(tarball_fname)) {
            mr_free(tarball_fname);
            mr_asprintf(tarball_fname, MNT_CDROM "/archives/%d.afio.lzma", current_tarball_number);
        }
        if (!does_file_exist(tarball_fname)) {
            mr_free(tarball_fname);
            mr_asprintf(tarball_fname, MNT_CDROM "/archives/%d.afio.gz", current_tarball_number);
        }
        if (!does_file_exist(tarball_fname)) {
            mr_free(tarball_fname);
            mr_asprintf(tarball_fname, MNT_CDROM "/archives/%d.afio.", current_tarball_number);
        }
        if (!does_file_exist(tarball_fname)) {
            mr_free(tarball_fname);
            mr_asprintf(tarball_fname, MNT_CDROM "/archives/%d.star.bz2", current_tarball_number);
        }
        if (!does_file_exist(tarball_fname)) {
            mr_free(tarball_fname);
            mr_asprintf(tarball_fname, MNT_CDROM "/archives/%d.star.", current_tarball_number);
        }
        if (!does_file_exist(tarball_fname)) {
            if (!does_file_exist(MNT_CDROM "/archives/NOT-THE-LAST") ||
                system("find " MNT_CDROM
                       "/archives/slice* > /dev/null 2> /dev/null")
                == 0) {
                log_msg(2, "OK, I think I'm done with tarballs...");
                mr_free(tarball_fname);
                break;
            }
            log_msg(2, "OK, I think it's time for another CD...");
            g_current_media_number++;

            mr_free(progress_str);
            mr_asprintf(progress_str, "Comparing with %s #%d ", mds, g_current_media_number);
            log_to_screen(progress_str);
        } else {
            res = compare_a_tarball(tarball_fname, current_tarball_number);

            g_current_progress++;
            current_tarball_number++;
        }
        mr_free(tarball_fname);
    }
    mr_free(progress_str);
    mr_free(mds);

    close_progress_form();
    if (retval) {
        mvaddstr_and_log_it(g_currentY++, 74, "Errors.");
    } else {
        mvaddstr_and_log_it(g_currentY++, 74, "Done.");
    }
    return (retval);
}

/**************************************************************************
 *END_COMPARE_ALL_TARBALLS                                                *
 **************************************************************************/

/* @} - end LLcompareGroup */


/**
 * @addtogroup compareGroup
 * @{
 */
/**
 * Compare all data on a CD-R/CD-RW/DVD/ISO/NFS-based backup.
 * @param bkpinfo The backup information structure. Passed to other functions.
 * @return 0 for success, nonzero for failure.
 */
int compare_to_CD()
{
  /** needs malloc *********/
    char *tmp = NULL;
    char *cwd, *new;
    char *command = NULL;
    int resA = 0;
    int resB = 0;
    long noof_changed_files;

    malloc_string(cwd);
    malloc_string(new);

    assert(bkpinfo != NULL);

    getcwd(cwd, MAX_STR_LEN - 1);
    chdir(bkpinfo->restore_path);
    getcwd(new, MAX_STR_LEN - 1);
    insist_on_this_cd_number(g_current_media_number);
    unlink(MONDO_CACHE"/changed.txt");

    resA = compare_all_tarballs();
    resB = compare_all_biggiefiles();
    chdir(cwd);
    noof_changed_files = count_lines_in_file(MONDO_CACHE"/changed.txt");
    if (noof_changed_files) {
        mr_asprintf(tmp, "%ld files do not match the backup            ", noof_changed_files);
        log_to_screen(tmp);
        mr_free(tmp);

        mr_asprintf(command, "cat "MONDO_CACHE"/changed.txt >> %s", MONDO_LOGFILE);
        paranoid_system(command);
        mr_free(command);
    } else {
        mr_asprintf(tmp, "All files match the backup                     ");
        mvaddstr_and_log_it(g_currentY++, 0, tmp);
        log_to_screen(tmp);
        mr_free(tmp);
    }

    paranoid_free(cwd);
    paranoid_free(new);

    return (resA + resB);
}

/**************************************************************************
 *END_COMPARE_TO_CD                                                       *
 **************************************************************************/




/**
 * Compare all data in the user's backup.
 * This function will mount filesystems, compare afioballs and biggiefiles,
 * and show the user the differences.
 * @param bkpinfo The backup information structure. Passed to other functions.
 * @param mountlist The mountlist containing partitions to mount.
 * @param raidlist The raidlist containing the user's RAID devices.
 * @return The number of errors/differences found.
 */
int
compare_mode(struct mountlist_itself *mountlist,
             struct raidlist_itself *raidlist)
{
    int retval = 0;
    int res = 0;
    long q;
    char *tmp = NULL;
    char *new;
    char *cwd;

    malloc_string(new);
    malloc_string(cwd);

  /**************************************************************************
   * also deletes tmp/filelist.full & tmp/biggielist.txt _and_ tries to     *
   * restore them from start of tape, if available                          *
   **************************************************************************/
    assert(bkpinfo != NULL);
    assert(mountlist != NULL);
    assert(raidlist != NULL);

    while (get_cfg_file_from_archive()) {
        if (!ask_me_yes_or_no
            ("Failed to find config file/archives. Choose another source?"))
        {
            fatal_error("Unable to find config file/archives. Aborting.");
        }
        interactively_obtain_media_parameters_from_user(FALSE);
    }

    read_cfg_file_into_bkpinfo(g_mondo_cfg_file);

    /* edit_mountlist if wanted */
    log_it("About to edit mountlist");
    if (g_text_mode) {
        save_mountlist_to_disk(mountlist, g_mountlist_fname);
        mr_asprintf(tmp, "%s %s", find_my_editor(), g_mountlist_fname);
        res = system(tmp);
        mr_free(tmp);

        load_mountlist(mountlist, g_mountlist_fname);
    } else {
        res = edit_mountlist(g_mountlist_fname, mountlist, raidlist);
    }
    log_it("Finished editing mountlist");
    if (res) {
        paranoid_MR_finish(1);
    }
    save_mountlist_to_disk(mountlist, g_mountlist_fname);
    save_raidlist_to_raidtab(raidlist, RAIDTAB_FNAME);

    g_current_media_number = 1;
    mvaddstr_and_log_it(1, 30, "Comparing Automatically");
    log_it("Pre-MAD");
    retval = mount_all_devices(mountlist, FALSE);
    log_it("Post-MAD");
    if (retval) {
        unmount_all_devices(mountlist);
        return (retval);
    }
    if (bkpinfo->backup_media_type == tape
        || bkpinfo->backup_media_type == udev) {
        retval += compare_to_tape();
    } else if (bkpinfo->backup_media_type == cdstream) {
        retval += compare_to_cdstream();
    } else {
        retval += compare_to_CD();
    }
    if (retval) {
        mvaddstr_and_log_it(g_currentY++,
                            0,
                            "Warning - differences found during the compare phase");
    }

    if (count_lines_in_file(MONDO_CACHE"/changed.txt") > 0) {
        mvaddstr_and_log_it(g_currentY++, 0,
                            "Differences found while files were being compared.");
        streamline_changes_file(MONDO_CACHE"/changed.files", MONDO_CACHE"/changed.txt");
        if (count_lines_in_file(MONDO_CACHE"/changed.files") <= 0) {
            mvaddstr_and_log_it(g_currentY++, 0,
                                "...but they were logfiles and temporary files. Your archives are fine.");
            log_to_screen
                ("The differences were logfiles and temporary files. Your archives are fine.");
        } else {
            q = count_lines_in_file(MONDO_CACHE"/changed.files");
            mr_asprintf(tmp, "%ld significant difference%s found.", q, (q != 1) ? "s" : "");
            mvaddstr_and_log_it(g_currentY++, 0, tmp);
            log_to_screen(tmp);
            mr_free(tmp);

            mr_asprintf(tmp, "Type 'less /tmp/changed.files' for a list of non-matching files");
            mvaddstr_and_log_it(g_currentY++, 0, tmp);
            log_to_screen(tmp);
            mr_free(tmp);

            log_msg(2, "calling popup_changelist_from_file()");
            getcwd(cwd, MAX_STR_LEN - 1);
            chdir(bkpinfo->restore_path);
            getcwd(new, MAX_STR_LEN - 1);
            popup_changelist_from_file(MONDO_CACHE"/changed.files");
            chdir(cwd);
            log_msg(2, "Returning from popup_changelist_from_file()");
        }
    } else {
        log_to_screen
            ("No significant differences were found. Your backup is perfect.");
    }
    retval += unmount_all_devices(mountlist);

    kill_petris();
    paranoid_free(new);
    paranoid_free(cwd);
    return (retval);
}

/**************************************************************************
 *END_COMPARE_MODE                                                        *
 **************************************************************************/

/**
 * Compare all data on a cdstream-based backup.
 * @param bkpinfo The backup information structure. Fields used:
 * - @c bkpinfo->disaster_recovery
 * - @c bkpinfo->media_device
 * - @c bkpinfo->restore_path
 * @return 0 for success, nonzero for failure.
 */
int compare_to_cdstream()
{
    int res;

  /** needs malloc **/
    char *dir;
    char *command = NULL;

    assert(bkpinfo != NULL);
    malloc_string(dir);
    getcwd(dir, MAX_STR_LEN);
    chdir(bkpinfo->restore_path);

    mr_asprintf(command, "cp -f /tmp/LAST-FILELIST-NUMBER %s/tmp", bkpinfo->restore_path);
    run_program_and_log_output(command, FALSE);
    mr_free(command);

    mvaddstr_and_log_it(g_currentY,
                        0, "Verifying archives against filesystem");

    mr_free(bkpinfo->media_device);
    if (bkpinfo->disaster_recovery && does_file_exist("/tmp/CDROM-LIVES-HERE")) {
        bkpinfo->media_device = last_line_of_file("/tmp/CDROM-LIVES-HERE");
    } else {
        bkpinfo->media_device = find_cdrom_device(FALSE);
    }
    res = verify_tape_backups();
    chdir(dir);
    if (length_of_file(MONDO_CACHE"/changed.txt") > 2
        && length_of_file(MONDO_CACHE"/changed.files") > 2) {
        log_msg(0,
                "Type 'less "MONDO_CACHE"/changed.files' to see which files don't match the archives");
        log_msg(2, "Calling popup_changelist_from_file()");
        popup_changelist_from_file(MONDO_CACHE"/changed.files");
        log_msg(2, "Returned from popup_changelist_from_file()");
    }

    mvaddstr_and_log_it(g_currentY++, 74, "Done.");
    paranoid_free(dir);
    return (res);
}

/**************************************************************************
 *END_COMPARE_CD_STREAM                                                   *
 **************************************************************************/


/**
 * Compare all data on a tape-based backup.
 * @param bkpinfo The backup information structure. Field used: @c bkpinfo->restore_path.
 * @return 0 for success, nonzero for failure.
 */
/**************************************************************************
 * F@COMPARE_TO_TAPE()                                                    *
 * compare_to_tape() -  gots me??                                         *
 *                                                                        *
 * returns: int                                                           *
 **************************************************************************/
int compare_to_tape()
{
    int res;
    char *dir;
    char *command = NULL;

    assert(bkpinfo != NULL);
    malloc_string(dir);

    getcwd(dir, MAX_STR_LEN);
    chdir(bkpinfo->restore_path);
    mr_asprintf(command, "cp -f /tmp/LAST-FILELIST-NUMBER %s/tmp", bkpinfo->restore_path);
    run_program_and_log_output(command, FALSE);
    mr_free(command);

    mvaddstr_and_log_it(g_currentY,
                        0, "Verifying archives against filesystem");
    res = verify_tape_backups();
    chdir(dir);
    if (res) {
        mvaddstr_and_log_it(g_currentY++, 74, "Failed.");
    } else {
        mvaddstr_and_log_it(g_currentY++, 74, "Done.");
    }
    paranoid_free(dir);
    return (res);
}

/**************************************************************************
 *END_COMPARE_TO_TAPE                                                     *
 **************************************************************************/

/* @} - end compareGroup */