--- util.c.orig 2015-12-21 19:54:02.000000000 +0100 +++ util.c 2017-10-10 23:34:49.000000000 +0200 @@ -1186,7 +1186,7 @@ } } else fn = fname; - if ((int)pathjoin(t, sz, partial_dir, fn) >= sz) + if ((int)pathjoin(t, sz, partial_dir, fn) > sz) return NULL; if (daemon_filter_list.head) { t = strrchr(partial_fname, '/'); --- generator.c.orig 2017-10-01 17:10:28.000000000 +0200 +++ generator.c 2017-10-12 15:13:27.000000000 +0200 @@ -682,16 +682,32 @@ * * Generate approximately one checksum every block_len bytes. */ -static int generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy) +static int generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy, OFF_T total_size, int fd_cmplnk, OFF_T len_cmplnk) { int32 i; + int32 pad_sums = 0; struct map_struct *mapbuf; struct sum_struct sum; OFF_T offset = 0; - sum_sizes_sqroot(&sum, len); + sum_sizes_sqroot(&sum, len + len_cmplnk); + if (len_cmplnk > 0 && (sum.count < 0 || append_mode > 0)) { + len_cmplnk = 0; + sum_sizes_sqroot(&sum, len); + } if (sum.count < 0) return -1; + + /* Let's send some dummy 0 blocks after the first checksumed (partial) file + * so that sender will see the second file's blocks as if they were after + * the end of the final file, thus it will be able to fully use them. + * Useful when inplace is used. */ + if (len_cmplnk > 0 && total_size > len) { + pad_sums = total_size / sum.blength + (total_size % sum.blength != 0) + - (len / sum.blength + (len % sum.blength != 0)); + sum.count += pad_sums; + } + write_sum_head(f_out, &sum); if (append_mode > 0 && f_copy < 0) @@ -717,6 +733,39 @@ continue; } + /* Time to checksum second file */ + if (n1 < sum.blength && len_cmplnk > 0) { + /* Stop copy */ + f_copy = -1; + /* Load the second mapfuf */ + if (mapbuf) + unmap_file(mapbuf); + mapbuf = map_file(fd_cmplnk, len_cmplnk, MAX_MAP_SIZE, sum.blength); + len = len_cmplnk; + len_cmplnk = 0; /* don't enter this if{} anymore */ + /* Don't checksum the last partial block if we have a second partial file + * as it will not have the correct size. + * Send a dummy O block instead. */ + i--; + char sum2[SUM_LENGTH] = {0}; + if (n1 > 0) { + write_int(f_out, 0); + write_buf(f_out, sum2, sum.s2length); + i++; + offset = sum.blength - n1; + } else { + offset = 0; + } + /* Sens the dummy 0 blocks between the 2 files */ + while (pad_sums > 0) { + write_int(f_out, 0); + write_buf(f_out, sum2, sum.s2length); + i++; + pad_sums--; + } + continue; + } + sum1 = get_checksum1(map, n1); get_checksum2(map, n1, sum2); @@ -1186,13 +1235,15 @@ static struct file_list *fuzzy_dirlist[MAX_BASIS_DIRS+1]; static int need_fuzzy_dirlist = 0; struct file_struct *fuzzy_file = NULL; - int fd = -1, f_copy = -1; + int fd = -1, f_copy = -1, fd_cmplnk = -1; stat_x sx, real_sx; STRUCT_STAT partial_st; + STRUCT_STAT cmplnk_st; struct file_struct *back_file = NULL; int statret, real_ret, stat_errno; char *fnamecmp, *partialptr, *backupptr = NULL; char fnamecmpbuf[MAXPATHLEN]; + char fnamecmplnk[MAXPATHLEN]; uchar fnamecmp_type; int del_opts = delete_mode || force_delete ? DEL_RECURSE : 0; int is_dir = !S_ISDIR(file->mode) ? 0 @@ -1713,14 +1764,6 @@ real_sx.st = sx.st; /* Don't copy xattr/acl pointers, as they would free wrong. */ real_ret = statret; - if (partial_dir && (partialptr = partial_dir_fname(fname)) != NULL - && link_stat(partialptr, &partial_st, 0) == 0 - && S_ISREG(partial_st.st_mode)) { - if (statret != 0) - goto prepare_to_open; - } else - partialptr = NULL; - if (statret != 0 && fuzzy_basis) { /* Sets fnamecmp_type to FNAMECMP_FUZZY or above. */ fuzzy_file = find_fuzzy(file, fuzzy_dirlist, &fnamecmp_type); @@ -1736,6 +1779,14 @@ } } + if (partial_dir && (partialptr = partial_dir_fname(fname)) != NULL + && link_stat(partialptr, &partial_st, 0) == 0 + && S_ISREG(partial_st.st_mode)) { + if (statret != 0) + goto prepare_to_open; + } else + partialptr = NULL; + if (statret != 0) { #ifdef SUPPORT_HARD_LINKS if (preserve_hard_links && F_HLINK_NOT_LAST(file)) { @@ -1783,7 +1834,18 @@ } prepare_to_open: + fnamecmplnk[0] = '\0'; + cmplnk_st.st_size = 0; if (partialptr) { + /* We have a partial file, and also a basis file. + Let's then link the basis file so that both generator and + receiver will be able to use it right after the partial one. */ + if (statret == 0) { + strcpy(fnamecmplnk, partialptr); + strcat(fnamecmplnk, "."); + unlink(fnamecmplnk); + symlink(fnamecmp, fnamecmplnk); + } sx.st = partial_st; fnamecmp = partialptr; fnamecmp_type = FNAMECMP_PARTIAL_DIR; @@ -1836,6 +1898,18 @@ statret = real_ret = -1; goto notify_others; } + if (strlen(fnamecmplnk) > 0) { + if ((fd_cmplnk = do_open(fnamecmplnk, O_RDONLY, 0)) < 0) { + rsyserr(FERROR, errno, "generator failed to open second basis %s, continuing", + full_fname(fnamecmplnk)); + } else { + if (do_stat(fnamecmplnk, &cmplnk_st) < 0) { + cmplnk_st.st_size = 0; + rsyserr(FERROR, errno, "generator failed to stat second basis %s, continuing", + full_fname(fnamecmplnk)); + } + } + } if (inplace && make_backups > 0 && fnamecmp_type == FNAMECMP_FNAME) { if (!(backupptr = get_backup_name(fname))) { @@ -1907,11 +1981,11 @@ if (statret != 0 || whole_file) write_sum_head(f_out, NULL); - else if (sx.st.st_size <= 0) { + else if (sx.st.st_size < 0) { write_sum_head(f_out, NULL); close(fd); } else { - if (generate_and_send_sums(fd, sx.st.st_size, f_out, f_copy) < 0) { + if (generate_and_send_sums(fd, sx.st.st_size, f_out, f_copy, F_LENGTH(file), fd_cmplnk, cmplnk_st.st_size) < 0) { rprintf(FWARNING, "WARNING: file is too large for checksum sending: %s\n", fnamecmp); @@ -1921,6 +1995,8 @@ } cleanup: + if (fd_cmplnk != -1) + close(fd_cmplnk); if (back_file) { int save_preserve_xattrs = preserve_xattrs; if (f_copy >= 0) --- receiver.c.orig 2017-10-02 17:05:17.000000000 +0200 +++ receiver.c 2017-10-12 15:02:43.000000000 +0200 @@ -229,16 +229,18 @@ } static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, - const char *fname, int fd, OFF_T total_size) + const char *fname, int fd, OFF_T total_size, char *cmplnk, int fd_cmplnk, OFF_T len_cmplnk) { static char file_sum1[MAX_DIGEST_LEN]; struct map_struct *mapbuf; + struct map_struct *mapbuf_cmplnk; struct sum_struct sum; int32 len; OFF_T offset = 0; OFF_T offset2; char *data; int32 i; + int32 pad_size = 0; char *map = NULL; #ifdef SUPPORT_PREALLOCATION #ifdef PREALLOCATE_NEEDS_TRUNCATE @@ -269,6 +271,22 @@ } else mapbuf = NULL; + if (fd_cmplnk >= 0 && len_cmplnk > 0) { + int32 read_size = MAX(sum.blength * 2, 16*1024); + mapbuf_cmplnk = map_file(fd_cmplnk, len_cmplnk, read_size, sum.blength); + if (DEBUG_GTE(DELTASUM, 2)) { + rprintf(FINFO, "recv mapped %s of size %s\n", + cmplnk, big_num(len_cmplnk)); + } + } else + mapbuf_cmplnk = NULL; + + if (len_cmplnk > 0 && total_size > size_r) { + pad_size = total_size / sum.blength + (total_size % sum.blength != 0) + - (size_r / sum.blength + (size_r % sum.blength != 0)); + pad_size *= sum.blength; + } + sum_init(checksum_seed); if (append_mode > 0) { @@ -338,15 +356,20 @@ updating_basis_or_equiv && offset == offset2 ? " (seek)" : ""); } - if (mapbuf) { + if (mapbuf && offset2 <= size_r) { map = map_ptr(mapbuf,offset2,len); - + see_token(map, len); + sum_update(map, len); + } + else if (mapbuf_cmplnk && offset2 > size_r) { + offset2 -= (size_r + pad_size); + map = map_ptr(mapbuf_cmplnk,offset2,len); see_token(map, len); sum_update(map, len); } if (updating_basis_or_equiv) { - if (offset == offset2 && fd != -1) { + if (offset == offset2 && offset2 < size_r && fd != -1) { OFF_T pos; if (flush_write_file(fd) < 0) goto report_write_error; @@ -398,6 +421,8 @@ if (mapbuf) unmap_file(mapbuf); + if (mapbuf_cmplnk) + unmap_file(mapbuf_cmplnk); read_buf(f_in, sender_file_sum, checksum_len); if (DEBUG_GTE(DELTASUM, 2)) @@ -410,7 +435,7 @@ static void discard_receive_data(int f_in, OFF_T length) { - receive_data(f_in, NULL, -1, 0, NULL, -1, length); + receive_data(f_in, NULL, -1, 0, NULL, -1, length, NULL, -1, 0); } static void handle_delayed_updates(char *local_name) @@ -525,6 +550,9 @@ char fnametmp[MAXPATHLEN]; char *fnamecmp, *partialptr; char fnamecmpbuf[MAXPATHLEN]; + char fnamecmplnk[MAXPATHLEN]; + int fd_cmplnk = -1; + STRUCT_STAT cmplnk_st; uchar fnamecmp_type; struct file_struct *file; int itemizing = am_server ? logfile_format_has_i : stdout_format_has_i; @@ -851,6 +879,22 @@ continue; } + /* open second basis file if it exists */ + fnamecmplnk[0] = '\0'; + cmplnk_st.st_size = 0; + if (fd1 != -1 && partialptr) { + strcpy(fnamecmplnk, partialptr); + strcat(fnamecmplnk, "."); + if ((fd_cmplnk = do_open(fnamecmplnk, O_RDONLY, 0)) >= 0) { + if (do_stat(fnamecmplnk, &cmplnk_st) < 0) { + cmplnk_st.st_size = 0; + rsyserr(FERROR, errno, "receiver failed to stat second basis %s, continuing", + full_fname(fnamecmplnk)); + } + unlink(fnamecmplnk); + } + } + /* log the transfer */ if (log_before_transfer) log_item(FCLIENT, file, iflags, NULL); @@ -859,10 +903,12 @@ /* recv file data */ recv_ok = receive_data(f_in, fnamecmp, fd1, st.st_size, - fname, fd2, F_LENGTH(file)); + fname, fd2, F_LENGTH(file), fnamecmplnk, fd_cmplnk, cmplnk_st.st_size); log_item(log_code, file, iflags, NULL); + if (fd_cmplnk != -1) + close(fd_cmplnk); if (fd1 != -1) close(fd1); if (close(fd2) < 0) {