From 13a9e0012d0935850b114f132b5be3764c189880 Mon Sep 17 00:00:00 2001 From: Joseph Benden Date: Sat, 22 Apr 2017 10:54:09 -0700 Subject: [PATCH] Improve execution speed on Windows Improves speed of execution under Windows by utilizing the Win32 APIs for directory traversals and path information queries (stat/lstat replacement.) --- configure.ac | 21 ++++++++++ flist.c | 133 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- mkproto.pl | 2 +- rsync.h | 15 +++++++ syscall.c | 79 +++++++++++++++++++++++++++++++++++ util.c | 88 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 334 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index e01e124..20aba15 100644 --- a/configure.ac +++ b/configure.ac @@ -336,6 +336,27 @@ case $host_os in * ) AC_MSG_RESULT(no);; esac +AC_ARG_WITH(w32api-include-path, + AS_HELP_STRING([--with-w32api-include-path=PATH],[set location of w32api headers (default: /usr/include/w32api)]), + [ W32API_INCLUDE_PATH="$with_w32api_include_path" ], + [ W32API_INCLUDE_PATH="/usr/include/w32api" ]) + +AC_CHECK_SIZEOF([void*]) + +AC_MSG_CHECKING([whether platform should use Win32 API]) +case $host_os in + *cygwin*|*msys* ) AC_MSG_RESULT(yes) + AS_IF([test "$ac_cv_sizeof_voidp" -eq 4], + [AC_DEFINE(_WIN32, 1, + [Define to 1 if Win32 API should be used])], + [AC_DEFINE(_WIN64, 1, + [Define to 1 if Win64 API should be used])]) + dnl Include W32API Headers for Cygwin and Msys + CFLAGS="-I${W32API_INCLUDE_PATH} $CFLAGS" + ;; + * ) AC_MSG_RESULT(no);; +esac + AC_C_BIGENDIAN AC_HEADER_DIRENT AC_HEADER_TIME diff --git a/flist.c b/flist.c index 28553fc..c4e07e6 100644 --- a/flist.c +++ b/flist.c @@ -1687,16 +1687,90 @@ static void interpret_stat_error(const char *fname, int is_dir) static void send_directory(int f, struct file_list *flist, char *fbuf, int len, int flags) { - struct dirent *di; +#ifdef _IS_WINDOWS + WIN32_FIND_DATAA ffd; + LARGE_INTEGER filesize; + char szDir[MAX_PATH]; + unsigned szDir_len; + HANDLE hFind = INVALID_HANDLE_VALUE; + DWORD dwError = 0; + char *szFname = (char*) fbuf; +#ifdef __CYGWIN__ + char *winpath = cygwin_create_path(CCP_POSIX_TO_WIN_A, fbuf); + if (!winpath) { + errno = ENOMEM; + return; + } + szFname = winpath; +#endif +#else + struct dirent *di = NULL; + DIR *d = NULL; +#endif unsigned remainder; - char *p; - DIR *d; + char *p = NULL; int divert_dirs = (flags & FLAG_DIVERT_DIRS) != 0; int start = flist->used; int filter_level = f == -2 ? SERVER_FILTERS : ALL_FILTERS; assert(flist != NULL); +#ifdef _IS_WINDOWS + // subtract an additional two character for the later strlcat! + if ((szDir_len = strlcpy(szDir, szFname, MAX_PATH - _WIN_PATH_SIZE_INTERNAL - 2)) >= MAX_PATH - _WIN_PATH_SIZE_INTERNAL - 2) { + io_error |= IOERR_GENERAL; + rprintf(FERROR_XFER, + "filename overflows max-path len by %u: %s\n", + szDir_len - MAX_PATH - _WIN_PATH_SIZE_INTERNAL - 2 + 1, szFname); + errno = EOVERFLOW; +#ifdef __CYGWIN__ + free(winpath); +#endif + return; + } + + if ((szDir_len = strlcat(szDir, "\\*", MAX_PATH - _WIN_PATH_SIZE_INTERNAL)) >= MAX_PATH - _WIN_PATH_SIZE_INTERNAL) { + io_error |= IOERR_GENERAL; + rprintf(FERROR_XFER, + "filename overflows max-path len by %u: %s\n", + szDir_len - MAX_PATH - _WIN_PATH_SIZE_INTERNAL + 1, szDir); + errno = EOVERFLOW; +#ifdef __CYGWIN__ + free(winpath); +#endif + return; + } + + if (DEBUG_GTE(TIME, 3)) { + rprintf(FINFO, + "[debug] FindFirstFileA on '%s' from '%s'; current error code %d\n", + szDir, + fbuf, + GetLastError()); + } + +#ifdef __CYGWIN__ + free(winpath); +#endif + + SetLastError(ERROR_SUCCESS); + + if ((hFind = FindFirstFileA(szDir, &ffd)) == INVALID_HANDLE_VALUE) { + if (GetLastError() == ERROR_FILE_NOT_FOUND) { + errno = ENOENT; + if (am_sender) /* Can abuse this for vanished error w/ENOENT: */ + interpret_stat_error(fbuf, True); + return; + } + errno = EINVAL; + io_error |= IOERR_GENERAL; + rprintf(FERROR_XFER, + "unable to open '%s'; FindFirstFileA failed with error code %d\n", + szDir, + GetLastError()); + return; + } +#else if (!(d = opendir(fbuf))) { if (errno == ENOENT) { if (am_sender) /* Can abuse this for vanished error w/ENOENT: */ @@ -1707,6 +1781,7 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len, rsyserr(FERROR_XFER, errno, "opendir %s failed", full_fname(fbuf)); return; } +#endif p = fbuf + len; if (len == 1 && *fbuf == '/') @@ -1718,9 +1793,39 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len, } else remainder = 0; +#ifdef _IS_WINDOWS + for (errno = 0, dwError = GetLastError(); dwError == ERROR_SUCCESS; dwError = (FindNextFileA(hFind, &ffd) != 0 ? ERROR_SUCCESS : GetLastError())) { +#else for (errno = 0, di = readdir(d); di; errno = 0, di = readdir(d)) { +#endif unsigned name_len; +#ifdef _IS_WINDOWS + STRUCT_STAT sst; + sst.st_uid = 0; + sst.st_gid = 0; + filesize.LowPart = ffd.nFileSizeLow; + filesize.HighPart = ffd.nFileSizeHigh; + sst.st_size = filesize.QuadPart; + sst.st_blocks = (sst.st_size / 512ULL) + 1ULL; + sst.st_blksize = 4096; + sst.st_rdev = 0; + sst.st_nlink = 0; + sst.st_dev = 0; + sst.st_ino = 0; + sst.st_mode = ((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) ? S_IFDIR : S_IFREG; + sst.st_mode = ((ffd.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0) ? S_IFLNK : sst.st_mode; + sst.st_mode += (S_IRWXU | S_IRWXG | S_IRWXO); + sst.st_atime = win32_filetime_to_epoch(&ffd.ftLastAccessTime); + sst.st_ctime = win32_filetime_to_epoch(&ffd.ftCreationTime); + sst.st_mtime = win32_filetime_to_epoch(&ffd.ftLastWriteTime); + char *dname = ffd.cFileName; + + if (DEBUG_GTE(TIME, 1)) { + rprintf(FINFO, "path %s with mode 0x%x and mtime %ld (dir: %d)\n", dname, sst.st_mode, sst.st_mtime, ((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) > 0) ? 1 : 0); + } +#else char *dname = d_name(di); +#endif if (dname[0] == '.' && (dname[1] == '\0' || (dname[1] == '.' && dname[2] == '\0'))) continue; @@ -1743,17 +1848,39 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len, continue; } +#ifdef _IS_WINDOWS + send_file_name(f, flist, fbuf, &sst, flags, filter_level); +#else send_file_name(f, flist, fbuf, NULL, flags, filter_level); +#endif } fbuf[len] = '\0'; +#ifdef _IS_WINDOWS + if (DEBUG_GTE(TIME, 3)) { + rprintf(FINFO, + "[debug] FindFirstFileA loop finished with error code %d inside '%s'\n", + GetLastError(), + full_fname(fbuf)); + } + + if (dwError != 0 && dwError != ERROR_NO_MORE_FILES) { + io_error |= IOERR_GENERAL; + rsyserr(FERROR_XFER, dwError, "FindNextFileA(%s)", full_fname(fbuf)); + } +#else if (errno) { io_error |= IOERR_GENERAL; rsyserr(FERROR_XFER, errno, "readdir(%s)", full_fname(fbuf)); } +#endif +#ifdef _IS_WINDOWS + FindClose(hFind); +#else closedir(d); +#endif if (f >= 0 && recurse && !divert_dirs) { int i, end = flist->used - 1; diff --git a/mkproto.pl b/mkproto.pl index cdeb2ea..e3d65be 100644 --- a/mkproto.pl +++ b/mkproto.pl @@ -28,7 +28,7 @@ while (<>) { $func = $3; $arg = $1 eq 'LOCAL' ? 'int module_id' : 'void'; $protos .= "$ret$func($arg);\n"; - } elsif (/^static|^extern/ || /[;]/ || !/^[A-Za-z][A-Za-z0-9_]* /) { + } elsif (/^static|^extern/ || /[;]/ || /win32/ || !/^[A-Za-z][A-Za-z0-9_]* /) { ; } elsif (/[(].*[)][ \t]*$/) { s/$/;/; diff --git a/rsync.h b/rsync.h index 1720293..c527944 100644 --- a/rsync.h +++ b/rsync.h @@ -1007,6 +1007,21 @@ typedef struct { #define ACL_READY(sx) ((sx).acc_acl != NULL) #define XATTR_READY(sx) ((sx).xattr != NULL) +#if defined(_WIN32) || defined(_WIN64) +#define _IS_WINDOWS +#define _WIN_FILETIME_TO_UTC_EPOCH_DIVISOR 10000000ULL +#define _WIN_SECONDS_TO_UNIX_EPOCH 11644473600ULL +#define _WIN_PATH_SIZE_INTERNAL 12 + +#include +#ifdef __CYGWIN__ +#include +#endif + +/* Forward Declarations */ +extern time_t win32_filetime_to_epoch(const FILETIME *ft); +#endif + #include "proto.h" #ifndef SUPPORT_XATTRS diff --git a/syscall.c b/syscall.c index d89eacd..4394768 100644 --- a/syscall.c +++ b/syscall.c @@ -317,15 +317,93 @@ int do_mkstemp(char *template, mode_t perms) int do_stat(const char *fname, STRUCT_STAT *st) { +#ifdef _IS_WINDOWS + ULARGE_INTEGER filesize; + WIN32_FILE_ATTRIBUTE_DATA fad; + char *szFname = (char*) fname; +#ifdef __CYGWIN__ + char *winpath = cygwin_create_path(CCP_POSIX_TO_WIN_A, fname); + if (!winpath) { + errno = ENOMEM; + return -1; + } + szFname = winpath; +#endif + + if (DEBUG_GTE(TIME, 1)) { + rprintf(FINFO, "do_stat on '%s'\n", fname); + } + + if (GetFileAttributesExA(szFname, GetFileExInfoStandard, &fad) == 0) { + if (DEBUG_GTE(TIME, 1)) { + rprintf(FINFO, "do_stat on '%s' errored with error code %d\n", fname, GetLastError()); + } + switch (GetLastError()) { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + errno = ENOENT; + break; + case ERROR_ACCESS_DENIED: + case ERROR_NETWORK_ACCESS_DENIED: + errno = EACCES; + break; + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + case ERROR_TOO_MANY_OPEN_FILES: + case ERROR_OUTOFMEMORY: + errno = ENOMEM; + break; + case ERROR_BAD_LENGTH: + errno = ENAMETOOLONG; + break; + case ERROR_INVALID_PARAMETER: + errno = EFAULT; + break; + default: + errno = EINVAL; + } +#ifdef __CYGWIN__ + free(winpath); +#endif + return -1; + } +#ifdef __CYGWIN__ + free(winpath); +#endif + + st->st_uid = 0; + st->st_gid = 0; + filesize.LowPart = fad.nFileSizeLow; + filesize.HighPart = fad.nFileSizeHigh; + st->st_size = filesize.QuadPart; + st->st_blocks = (st->st_size / 512ULL) + 1ULL; + st->st_blksize = 4096; + st->st_rdev = 0; + st->st_nlink = 0; + st->st_dev = 0; + st->st_ino = 0; + st->st_mode = ((fad.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) > 0) ? S_IFDIR : S_IFREG; + st->st_mode += (S_IRWXU | S_IRWXG | S_IRWXO); + st->st_atime = win32_filetime_to_epoch(&fad.ftLastAccessTime); + st->st_ctime = win32_filetime_to_epoch(&fad.ftCreationTime); + st->st_mtime = win32_filetime_to_epoch(&fad.ftLastWriteTime); + + return 0; +#else #ifdef USE_STAT64_FUNCS return stat64(fname, st); #else return stat(fname, st); #endif +#endif } int do_lstat(const char *fname, STRUCT_STAT *st) { +#ifdef _IS_WINDOWS1 +return do_stat(fname, st); +#else #ifdef SUPPORT_LINKS # ifdef USE_STAT64_FUNCS return lstat64(fname, st); @@ -335,6 +413,7 @@ int do_lstat(const char *fname, STRUCT_STAT *st) #else return do_stat(fname, st); #endif +#endif } int do_fstat(int fd, STRUCT_STAT *st) diff --git a/util.c b/util.c index 49c5b71..df3eed7 100644 --- a/util.c +++ b/util.c @@ -115,11 +115,36 @@ void print_child_argv(const char *prefix, char **cmd) rprintf(FCLIENT, " (%d args)\n", cnt); } +#ifdef _IS_WINDOWS +time_t win32_filetime_to_epoch(const FILETIME *ft) +{ + ULARGE_INTEGER liFileTime; + ULONGLONG llSeconds; + time_t retval; + + if (!ft) return (time_t) -1; + + liFileTime.LowPart = ft->dwLowDateTime; + liFileTime.HighPart = ft->dwHighDateTime; + llSeconds = ((ULONGLONG) liFileTime.QuadPart / (ULONGLONG) _WIN_FILETIME_TO_UTC_EPOCH_DIVISOR - _WIN_SECONDS_TO_UNIX_EPOCH); + retval = (time_t) llSeconds; + + if (llSeconds != (ULONGLONG) retval) { + // value exceed POSIX epoch time, fail + return (time_t) -1; + } + + return retval; +} +#endif + /* This returns 0 for success, 1 for a symlink if symlink time-setting * is not possible, or -1 for any other error. */ int set_modtime(const char *fname, time_t modtime, uint32 mod_nsec, mode_t mode) { +#ifndef _IS_WINDOWS static int switch_step = 0; +#endif if (DEBUG_GTE(TIME, 1)) { rprintf(FINFO, "set modtime of %s to (%ld) %s", @@ -127,6 +152,68 @@ int set_modtime(const char *fname, time_t modtime, uint32 mod_nsec, mode_t mode) asctime(localtime(&modtime))); } +#ifdef _IS_WINDOWS + HANDLE hFile; + ULARGE_INTEGER uliModtime; + FILETIME ftModtime; + DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL; + char *szFname = (char*) fname; +#ifdef __CYGWIN__ + char *winpath = cygwin_create_path(CCP_POSIX_TO_WIN_A, fname); + if (!winpath) { + errno = ENOMEM; + return -1; + } + szFname = winpath; +#endif + + if (S_ISDIR(mode)) { + dwFlagsAndAttributes = FILE_FLAG_BACKUP_SEMANTICS; + } + + // convert POSIX epoch time to win32 FILETIME + uliModtime.QuadPart = (((ULONGLONG) modtime + (ULONGLONG) _WIN_SECONDS_TO_UNIX_EPOCH) * (ULONGLONG) _WIN_FILETIME_TO_UTC_EPOCH_DIVISOR); + ftModtime.dwLowDateTime = uliModtime.LowPart; + ftModtime.dwHighDateTime = uliModtime.HighPart; + + hFile = CreateFileA(szFname, // file to open + FILE_WRITE_ATTRIBUTES, // file opts + FILE_SHARE_WRITE, // share opts + NULL, //default security + OPEN_EXISTING, // existing file only + dwFlagsAndAttributes, // normal file + NULL); // no attr. template + + if (hFile == INVALID_HANDLE_VALUE) { + // error opening.... + rprintf(FERROR, "failed to open inside of set modtime of %s to (%ld) %s: error code %d\n", + szFname, (long)modtime, + asctime(localtime(&modtime)), + GetLastError()); +#ifdef __CYGWIN__ + free(winpath); +#endif + return -1; + } + + if (SetFileTime(hFile, NULL, NULL, &ftModtime) == 0) { + // error setting + rprintf(FERROR, "failed to set modtime of %s to (%ld) %s: error code %d\n", + szFname, (long)modtime, + asctime(localtime(&modtime)), + GetLastError()); + CloseHandle(hFile); +#ifdef __CYGWIN__ + free(winpath); +#endif + return -1; + } + + CloseHandle(hFile); +#ifdef __CYGWIN__ + free(winpath); +#endif +#else switch (switch_step) { #ifdef HAVE_UTIMENSAT #include "case_N.h" @@ -168,6 +255,7 @@ int set_modtime(const char *fname, time_t modtime, uint32 mod_nsec, mode_t mode) return -1; } +#endif return 0; } -- 2.10.2.windows.1