diff --git a/rsync.c b/rsync.c
index 1f9b029..cf238fe 100644
--- a/rsync.c
+++ b/rsync.c
@@ -489,13 +489,6 @@ int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp,
 		get_acl(fname, sxp);
 #endif
 
-#ifdef SUPPORT_XATTRS
-	if (am_root < 0)
-		set_stat_xattr(fname, file, new_mode);
-	if (preserve_xattrs && fnamecmp)
-		set_xattr(fname, file, fnamecmp, sxp);
-#endif
-
 	if (!preserve_times
 	 || (!(preserve_times & PRESERVE_DIR_TIMES) && S_ISDIR(sxp->st.st_mode))
 	 || (!(preserve_times & PRESERVE_LINK_TIMES) && S_ISLNK(sxp->st.st_mode)))
@@ -561,6 +554,13 @@ int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp,
 		updated = 1;
 	}
 
+#ifdef SUPPORT_XATTRS
+	if (am_root < 0)
+		set_stat_xattr(fname, file, new_mode);
+	if (preserve_xattrs && fnamecmp)
+		set_xattr(fname, file, fnamecmp, sxp);
+#endif
+
 #ifdef SUPPORT_ACLS
 	/* It's OK to call set_acl() now, even for a dir, as the generator
 	 * will enable owner-writability using chmod, if necessary.