From 80c64dc3b3e82c84ea52528722ee2db1567ab4c1 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Fri, 29 Oct 2021 20:06:06 -0700 Subject: [PATCH 001/350] Fix the ability to read the user's numeric locale. --- main.c | 1 + 1 file changed, 1 insertion(+) diff --git a/main.c b/main.c index 361dbc4e5..013a05553 100644 --- a/main.c +++ b/main.c @@ -1761,6 +1761,7 @@ int main(int argc,char *argv[]) #if defined CONFIG_LOCALE && defined HAVE_SETLOCALE setlocale(LC_CTYPE, ""); + setlocale(LC_NUMERIC, ""); #endif if (!parse_arguments(&argc, (const char ***) &argv)) { From 1b9308b727f3b7b6a254ef6c2359f95f0907ff5e Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Sat, 30 Oct 2021 15:58:01 -0700 Subject: [PATCH 002/350] More NEWS changes. --- NEWS.md | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/NEWS.md b/NEWS.md index cfcad9f41..b866e0007 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,6 +4,30 @@ ## Changes in this version: +### OUTPUT CHANGES: + + - A long-standing bug was preventing rsync from figuring out the current + locale's decimal point character, which made rsync always output numbers + using the "C" locale. Since this is now fixed in 3.2.4, a script that + parses rsync's decimal numbers (e.g. from the verbose footer) should be sure + to setup the environment in a way that the output continues to be in the C + locale. For instance, one of the following should work fine: + + ```shell + export LC_ALL=C.UTF-8 + ``` + + or maybe: + + ```shell + if [ "${LC_ALL:-}" ]; then + export LANG="$LC_ALL" + export LC_CTYPE="$LC_ALL" + unset LC_ALL + fi + export LC_NUMERIC=C.UTF-8 + ``` + ### BUG FIXES: - Fixed a bug with `--inplace` + `--sparse` where the destination file could @@ -49,7 +73,7 @@ - Added the `--fsync` option (promoted from the patches repo). - Reduced memory usage for an incremental transfer that has a bunch of small - diretories. + directories. - The rsync daemon can now handle a client address with an implied "%scope" suffix. @@ -84,7 +108,7 @@ - Improved the IPv6 determination in configure. - - Made SIMD & ASM configure default to "no" on non-linux hosts due to various + - Made SIMD & ASM configure default to "no" on non-Linux hosts due to various reports of problems on NetBSD & macOS hosts. These tests were also tweaked to support a host_cpu of amd64 in addition to x86_64. @@ -186,7 +210,7 @@ `hosts deny` daemon parameters. This is a finalized version of the netgroup-auth patch from the patches repo. - - Rsync can now hard-link symlinks on FreeBSD due to it making ues of the + - Rsync can now hard-link symlinks on FreeBSD due to it making use of the linkat() function when it is available. - Output file+line info on out-of-memory & overflow errors while also avoiding @@ -866,7 +890,7 @@ - Fixed a bug in the iconv code when EINVAL or EILSEQ is returned with a full output buffer. - - Fixed some rare bugs in `--iconv` processing that might cause a multibyte + - Fixed some rare bugs in `--iconv` processing that might cause a multi-byte character to get translated incorrectly. - Fixed a bogus `vanished file` error if some files were specified with `./` @@ -2377,7 +2401,7 @@ option, below. - The way rsync escapes unreadable characters has changed. First, rsync now - has support for recognizing valid multibyte character sequences in your + has support for recognizing valid multi-byte character sequences in your current locale, allowing it to escape fewer characters than before for a locale such as UTF-8. Second, it now uses an escape idiom of `\#123`, which is the literal string `\#` followed by exactly 3 octal digits. Rsync no From e4669b81ae0b1e23c4e122022d2507bccc2df5f9 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Wed, 3 Nov 2021 09:35:50 -0700 Subject: [PATCH 003/350] Add the --info=NONREG setting. --- NEWS.md | 5 +++++ backup.c | 3 ++- generator.c | 8 +++++--- options.c | 13 +++++++------ rsync.1.md | 24 +++++++++++++++++++----- rsync.h | 3 ++- 6 files changed, 40 insertions(+), 16 deletions(-) diff --git a/NEWS.md b/NEWS.md index b866e0007..9f9433d20 100644 --- a/NEWS.md +++ b/NEWS.md @@ -90,6 +90,11 @@ - When `--chown`, `--usermap`, or `--groupmap` is used, rsync now implies the appropriate `--owner` and/or `--group` option. + - Added the `--info=NONREG` setting to control if rsync should warn about + non-regular files in the transfer. This is enabled by default (keeping the + behavior the same as before), so specifying `--info=nonreg0` can be used to + turn the warnings off. + - More ASM optimizations from Shark64. - Make rrsync handle the latest options. diff --git a/backup.c b/backup.c index be406bef8..5036c23a4 100644 --- a/backup.c +++ b/backup.c @@ -304,7 +304,8 @@ int make_backup(const char *fname, BOOL prefer_rename) #endif if (!ret && !S_ISREG(file->mode)) { - rprintf(FINFO, "make_bak: skipping non-regular file %s\n", fname); + if (INFO_GTE(NONREG, 1)) + rprintf(FINFO, "make_bak: skipping non-regular file %s\n", fname); unmake_file(file); #ifdef SUPPORT_ACLS uncache_tmp_acls(); diff --git a/generator.c b/generator.c index 4e75ae2e9..6e1cbe916 100644 --- a/generator.c +++ b/generator.c @@ -1678,9 +1678,11 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, } if (ftype != FT_REG) { - if (solo_file) - fname = f_name(file, NULL); - rprintf(FINFO, "skipping non-regular file \"%s\"\n", fname); + if (INFO_GTE(NONREG, 1)) { + if (solo_file) + fname = f_name(file, NULL); + rprintf(FINFO, "skipping non-regular file \"%s\"\n", fname); + } goto cleanup; } diff --git a/options.c b/options.c index 3f0354624..eb5744a12 100644 --- a/options.c +++ b/options.c @@ -230,7 +230,7 @@ static const char *debug_verbosity[] = { #define MAX_VERBOSITY ((int)(sizeof debug_verbosity / sizeof debug_verbosity[0]) - 1) static const char *info_verbosity[1+MAX_VERBOSITY] = { - /*0*/ NULL, + /*0*/ "NONREG", /*1*/ "COPY,DEL,FLIST,MISC,NAME,STATS,SYMSAFE", /*2*/ "BACKUP,MISC2,MOUNT,NAME2,REMOVE,SKIP", }; @@ -268,9 +268,10 @@ static struct output_struct info_words[COUNT_INFO+1] = { INFO_WORD(MISC, W_SND|W_REC, "Mention miscellaneous information (levels 1-2)"), INFO_WORD(MOUNT, W_SND|W_REC, "Mention mounts that were found or skipped"), INFO_WORD(NAME, W_SND|W_REC, "Mention 1) updated file/dir names, 2) unchanged names"), + INFO_WORD(NONREG, W_REC, "Mention skipped non-regular files (default 1, 0 disables)"), INFO_WORD(PROGRESS, W_CLI, "Mention 1) per-file progress or 2) total transfer progress"), INFO_WORD(REMOVE, W_SND, "Mention files removed on the sending side"), - INFO_WORD(SKIP, W_REC, "Mention files that are skipped due to options used (levels 1-2)"), + INFO_WORD(SKIP, W_REC, "Mention files skipped due to transfer overrides (levels 1-2)"), INFO_WORD(STATS, W_CLI|W_SRV, "Mention statistics at end of run (levels 1-3)"), INFO_WORD(SYMSAFE, W_SND|W_REC, "Mention symlinks that are unsafe"), { NULL, "--info", 0, 0, 0, 0 } @@ -488,9 +489,9 @@ static void output_item_help(struct output_struct *words) rprintf(FINFO, fmt, "HELP", "Output this help message"); rprintf(FINFO, "\n"); - rprintf(FINFO, "Options added for each increase in verbose level:\n"); + rprintf(FINFO, "Options added at each level of verbosity:\n"); - for (j = 1; j <= MAX_VERBOSITY; j++) { + for (j = 0; j <= MAX_VERBOSITY; j++) { parse_output_words(words, levels, verbosity[j], HELP_PRIORITY); opt = make_output_option(words, levels, W_CLI|W_SRV|W_SND|W_REC); if (opt) { @@ -509,7 +510,7 @@ static void set_output_verbosity(int level, uchar priority) if (level > MAX_VERBOSITY) level = MAX_VERBOSITY; - for (j = 1; j <= level; j++) { + for (j = 0; j <= level; j++) { parse_output_words(info_words, info_levels, info_verbosity[j], priority); parse_output_words(debug_words, debug_levels, debug_verbosity[j], priority); } @@ -528,7 +529,7 @@ void limit_output_verbosity(int level) memset(debug_limits, 0, sizeof debug_limits); /* Compute the level limits in the above arrays. */ - for (j = 1; j <= level; j++) { + for (j = 0; j <= level; j++) { parse_output_words(info_words, info_limits, info_verbosity[j], LIMIT_PRIORITY); parse_output_words(debug_words, debug_limits, debug_verbosity[j], LIMIT_PRIORITY); } diff --git a/rsync.1.md b/rsync.1.md index d154a98ca..3b463901b 100644 --- a/rsync.1.md +++ b/rsync.1.md @@ -1019,6 +1019,10 @@ your home directory (remove the '=' for that). When symlinks are encountered, recreate the symlink on the destination. + By default, rsync generates a "non-regular file" warning for each symlink + encountered when this option is not set. You can silence the warning by + specifying ``--info=nonreg0``. + 0. `--copy-links`, `-L` When symlinks are encountered, the item that they point to (the referent) @@ -1325,14 +1329,24 @@ your home directory (remove the '=' for that). 0. `--devices` This option causes rsync to transfer character and block device files to - the remote system to recreate these devices. This option has no effect if - the receiving rsync is not run as the super-user (see also the `--super` - and `--fake-super` options). + the remote system to recreate these devices. If the receiving rsync is not + being run as the super-user, rsync silently skips creating the device files + (see also the `--super` and `--fake-super` options). + + By default, rsync generates a "non-regular file" warning for each device + file encountered when this option is not set. You can silence the warning + by specifying ``--info=nonreg0``. 0. `--specials` - This option causes rsync to transfer special files such as named sockets - and fifos. + This option causes rsync to transfer special files, such as named sockets + and fifos. If the receiving rsync is not being run as the super-user, + rsync silently skips creating the special files (see also the `--super` and + `--fake-super` options). + + By default, rsync generates a "non-regular file" warning for each special + file encountered when this option is not set. You can silence the warning + by specifying ``--info=nonreg0``. 0. `-D` diff --git a/rsync.h b/rsync.h index a6c0d1bb3..86105afe3 100644 --- a/rsync.h +++ b/rsync.h @@ -1416,7 +1416,8 @@ extern short info_levels[], debug_levels[]; #define INFO_MISC (INFO_FLIST+1) #define INFO_MOUNT (INFO_MISC+1) #define INFO_NAME (INFO_MOUNT+1) -#define INFO_PROGRESS (INFO_NAME+1) +#define INFO_NONREG (INFO_NAME+1) +#define INFO_PROGRESS (INFO_NONREG+1) #define INFO_REMOVE (INFO_PROGRESS+1) #define INFO_SKIP (INFO_REMOVE+1) #define INFO_STATS (INFO_SKIP+1) From ca538965d81290ebd514397916594bdb2857e378 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Sun, 7 Nov 2021 10:11:12 -0800 Subject: [PATCH 004/350] Add closing backticks that Itzoke pointed out. --- rsync.1.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rsync.1.md b/rsync.1.md index 3b463901b..da2271712 100644 --- a/rsync.1.md +++ b/rsync.1.md @@ -3237,7 +3237,7 @@ your home directory (remove the '=' for that). buffered, while other can show up as very slow when the flushing of the output buffer occurs. This may be fixed in a future version. -0. `--stop-after=MINS +0. `--stop-after=MINS` This option tells rsync to stop copying when the specified number of minutes has elapsed. @@ -3250,7 +3250,7 @@ your home directory (remove the '=' for that). of the connection supports it. You can tell the remote side about the time limit using `--remote-option` (`-M`), should the need arise. -0. `--stop-at=y-m-dTh:m +0. `--stop-at=y-m-dTh:m` This option tells rsync to stop copying when the specified point in time has been reached. The date & time can be fully specified in a numeric From 8f383e8987482ce8194f40d0486dac6f95d83033 Mon Sep 17 00:00:00 2001 From: Issam Maghni Date: Sun, 7 Nov 2021 13:23:01 -0500 Subject: [PATCH 005/350] shell: test -a|o is not POSIX (#250) --- Makefile.in | 2 +- configure.ac | 8 ++++---- install-sh | 2 +- packaging/prep-auto-dir | 2 +- prepare-source | 2 +- runtests.sh | 8 ++++---- testsuite/chmod-temp-dir.test | 2 +- testsuite/rsync.fns | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Makefile.in b/Makefile.in index c4c00e964..3c8c22405 100644 --- a/Makefile.in +++ b/Makefile.in @@ -200,7 +200,7 @@ configure.sh config.h.in: configure.ac aclocal.m4 else \ echo "config.h.in has CHANGED."; \ fi - @if test -f configure.sh.old -o -f config.h.in.old; then \ + @if test -f configure.sh.old || test -f config.h.in.old; then \ if test "$(MAKECMDGOALS)" = reconfigure; then \ echo 'Continuing with "make reconfigure".'; \ else \ diff --git a/configure.ac b/configure.ac index d80194ee4..fbdd17d83 100644 --- a/configure.ac +++ b/configure.ac @@ -262,7 +262,7 @@ fi if test x"$enable_simd" != x"no"; then # For x86-64 SIMD, g++ >=5 or clang++ >=7 is required - if test x"$host_cpu" = x"x86_64" -o x"$host_cpu" = x"amd64"; then + if test x"$host_cpu" = x"x86_64" || test x"$host_cpu" = x"amd64"; then AC_LANG(C++) if test x"$host_cpu" = x"$build_cpu"; then AC_RUN_IFELSE([AC_LANG_PROGRAM([SIMD_X86_64_TEST],[[if (test_ssse3(42) != 42 || test_sse2(42) != 42 || test_avx2(42) != 42) exit(1);]])], @@ -326,7 +326,7 @@ if test x"$enable_asm" = x""; then fi if test x"$enable_asm" != x"no"; then - if test x"$host_cpu" = x"x86_64" -o x"$host_cpu" = x"amd64"; then + if test x"$host_cpu" = x"x86_64" || test x"$host_cpu" = x"amd64"; then ASM="$host_cpu" elif test x"$enable_asm" = x"yes"; then AC_MSG_RESULT(unavailable) @@ -712,7 +712,7 @@ yes #endif], rsync_cv_HAVE_GETADDR_DEFINES=yes, rsync_cv_HAVE_GETADDR_DEFINES=no)]) -AS_IF([test x"$rsync_cv_HAVE_GETADDR_DEFINES" = x"yes" -a x"$ac_cv_type_struct_addrinfo" = x"yes"],[ +AS_IF([test x"$rsync_cv_HAVE_GETADDR_DEFINES" = x"yes" && test x"$ac_cv_type_struct_addrinfo" = x"yes"],[ # Tru64 UNIX has getaddrinfo() but has it renamed in libc as # something else so we must include to get the # redefinition. @@ -1384,7 +1384,7 @@ else esac fi -if test x"$enable_acl_support" = x"no" -o x"$enable_xattr_support" = x"no" -o x"$enable_iconv" = x"no"; then +if test x"$enable_acl_support" = x"no" || test x"$enable_xattr_support" = x"no" || test x"$enable_iconv" = x"no"; then AC_MSG_CHECKING([whether $CC supports -Wno-unused-parameter]) OLD_CFLAGS="$CFLAGS" CFLAGS="$CFLAGS -Wno-unused-parameter" diff --git a/install-sh b/install-sh index 956817d4b..8c409fbb9 100755 --- a/install-sh +++ b/install-sh @@ -115,7 +115,7 @@ else # might cause directories to be created, which would be especially bad # if $src (and thus $dsttmp) contains '*'. - if [ -f $src -o -d $src ] + if [ -f $src ] || [ -d $src ] then true else diff --git a/packaging/prep-auto-dir b/packaging/prep-auto-dir index 1e5a29656..b67f390ab 100755 --- a/packaging/prep-auto-dir +++ b/packaging/prep-auto-dir @@ -13,7 +13,7 @@ # run "make distclean" before creating the auto-build-save dir. auto_top='auto-build-save' -if test -d $auto_top -a -d .git; then +if test -d $auto_top && test -d .git; then desired_branch=`git rev-parse --abbrev-ref HEAD | tr / %` if test "$desired_branch" = HEAD; then echo "ERROR: switch to the right build dir manually when in detached HEAD mode." 1>&2 diff --git a/prepare-source b/prepare-source index f5b7b46ca..5c56efadb 100755 --- a/prepare-source +++ b/prepare-source @@ -32,7 +32,7 @@ if test "$dir" != '.'; then fi done for fn in configure.sh config.h.in aclocal.m4; do - test ! -f $fn -a -f "$dir/$fn" && cp -p "$dir/$fn" $fn + test ! -f $fn && test -f "$dir/$fn" && cp -p "$dir/$fn" $fn done fi diff --git a/runtests.sh b/runtests.sh index 38f814d22..8c573041d 100755 --- a/runtests.sh +++ b/runtests.sh @@ -155,7 +155,7 @@ if test x"$TOOLDIR" = x; then TOOLDIR=`pwd` fi srcdir=`dirname $0` -if test x"$srcdir" = x -o x"$srcdir" = x.; then +if test x"$srcdir" = x || test x"$srcdir" = x.; then srcdir="$TOOLDIR" fi if test x"$rsync_bin" = x; then @@ -288,7 +288,7 @@ for testscript in $suitedir/$whichtests; do result=$? set -e - if [ "x$always_log" = xyes -o \( $result != 0 -a $result != 77 -a $result != 78 \) ] + if [ "x$always_log" = xyes ] || ( [ $result != 0 ] && [ $result != 77 ] && [ $result != 78 ] ) then echo "----- $testbase log follows" cat "$scratchdir/test.log" @@ -336,7 +336,7 @@ echo " $passed passed" [ "$failed" -gt 0 ] && echo " $failed failed" [ "$skipped" -gt 0 ] && echo " $skipped skipped" [ "$missing" -gt 0 ] && echo " $missing missing" -if [ "$full_run" = yes -a "$expect_skipped" != IGNORE ]; then +if [ "$full_run" = yes ] && [ "$expect_skipped" != IGNORE ]; then skipped_list=`echo "$skipped_list" | sed 's/^,//'` echo "----- skipped results:" echo " expected: $expect_skipped" @@ -353,7 +353,7 @@ echo '------------------------------------------------------------' # because -e is set. result=`expr $failed + $missing || true` -if [ "$result" = 0 -a "$skipped_list" != "$expect_skipped" ]; then +if [ "$result" = 0 ] && [ "$skipped_list" != "$expect_skipped" ]; then result=1 fi echo "overall result is $result" diff --git a/testsuite/chmod-temp-dir.test b/testsuite/chmod-temp-dir.test index e5541e4c9..22b2df816 100644 --- a/testsuite/chmod-temp-dir.test +++ b/testsuite/chmod-temp-dir.test @@ -17,7 +17,7 @@ sdev=`$TOOLDIR/getfsdev $scratchdir` tdev=$sdev for tmpdir2 in "${RSYNC_TEST_TMP:-/override-tmp-not-specified}" /run/shm /var/tmp /tmp; do - [ -d "$tmpdir2" -a -w "$tmpdir2" ] || continue + [ -d "$tmpdir2" ] && [ -w "$tmpdir2" ] || continue tdev=`$TOOLDIR/getfsdev "$tmpdir2"` [ x$sdev != x$tdev ] && break done diff --git a/testsuite/rsync.fns b/testsuite/rsync.fns index 1e2b399f0..2ab97b69c 100644 --- a/testsuite/rsync.fns +++ b/testsuite/rsync.fns @@ -65,7 +65,7 @@ set_cp_destdir() { # even if the copy rounded microsecond times on the destination file. cp_touch() { cp_p "${@}" - if test $# -gt 2 -o -d "$2"; then + if test $# -gt 2 || test -d "$2"; then set_cp_destdir "${@}" # sets destdir var while test $# -gt 1; do destname="$destdir/`basename $1`" From 7d830ff52ff7b01f528f39aa27b1ab36ea8c1356 Mon Sep 17 00:00:00 2001 From: Andrew Aladjev Date: Sun, 7 Nov 2021 22:45:49 +0300 Subject: [PATCH 006/350] improved cross compilation detection (#252) --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index fbdd17d83..9e7338cfc 100644 --- a/configure.ac +++ b/configure.ac @@ -264,7 +264,7 @@ if test x"$enable_simd" != x"no"; then # For x86-64 SIMD, g++ >=5 or clang++ >=7 is required if test x"$host_cpu" = x"x86_64" || test x"$host_cpu" = x"amd64"; then AC_LANG(C++) - if test x"$host_cpu" = x"$build_cpu"; then + if test x"$host" = x"$build"; then AC_RUN_IFELSE([AC_LANG_PROGRAM([SIMD_X86_64_TEST],[[if (test_ssse3(42) != 42 || test_sse2(42) != 42 || test_avx2(42) != 42) exit(1);]])], [CXX_OK=yes],[CXX_OK=no]) else From 1f0e62f139565f39020e1f76700828b415b01bcf Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Sat, 13 Nov 2021 09:30:08 -0800 Subject: [PATCH 007/350] Improve a couple support scripts: - rsync-no-vanished now avoids joining stdout & stderr, avoids affecting a non-client run, and gets the rsync status code correctly. - rsync-slash-strip now avoids affecting a non-client run. --- support/rsync-no-vanished | 13 +++++++++++-- support/rsync-slash-strip | 9 ++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/support/rsync-no-vanished b/support/rsync-no-vanished index 0f0bb22f5..b31a5d210 100755 --- a/support/rsync-no-vanished +++ b/support/rsync-no-vanished @@ -1,12 +1,21 @@ #!/usr/bin/env bash +REAL_RSYNC=/usr/bin/rsync IGNOREEXIT=24 IGNOREOUT='^(file has vanished: |rsync warning: some files vanished before they could be transferred)' +# If someone installs this as "rsync", make sure we don't affect a server run. +for arg in "${@}"; do + if [[ "$arg" == --server ]]; then + exec $REAL_RSYNC "${@}" + exit $? # Not reached + fi +done + set -o pipefail -rsync "${@}" 2>&1 | (grep -E -v "$IGNOREOUT" || true) -ret=$? +# This filters stderr without merging it with stdout: +{ $REAL_RSYNC "${@}" 2>&1 1>&3 3>&- | grep -E -v "$IGNOREOUT"; ret=${PIPESTATUS[0]}; } 3>&1 1>&2 if [[ $ret == $IGNOREEXIT ]]; then ret=0 diff --git a/support/rsync-slash-strip b/support/rsync-slash-strip index 2869e45cb..b57e61c53 100755 --- a/support/rsync-slash-strip +++ b/support/rsync-slash-strip @@ -6,12 +6,19 @@ # # To use this, name it something like "rs", put it somewhere in your path, and # then use "rs" in place of "rsync" when you are typing your copy commands. + +REAL_RSYNC=/usr/bin/rsync + args=() for arg in "${@}"; do + if [[ "$arg" == --server ]]; then + exec $REAL_RSYNC "${@}" + exit $? # Not reached + fi if [[ "$arg" == / ]]; then args=("${args[@]}" /) else args=("${args[@]}" "${arg%/}") fi done -exec /usr/bin/rsync "${args[@]}" +exec $REAL_RSYNC "${args[@]}" From d9015da151fff4fe1d844a5c8b3f07f7ecff63e9 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Mon, 20 Dec 2021 13:34:05 -0800 Subject: [PATCH 008/350] Add --munge-links rsync option; convert to python. --- packaging/cull_options | 205 ++++++++++++++++++++++++++--------------- 1 file changed, 129 insertions(+), 76 deletions(-) diff --git a/packaging/cull_options b/packaging/cull_options index 7588002be..85311c7c7 100755 --- a/packaging/cull_options +++ b/packaging/cull_options @@ -1,85 +1,138 @@ -#!/usr/bin/env perl -# This script outputs some perl code that parses all possible options -# that the code in options.c might send to the server. This perl code -# is included in the rrsync script. -use strict; - -our %short_no_arg; -our %short_with_num = ( '@' => 1 ); -our %long_opt = ( # These include some extra long-args that BackupPC uses: - 'block-size' => 1, - 'daemon' => -1, - 'debug' => 1, - 'fake-super' => 0, - 'fuzzy' => 0, - 'group' => 0, - 'hard-links' => 0, - 'ignore-times' => 0, - 'info' => 1, - 'links' => 0, - 'log-file' => 3, - 'one-file-system' => 0, - 'owner' => 0, - 'perms' => 0, - 'recursive' => 0, - 'times' => 0, - 'write-devices' => -1, -); -our $last_long_opt; - -open(IN, '../options.c') or die "Unable to open ../options.c: $!\n"; - -while () { - if (/\Qargstr[x++]\E = '([^.ie])'/) { - $short_no_arg{$1} = 1; - undef $last_long_opt; - } elsif (/\Qasprintf(\E[^,]+, "-([a-zA-Z0-9])\%l?[ud]"/) { - $short_with_num{$1} = 1; - undef $last_long_opt; - } elsif (/\Qargs[ac++]\E = "--([^"=]+)"/) { - $last_long_opt = $1; - $long_opt{$1} = 0 unless exists $long_opt{$1}; - } elsif (defined($last_long_opt) - && /\Qargs[ac++]\E = ([^["\s]+);/) { - $long_opt{$last_long_opt} = 2; - undef $last_long_opt; - } elsif (/return "--([^"]+-dest)";/) { - $long_opt{$1} = 2; - undef $last_long_opt; - } elsif (/\Qasprintf(\E[^,]+, "--([^"=]+)=/ || /\Qargs[ac++]\E = "--([^"=]+)=/ || /fmt = .*: "--([^"=]+)=/) { - $long_opt{$1} = 1; - undef $last_long_opt; - } -} -close IN; - -my $short_no_arg = join('', sort keys %short_no_arg); -my $short_with_num = join('', sort keys %short_with_num); - -print < $val,\n"; -} - -print ");\n\n"; +""" + + print(txt, end='') + + if args.python: + print("long_opt = {") + sep = ':' + else: + print("our %long_opt = (") + sep = ' =>' + + for opt in sorted(long_opt): + if opt.startswith(('min-', 'max-')): + val = 1 + else: + val = long_opt[opt] + print(' ', repr(opt) + sep, str(val) + ',') + + if args.python: + print("}") + else: + print(");") + print('') + + +def str_assign(name, val, comment=None): + comment = ' # ' + comment if comment else '' + if args.python: + return name + ' = ' + repr(val) + comment + "\n" + return 'our $' + name + ' = ' + repr(val) + ';' + comment + "\n" + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Output culled rsync options for rrsync.", add_help=False) + out_group = parser.add_mutually_exclusive_group() + out_group.add_argument('--perl', action='store_true', help="Output perl code (the default).") + out_group.add_argument('--python', action='store_true', help="Output python code.") + parser.add_argument('--help', '-h', action='help', help="Output this help message and exit.") + args = parser.parse_args() + main() + +# vim: sw=4 et From dc1b9febf1056c723003ccd9be1a4d7adef6425d Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Mon, 20 Dec 2021 14:25:19 -0800 Subject: [PATCH 009/350] Add options to assist in localhost rrsync testing. --- support/lsh | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/support/lsh b/support/lsh index be29310c7..ebfe898c4 100755 --- a/support/lsh +++ b/support/lsh @@ -15,6 +15,9 @@ GetOptions( 'b|c|D|e|F|i|L|m|O|o|p|R|S|w=s' => sub { }, # Ignore 'no-cd' => \( my $no_chdir ), 'sudo' => \( my $use_sudo ), + 'rrsync=s' => \( my $rrsync_dir ), + 'ro' => \( my $rrsync_ro = '' ), + 'wo' => \( my $rrsync_wo = '' ), ) or &usage; &usage unless @ARGV > 1; @@ -67,22 +70,42 @@ unless ($no_chdir) { chdir $home_dir or die "Unable to chdir to $home_dir: $!\n"; } -push @cmd, '/bin/sh', '-c', "@ARGV"; +if ($rrsync_dir) { + my $cmd = ''; + foreach (@ARGV) { + (my $arg = $_) =~ s/(['";|()\[\]{}\$!*?<> \t&~\\])/\\$1/g; + $cmd .= ' ' . $arg; + } + $cmd =~ s/^\s+//; + $ENV{SSH_ORIGINAL_COMMAND} = $cmd; + push @cmd, 'rrsync'; + push @cmd, '-ro' if $rrsync_ro; + push @cmd, '-wo' if $rrsync_wo; + push @cmd, $rrsync_dir; +} else { + push @cmd, '/bin/sh', '-c', "@ARGV"; +} exec @cmd; die "Failed to exec: $!\n"; sub usage { die < Date: Mon, 20 Dec 2021 15:13:50 -0800 Subject: [PATCH 010/350] Make rrsync default to munged symlinks. --- NEWS.md | 4 +++- support/rrsync | 53 +++++++++++++++++++++++++++++++++++++------------- 2 files changed, 42 insertions(+), 15 deletions(-) mode change 100644 => 100755 support/rrsync diff --git a/NEWS.md b/NEWS.md index 9f9433d20..eaa82b395 100644 --- a/NEWS.md +++ b/NEWS.md @@ -97,7 +97,9 @@ - More ASM optimizations from Shark64. - - Make rrsync handle the latest options. + - Make rrsync pass --munge-links to rsync by default to make the restricted + dir extra safe (with an option to turn it off if you trust your users). + Also updated the known options list. - Work around a glibc bug where lchmod() breaks in a chroot w/o /proc mounted. diff --git a/support/rrsync b/support/rrsync old mode 100644 new mode 100755 index 438e3a24a..4c5dd2aa7 --- a/support/rrsync +++ b/support/rrsync @@ -15,19 +15,26 @@ use constant RSYNC => '/usr/bin/rsync'; use constant LOGFILE => 'rrsync.log'; my $Usage = < 2, 'links' => 0, 'list-only' => 0, - 'log-file' => $only eq 'r' ? -1 : 3, + 'log-file' => 3, 'log-format' => 1, 'max-alloc' => 1, 'max-delete' => 1, @@ -119,10 +126,12 @@ our %long_opt = ( 'mkpath' => 0, 'modify-window' => 1, 'msgs2stderr' => 0, + 'munge-links' => 0, 'new-compress' => 0, 'no-W' => 0, 'no-implied-dirs' => 0, 'no-msgs2stderr' => 0, + 'no-munge-links' => -1, 'no-r' => 0, 'no-relative' => 0, 'no-specials' => 0, @@ -137,10 +146,10 @@ our %long_opt = ( 'perms' => 0, 'preallocate' => 0, 'recursive' => 0, - 'remove-sent-files' => $only eq 'r' ? -1 : 0, - 'remove-source-files' => $only eq 'r' ? -1 : 0, + 'remove-sent-files' => 0, + 'remove-source-files' => 0, 'safe-links' => 0, - 'sender' => $only eq 'w' ? -1 : 0, + 'sender' => 0, 'server' => 0, 'size-only' => 0, 'skip-compress' => 1, @@ -158,6 +167,16 @@ our %long_opt = ( ### END of options data produced by the cull_options script. ### +if ($only eq 'r') { + foreach my $opt (keys %long_opt) { + if ($opt =~ /^(remove-|log-file)/) { + $long_opt{$opt} = -1; + } + } +} elsif ($only eq 'w') { + $long_opt{'sender'} = -1; +} + if ($short_disabled ne '') { $short_no_arg =~ s/[$short_disabled]//go; $short_with_num =~ s/[$short_disabled]//go; @@ -179,11 +198,11 @@ while ($command =~ /((?:[^\s\\]+|\\.[^\s\\]*)+)/g) { push(@opts, check_arg($last_opt, $_, $check_type)); $check_type = 0; } elsif ($in_options) { - push(@opts, $_); if ($_ eq '.') { $in_options = 0; } else { die "$0: invalid option: '-'\n" if $_ eq '-'; + push(@opts, $_); next if /^-$short_no_arg*(e\d*\.\w*)?$/o || /^-$short_with_num\d+$/o; my($opt,$arg) = /^--([^=]+)(?:=(.*))?$/; @@ -225,7 +244,11 @@ while ($command =~ /((?:[^\s\\]+|\\.[^\s\\]*)+)/g) { die "$0: invalid rsync-command syntax or options\n" if $in_options; if ($subdir ne '/') { - die "$0: do not use .. in any path!\n" if grep m{(^|/)\.\.(/|$)}, @args; + die "$0: do not use .. in any path!\n" if grep m{(^|/)\.\.(/|$)}, @args; +} + +if ($force_munge) { + push(@opts, '--munge-links'); } @args = ( '.' ) if !@args; @@ -241,7 +264,7 @@ if ($write_log) { } # Note: This assumes that the rsync protocol will not be maliciously hijacked. -exec(RSYNC, @opts, '--', @args) or die "exec(rsync @opts -- @args) failed: $? $!"; +exec(RSYNC, @opts, '--', '.', @args) or die "exec(rsync @opts -- . @args) failed: $? $!"; sub check_arg { @@ -255,3 +278,5 @@ sub check_arg } $arg; } + +# vim: sw=2 From 39c3ae0ea3a4257c3be9e967549a953a95304b73 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Mon, 20 Dec 2021 16:08:34 -0800 Subject: [PATCH 011/350] Convert munge-symlinks to python. --- support/munge-symlinks | 125 ++++++++++++++++++++++------------------- 1 file changed, 68 insertions(+), 57 deletions(-) diff --git a/support/munge-symlinks b/support/munge-symlinks index 3e5f3ad2a..e7a547416 100755 --- a/support/munge-symlinks +++ b/support/munge-symlinks @@ -1,60 +1,71 @@ -#!/usr/bin/env perl +#!/usr/bin/env python3 # This script will either prefix all symlink values with the string # "/rsyncd-munged/" or remove that prefix. -use strict; -use Getopt::Long; - -my $SYMLINK_PREFIX = '/rsyncd-munged/'; - -my $munge_opt; - -&GetOptions( - 'munge' => sub { $munge_opt = 1 }, - 'unmunge' => sub { $munge_opt = 0 }, - 'all' => \( my $all_opt ), - 'help|h' => \( my $help_opt ), -) or &usage; - -&usage if $help_opt || !defined $munge_opt; - -my $munged_re = $all_opt ? qr/^($SYMLINK_PREFIX)+(?=.)/ : qr/^$SYMLINK_PREFIX(?=.)/; - -push(@ARGV, '.') unless @ARGV; - -open(PIPE, '-|', 'find', @ARGV, '-type', 'l') or die $!; - -while () { - chomp; - my $lnk = readlink($_) or next; - if ($munge_opt) { - next if !$all_opt && $lnk =~ /$munged_re/; - $lnk =~ s/^/$SYMLINK_PREFIX/; - } else { - next unless $lnk =~ s/$munged_re//; - } - if (!unlink($_)) { - warn "Unable to unlink symlink: $_ ($!)\n"; - } elsif (!symlink($lnk, $_)) { - warn "Unable to recreate symlink: $_ -> $lnk ($!)\n"; - } else { - print "$_ -> $lnk\n"; - } -} - -close PIPE; -exit; - -sub usage -{ - die <', lnk + ':', str(e), file=sys.stderr) + return + print(fn, '->', lnk) + + +if __name__ == '__main__': + our_desc = """\ +Adds or removes the %s prefix to/from the start of each symlink's value. +When given the name of a directory, affects all the symlinks in that directory hierarchy. +""" % SYMLINK_PREFIX + epilog = 'See the "munge symlinks" option in the rsyncd.conf manpage for more details.' + parser = argparse.ArgumentParser(description=our_desc, epilog=epilog, add_help=False) + uniq_group = parser.add_mutually_exclusive_group() + uniq_group.add_argument('--munge', action='store_true', help="Add the prefix to symlinks (the default).") + uniq_group.add_argument('--unmunge', action='store_true', help="Remove the prefix from symlinks.") + parser.add_argument('--all', action='store_true', help="Always adds the prefix when munging (even if already munged) or removes multiple instances of the prefix when unmunging.") + parser.add_argument('--help', '-h', action='help', help="Output this help message and exit.") + parser.add_argument('names', metavar='NAME', nargs='+', help="One or more directories and/or symlinks to process.") + args = parser.parse_args() + main() + +# vim: sw=4 et From 73ceea6ad2af00f251a5e79a0a258f9fee97d531 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Mon, 20 Dec 2021 17:28:18 -0800 Subject: [PATCH 012/350] Convert atomic-rsync to python. --- support/atomic-rsync | 179 ++++++++++++++++++++++--------------------- 1 file changed, 91 insertions(+), 88 deletions(-) diff --git a/support/atomic-rsync b/support/atomic-rsync index 0346fb49a..37363e435 100755 --- a/support/atomic-rsync +++ b/support/atomic-rsync @@ -1,92 +1,83 @@ -#!/usr/bin/env perl -# +#!/usr/bin/env python3 # This script lets you update a hierarchy of files in an atomic way by # first creating a new hierarchy using rsync's --link-dest option, and # then swapping the hierarchy into place. **See the usage message for # more details and some important caveats!** -use strict; -use warnings; -use Cwd 'abs_path'; - -my $RSYNC_PROG = '/usr/bin/rsync'; -my $RM_PROG = '/bin/rm'; - -my $dest_dir = $ARGV[-1]; -&usage if !defined $dest_dir || $dest_dir =~ /(^-|^$)/ || grep(/^--help/, @ARGV); -$dest_dir =~ s{(?<=.)/+$} {}; - -if (!-d $dest_dir) { - die "$dest_dir is not a directory.\nUse --help for help.\n"; -} - -if (@_ = grep(/^--[a-z]+-dest\b/, @ARGV)) { - $_ = join(' or ', @_); - die "You cannot use the $_ option with atomic-rsync.\nUse --help for help.\n"; -} - -my $symlink_content = readlink $dest_dir; # undef when a real dir - -my $dest_arg = $dest_dir; -# This gives us the real destination dir, with all symlinks dereferenced. -$dest_dir = abs_path($dest_dir); -if ($dest_dir eq '/') { - die qq|You must not use "/" as the destination directory.\nUse --help for help.\n|; -} - -my($old_dir, $new_dir); -if (defined $symlink_content && $dest_dir =~ /-([12])$/) { - my $num = 3 - $1; - $old_dir = undef; - ($new_dir = $dest_dir) =~ s/-[12]$/-$num/; - $symlink_content =~ s/-[12]$/-$num/; -} else { - $old_dir = "$dest_dir~old~"; - $new_dir = "$dest_dir~new~"; -} - -$ARGV[-1] = "$new_dir/"; - -system($RM_PROG, '-rf', $old_dir) if defined $old_dir && -d $old_dir; -system($RM_PROG, '-rf', $new_dir) if -d $new_dir; - -if (system($RSYNC_PROG, "--link-dest=$dest_dir", @ARGV)) { - if ($? == -1) { - print "failed to execute $RSYNC_PROG: $!\n"; - } elsif ($? & 127) { - printf "child died with signal %d, %s coredump\n", - ($? & 127), ($? & 128) ? 'with' : 'without'; - } else { - printf "child exited with value %d\n", $? >> 8; - } - exit 1; -} - -if (!defined $old_dir) { - atomic_symlink($symlink_content, $dest_arg); - exit; -} - -rename($dest_dir, $old_dir) or die "Unable to rename $dest_dir to $old_dir: $!"; -rename($new_dir, $dest_dir) or die "Unable to rename $new_dir to $dest_dir: $!"; - -exit; - -sub atomic_symlink -{ - my($target, $link) = @_; - my $newlink = "$link~new~"; - - unlink($newlink); # Just in case - symlink($target, $newlink) or die "Unable to symlink $newlink -> $target: $!\n"; - rename($newlink, $link) or die "Unable to rename $newlink to $link: $!\n"; -} - - -sub usage -{ - die < Date: Sun, 26 Dec 2021 12:29:00 -0800 Subject: [PATCH 013/350] rrsync improvements - Convert rrsync to python. - Enhance security of arg & option checking. - Reject `-L` (`--copy-links`) by default. - Add `-munge` and `-no-del` options. - Tweak the logfile line format. - Created an rrsync man page. - Use `configure --with-rrsync` if you want `make install` to install rrsync and its man page. - Give lsh more rrsync testing support. --- Makefile.in | 10 +- NEWS.md | 18 +- configure.ac | 7 + maybe-make-man | 2 +- md2man | 13 +- packaging/cull_options | 36 +-- support/lsh | 12 +- support/rrsync | 595 +++++++++++++++++++++++------------------ support/rrsync.1.md | 89 ++++++ 9 files changed, 488 insertions(+), 294 deletions(-) create mode 100644 support/rrsync.1.md diff --git a/Makefile.in b/Makefile.in index 3c8c22405..5eed339e3 100644 --- a/Makefile.in +++ b/Makefile.in @@ -6,6 +6,7 @@ exec_prefix=@exec_prefix@ bindir=@bindir@ libdir=@libdir@/rsync mandir=@mandir@ +with_rrsync=@with_rrsync@ LIBS=@LIBS@ CC=@CC@ @@ -80,6 +81,10 @@ install: all if test -f rsync.1; then $(INSTALLMAN) -m 644 rsync.1 $(DESTDIR)$(mandir)/man1; fi if test -f rsync-ssl.1; then $(INSTALLMAN) -m 644 rsync-ssl.1 $(DESTDIR)$(mandir)/man1; fi if test -f rsyncd.conf.5; then $(INSTALLMAN) -m 644 rsyncd.conf.5 $(DESTDIR)$(mandir)/man5; fi + if test "$(with_rrsync)" = yes; then \ + $(INSTALLCMD) -m 755 $(srcdir)/support/rrsync $(DESTDIR)$(bindir); \ + if test -f rrsync.1; then $(INSTALLMAN) -m 644 rrsync.1 $(DESTDIR)$(mandir)/man1; fi; \ + fi install-ssl-daemon: stunnel-rsyncd.conf -$(MKDIR_P) $(DESTDIR)/etc/stunnel @@ -247,7 +252,7 @@ proto.h-tstamp: $(srcdir)/*.c $(srcdir)/lib/compat.c daemon-parm.h $(AWK) -f $(srcdir)/mkproto.awk $(srcdir)/*.c $(srcdir)/lib/compat.c daemon-parm.h .PHONY: man -man: rsync.1 rsync-ssl.1 rsyncd.conf.5 +man: rsync.1 rsync-ssl.1 rsyncd.conf.5 rrsync.1 rsync.1: rsync.1.md md2man version.h Makefile @$(srcdir)/maybe-make-man $(srcdir) rsync.1.md @@ -258,6 +263,9 @@ rsync-ssl.1: rsync-ssl.1.md md2man version.h Makefile rsyncd.conf.5: rsyncd.conf.5.md md2man version.h Makefile @$(srcdir)/maybe-make-man $(srcdir) rsyncd.conf.5.md +rrsync.1: support/rrsync.1.md md2man Makefile + @$(srcdir)/maybe-make-man $(srcdir) support/rrsync.1.md + .PHONY: clean clean: cleantests rm -f *~ $(OBJS) $(CHECK_PROGS) $(CHECK_OBJS) $(CHECK_SYMLINKS) \ diff --git a/NEWS.md b/NEWS.md index eaa82b395..b3002e893 100644 --- a/NEWS.md +++ b/NEWS.md @@ -97,9 +97,15 @@ - More ASM optimizations from Shark64. - - Make rrsync pass --munge-links to rsync by default to make the restricted - dir extra safe (with an option to turn it off if you trust your users). - Also updated the known options list. + - Transformed rrsync into a python script with improvements: security has been + beefed up; the known rsync options were updated to include recent additions; + rrsync rejects `-L` (`--copy-links`) by default to make it harder to exploit + any out-of-subdir symlinks; a new rrsync option of `-munge` tells rrsync to + always enable the `--munge-links` rsync option on the server side; a new + rrsync option of `-no-del` disables all `--remove*` and `--delete*` rsync + options on the server side; the log format has been tweaked slightly to add + seconds to the timestamp and output the command executed as a tuple; an + rrsync.1 manpage is now created. - Work around a glibc bug where lchmod() breaks in a chroot w/o /proc mounted. @@ -107,6 +113,12 @@ ### PACKAGING RELATED: + - Give configure the --with-rrsync option if you want `make install` to + install the (now python3) rrsync script and its (new) man page. + + - If the rrsync script is installed, make its package depend on python3 and + (suggested but not required) the python3 braceexpand lib. + - When creating a package from a non-release version (w/o a git checkout), the packager can elect to create git-version.h and define RSYNC_GITVER to the string they want `--version` to output. (The file is still auto-generated diff --git a/configure.ac b/configure.ac index 9e7338cfc..84111de8b 100644 --- a/configure.ac +++ b/configure.ac @@ -136,6 +136,13 @@ if test x"$GCC" = x"yes"; then CFLAGS="$CFLAGS -Wall -W" fi +AC_ARG_WITH(rrsync, + AS_HELP_STRING([--with-rrsync],[also install the rrsync script and its man page])) +if test x"$with_rrsync" != x"yes"; then + with_rrsync=no +fi +AC_SUBST(with_rrsync) + AC_ARG_WITH(included-popt, AS_HELP_STRING([--with-included-popt],[use bundled popt library, not from system])) diff --git a/maybe-make-man b/maybe-make-man index b7f0a9f1a..59f2dce42 100755 --- a/maybe-make-man +++ b/maybe-make-man @@ -37,4 +37,4 @@ if [ ! -f "$flagfile" ]; then fi fi -"$srcdir/md2man" "$srcdir/$inname" +"$srcdir/md2man" -s "$srcdir" "$srcdir/$inname" diff --git a/md2man b/md2man index fa1d2e824..fd546f190 100755 --- a/md2man +++ b/md2man @@ -85,7 +85,9 @@ def main(): die('Failed to parse NAME.NUM.md out of input file:', args.mdfile) fi = argparse.Namespace(**fi.groupdict()) - if not fi.srcdir: + if args.srcdir: + fi.srcdir = args.srcdir + '/' + elif not fi.srcdir: fi.srcdir = './' fi.title = fi.prog + '(' + fi.sect + ') man page' @@ -105,7 +107,7 @@ def main(): for fn in (fi.srcdir + 'version.h', 'Makefile'): try: st = os.lstat(fn) - except: + except OSError: die('Failed to find', fi.srcdir + fn) if not fi.mtime: fi.mtime = st.st_mtime @@ -129,6 +131,10 @@ def main(): if var == 'srcdir': break + fi.prog_ver = 'rsync ' + env_subs['VERSION'] + if fi.prog != 'rsync': + fi.prog_ver = fi.prog + ' from ' + fi.prog_ver + with open(fi.fn, 'r', encoding='utf-8') as fh: txt = fh.read() @@ -140,7 +146,7 @@ def main(): txt = None fi.date = time.strftime('%d %b %Y', time.localtime(fi.mtime)) - fi.man_headings = (fi.prog, fi.sect, fi.date, fi.prog + ' ' + env_subs['VERSION'], env_subs['prefix']) + fi.man_headings = (fi.prog, fi.sect, fi.date, fi.prog_ver, env_subs['prefix']) HtmlToManPage(fi) @@ -374,6 +380,7 @@ def die(*msg): if __name__ == '__main__': parser = argparse.ArgumentParser(description='Transform a NAME.NUM.md markdown file into a NAME.NUM.html web page & a NAME.NUM man page.', add_help=False) + parser.add_argument('--srcdir', '-s', help='Specify the source dir if the input file is not in it.') parser.add_argument('--test', action='store_true', help='Test if we can parse the input w/o updating any files.') parser.add_argument('--debug', '-D', action='count', default=0, help='Output copious info on the html parsing. Repeat for even more.') parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.") diff --git a/packaging/cull_options b/packaging/cull_options index 85311c7c7..d4e1c626d 100755 --- a/packaging/cull_options +++ b/packaging/cull_options @@ -7,7 +7,7 @@ import re, argparse short_no_arg = { } short_with_num = { '@': 1 }; -long_opt = { # These include some extra long-args that BackupPC uses: +long_opts = { # These include some extra long-args that BackupPC uses: 'block-size': 1, 'daemon': -1, 'debug': 1, @@ -25,6 +25,7 @@ long_opt = { # These include some extra long-args that BackupPC uses: 'owner': 0, 'perms': 0, 'recursive': 0, + 'stderr': 1, 'times': 0, 'write-devices': -1, } @@ -49,8 +50,8 @@ def main(): m = re.search(r'args\[ac\+\+\] = "--([^"=]+)"', line) if m: last_long_opt = m.group(1) - if last_long_opt not in long_opt: - long_opt[last_long_opt] = 0 + if last_long_opt not in long_opts: + long_opts[last_long_opt] = 0 else: last_long_opt = None continue @@ -58,13 +59,13 @@ def main(): if last_long_opt: m = re.search(r'args\[ac\+\+\] = ([^["\s]+);', line) if m: - long_opt[last_long_opt] = 2 + long_opts[last_long_opt] = 2 last_long_opt = None continue m = re.search(r'return "--([^"]+-dest)";', line) if m: - long_opt[m.group(1)] = 2 + long_opts[m.group(1)] = 2 last_long_opt = None continue @@ -74,19 +75,18 @@ def main(): if not m: m = re.search(r'fmt = .*: "--([^"=]+)=', line) if m: - long_opt[m.group(1)] = 1 + long_opts[m.group(1)] = 1 last_long_opt = None - long_opt['files-from'] = 3 + long_opts['files-from'] = 3 - txt = """ -# These options are the only options that rsync might send to the server, -# and only in the option format that the stock rsync produces. + txt = """\ +### START of options data produced by the cull_options script. ### # To disable a short-named option, add its letter to this string: """ - txt += str_assign('short_disabled', 's') + "\n" + txt += str_assign('short_disabled', 'Ls') + "\n" txt += str_assign('short_no_arg', ''.join(sorted(short_no_arg)), 'DO NOT REMOVE ANY') txt += str_assign('short_with_num', ''.join(sorted(short_with_num)), 'DO NOT REMOVE ANY') @@ -99,24 +99,24 @@ def main(): print(txt, end='') if args.python: - print("long_opt = {") + print("long_opts = {") sep = ':' else: print("our %long_opt = (") sep = ' =>' - for opt in sorted(long_opt): + for opt in sorted(long_opts): if opt.startswith(('min-', 'max-')): val = 1 else: - val = long_opt[opt] + val = long_opts[opt] print(' ', repr(opt) + sep, str(val) + ',') if args.python: print("}") else: print(");") - print('') + print("\n### END of options data produced by the cull_options script. ###") def str_assign(name, val, comment=None): @@ -129,10 +129,12 @@ def str_assign(name, val, comment=None): if __name__ == '__main__': parser = argparse.ArgumentParser(description="Output culled rsync options for rrsync.", add_help=False) out_group = parser.add_mutually_exclusive_group() - out_group.add_argument('--perl', action='store_true', help="Output perl code (the default).") - out_group.add_argument('--python', action='store_true', help="Output python code.") + out_group.add_argument('--perl', action='store_true', help="Output perl code.") + out_group.add_argument('--python', action='store_true', help="Output python code (the default).") parser.add_argument('--help', '-h', action='help', help="Output this help message and exit.") args = parser.parse_args() + if not args.perl: + args.python = True main() # vim: sw=4 et diff --git a/support/lsh b/support/lsh index ebfe898c4..40fe3d738 100755 --- a/support/lsh +++ b/support/lsh @@ -18,6 +18,8 @@ GetOptions( 'rrsync=s' => \( my $rrsync_dir ), 'ro' => \( my $rrsync_ro = '' ), 'wo' => \( my $rrsync_wo = '' ), + 'munge' => \( my $rrsync_munge = '' ), + 'no-del' => \( my $rrsync_no_del = '' ), ) or &usage; &usage unless @ARGV > 1; @@ -71,16 +73,12 @@ unless ($no_chdir) { } if ($rrsync_dir) { - my $cmd = ''; - foreach (@ARGV) { - (my $arg = $_) =~ s/(['";|()\[\]{}\$!*?<> \t&~\\])/\\$1/g; - $cmd .= ' ' . $arg; - } - $cmd =~ s/^\s+//; - $ENV{SSH_ORIGINAL_COMMAND} = $cmd; + $ENV{SSH_ORIGINAL_COMMAND} = join(' ', @ARGV); push @cmd, 'rrsync'; push @cmd, '-ro' if $rrsync_ro; push @cmd, '-wo' if $rrsync_wo; + push @cmd, '-munge' if $rrsync_munge; + push @cmd, '-no-del' if $rrsync_no_del; push @cmd, $rrsync_dir; } else { push @cmd, '/bin/sh', '-c', "@ARGV"; diff --git a/support/rrsync b/support/rrsync index 4c5dd2aa7..5b43a8198 100755 --- a/support/rrsync +++ b/support/rrsync @@ -1,282 +1,353 @@ -#!/usr/bin/env perl -# Name: /usr/local/bin/rrsync (should also have a symlink in /usr/bin) -# Purpose: Restricts rsync to subdirectory declared in .ssh/authorized_keys -# Author: Joe Smith 30-Sep-2004 -# Modified by: Wayne Davison -use strict; - -use Socket; -use Cwd 'abs_path'; -use File::Glob ':glob'; - -# You may configure these values to your liking. See also the section -# of options if you want to disable any options that rsync accepts. -use constant RSYNC => '/usr/bin/rsync'; -use constant LOGFILE => 'rrsync.log'; - -my $Usage = < 30-Sep-2004 +# Python version by: Wayne Davison + +# You may configure these 2 values to your liking. See also the section of +# short & long options if you want to disable any options that rsync accepts. +RSYNC = '/usr/bin/rsync' +LOGFILE = 'rrsync.log' # NOTE: the file must exist for a line to be appended! + +# The following options are mainly the options that a client rsync can send +# to the server, and usually just in the one option format that the stock +# rsync produces. However, there are some additional convenience options +# added as well, and thus a few options are present in both the short and +# long lists (such as --group, --owner, and --perms). -# These options are the only options that rsync might send to the server, -# and only in the option format that the stock rsync produces. +# NOTE when disabling: check for both a short & long version of the option! + +### START of options data produced by the cull_options script. ### # To disable a short-named option, add its letter to this string: -our $short_disabled = 's'; +short_disabled = 'Ls' -our $short_no_arg = 'ACDEHIJKLNORSUWXbcdgklmnopqrstuvxyz'; # DO NOT REMOVE ANY -our $short_with_num = '@B'; # DO NOT REMOVE ANY +short_no_arg = 'ACDEHIJKLNORSUWXbcdgklmnopqrstuvxyz' # DO NOT REMOVE ANY +short_with_num = '@B' # DO NOT REMOVE ANY # To disable a long-named option, change its value to a -1. The values mean: # 0 = the option has no arg; 1 = the arg doesn't need any checking; 2 = only # check the arg when receiving; and 3 = always check the arg. -our %long_opt = ( - 'append' => 0, - 'backup-dir' => 2, - 'block-size' => 1, - 'bwlimit' => 1, - 'checksum-choice' => 1, - 'checksum-seed' => 1, - 'compare-dest' => 2, - 'compress-choice' => 1, - 'compress-level' => 1, - 'copy-dest' => 2, - 'copy-unsafe-links' => 0, - 'daemon' => -1, - 'debug' => 1, - 'delay-updates' => 0, - 'delete' => 0, - 'delete-after' => 0, - 'delete-before' => 0, - 'delete-delay' => 0, - 'delete-during' => 0, - 'delete-excluded' => 0, - 'delete-missing-args' => 0, - 'existing' => 0, - 'fake-super' => 0, - 'files-from' => 3, - 'force' => 0, - 'from0' => 0, - 'fsync' => 2, - 'fuzzy' => 0, - 'group' => 0, - 'groupmap' => 1, - 'hard-links' => 0, - 'iconv' => 1, - 'ignore-errors' => 0, - 'ignore-existing' => 0, - 'ignore-missing-args' => 0, - 'ignore-times' => 0, - 'info' => 1, - 'inplace' => 0, - 'link-dest' => 2, - 'links' => 0, - 'list-only' => 0, - 'log-file' => 3, - 'log-format' => 1, - 'max-alloc' => 1, - 'max-delete' => 1, - 'max-size' => 1, - 'min-size' => 1, - 'mkpath' => 0, - 'modify-window' => 1, - 'msgs2stderr' => 0, - 'munge-links' => 0, - 'new-compress' => 0, - 'no-W' => 0, - 'no-implied-dirs' => 0, - 'no-msgs2stderr' => 0, - 'no-munge-links' => -1, - 'no-r' => 0, - 'no-relative' => 0, - 'no-specials' => 0, - 'numeric-ids' => 0, - 'old-compress' => 0, - 'one-file-system' => 0, - 'only-write-batch' => 1, - 'open-noatime' => 0, - 'owner' => 0, - 'partial' => 0, - 'partial-dir' => 2, - 'perms' => 0, - 'preallocate' => 0, - 'recursive' => 0, - 'remove-sent-files' => 0, - 'remove-source-files' => 0, - 'safe-links' => 0, - 'sender' => 0, - 'server' => 0, - 'size-only' => 0, - 'skip-compress' => 1, - 'specials' => 0, - 'stats' => 0, - 'suffix' => 1, - 'super' => 0, - 'temp-dir' => 2, - 'timeout' => 1, - 'times' => 0, - 'use-qsort' => 0, - 'usermap' => 1, - 'write-devices' => -1, -); +long_opts = { + 'append': 0, + 'backup-dir': 2, + 'block-size': 1, + 'bwlimit': 1, + 'checksum-choice': 1, + 'checksum-seed': 1, + 'compare-dest': 2, + 'compress-choice': 1, + 'compress-level': 1, + 'copy-dest': 2, + 'copy-unsafe-links': 0, + 'daemon': -1, + 'debug': 1, + 'delay-updates': 0, + 'delete': 0, + 'delete-after': 0, + 'delete-before': 0, + 'delete-delay': 0, + 'delete-during': 0, + 'delete-excluded': 0, + 'delete-missing-args': 0, + 'existing': 0, + 'fake-super': 0, + 'files-from': 3, + 'force': 0, + 'from0': 0, + 'fsync': 2, + 'fuzzy': 0, + 'group': 0, + 'groupmap': 1, + 'hard-links': 0, + 'iconv': 1, + 'ignore-errors': 0, + 'ignore-existing': 0, + 'ignore-missing-args': 0, + 'ignore-times': 0, + 'info': 1, + 'inplace': 0, + 'link-dest': 2, + 'links': 0, + 'list-only': 0, + 'log-file': 3, + 'log-format': 1, + 'max-alloc': 1, + 'max-delete': 1, + 'max-size': 1, + 'min-size': 1, + 'mkpath': 0, + 'modify-window': 1, + 'msgs2stderr': 0, + 'munge-links': 0, + 'new-compress': 0, + 'no-W': 0, + 'no-implied-dirs': 0, + 'no-msgs2stderr': 0, + 'no-munge-links': -1, + 'no-r': 0, + 'no-relative': 0, + 'no-specials': 0, + 'numeric-ids': 0, + 'old-compress': 0, + 'one-file-system': 0, + 'only-write-batch': 1, + 'open-noatime': 0, + 'owner': 0, + 'partial': 0, + 'partial-dir': 2, + 'perms': 0, + 'preallocate': 0, + 'recursive': 0, + 'remove-sent-files': 0, + 'remove-source-files': 0, + 'safe-links': 0, + 'sender': 0, + 'server': 0, + 'size-only': 0, + 'skip-compress': 1, + 'specials': 0, + 'stats': 0, + 'stderr': 1, + 'suffix': 1, + 'super': 0, + 'temp-dir': 2, + 'timeout': 1, + 'times': 0, + 'use-qsort': 0, + 'usermap': 1, + 'write-devices': -1, +} ### END of options data produced by the cull_options script. ### -if ($only eq 'r') { - foreach my $opt (keys %long_opt) { - if ($opt =~ /^(remove-|log-file)/) { - $long_opt{$opt} = -1; - } - } -} elsif ($only eq 'w') { - $long_opt{'sender'} = -1; -} +import os, sys, re, argparse, glob, socket, time +from argparse import RawTextHelpFormatter -if ($short_disabled ne '') { - $short_no_arg =~ s/[$short_disabled]//go; - $short_with_num =~ s/[$short_disabled]//go; -} -$short_no_arg = "[$short_no_arg]" if length($short_no_arg) > 1; -$short_with_num = "[$short_with_num]" if length($short_with_num) > 1; - -my $write_log = -f LOGFILE && open(LOG, '>>', LOGFILE); - -chdir($subdir) or die "$0: Unable to chdir to restricted dir: $!\n"; - -my(@opts, @args); -my $in_options = 1; -my $last_opt = ''; -my $check_type; -while ($command =~ /((?:[^\s\\]+|\\.[^\s\\]*)+)/g) { - $_ = $1; - if ($check_type) { - push(@opts, check_arg($last_opt, $_, $check_type)); - $check_type = 0; - } elsif ($in_options) { - if ($_ eq '.') { - $in_options = 0; - } else { - die "$0: invalid option: '-'\n" if $_ eq '-'; - push(@opts, $_); - next if /^-$short_no_arg*(e\d*\.\w*)?$/o || /^-$short_with_num\d+$/o; - - my($opt,$arg) = /^--([^=]+)(?:=(.*))?$/; - my $disabled; - if (defined $opt) { - my $ct = $long_opt{$opt}; - last unless defined $ct; - next if $ct == 0; - if ($ct > 0) { - if (!defined $arg) { - $check_type = $ct; - $last_opt = $opt; - next; - } - $arg = check_arg($opt, $arg, $ct); - $opts[-1] =~ s/=.*/=$arg/; - next; - } - $disabled = 1; - $opt = "--$opt"; - } elsif ($short_disabled ne '') { - $disabled = /^-$short_no_arg*([$short_disabled])/o; - $opt = "-$1"; - } - - last unless $disabled; # Generate generic failure - die "$0: option $opt has been disabled on this server.\n"; - } - } else { - if ($subdir ne '/') { - # Validate args to ensure they don't try to leave our restricted dir. - s{//+}{/}g; - s{^/}{}; - s{^$}{.}; - } - push(@args, bsd_glob($_, GLOB_LIMIT|GLOB_NOCHECK|GLOB_BRACE|GLOB_QUOTE)); - } -} -die "$0: invalid rsync-command syntax or options\n" if $in_options; +try: + from braceexpand import braceexpand +except: + braceexpand = lambda x: [ DE_BACKSLASH_RE.sub(r'\1', x) ] -if ($subdir ne '/') { - die "$0: do not use .. in any path!\n" if grep m{(^|/)\.\.(/|$)}, @args; -} +HAS_DOT_DOT_RE = re.compile(r'(^|/)\.\.(/|$)') +LONG_OPT_RE = re.compile(r'^--([^=]+)(?:=(.*))?$') +DE_BACKSLASH_RE = re.compile(r'\\(.)') -if ($force_munge) { - push(@opts, '--munge-links'); -} +def main(): + if not os.path.isdir(args.dir): + die("Restricted directory does not exist!") -@args = ( '.' ) if !@args; + # The format of the environment variables set by sshd: + # SSH_ORIGINAL_COMMAND: + # rsync --server -vlogDtpre.iLsfxCIvu --etc . ARG # push + # rsync --server --sender -vlogDtpre.iLsfxCIvu --etc . ARGS # pull + # SSH_CONNECTION (client_ip client_port server_ip server_port): + # 192.168.1.100 64106 192.168.1.2 22 -if ($write_log) { - my ($mm,$hh) = (localtime)[1,2]; - my $host = $ENV{SSH_CONNECTION} || 'unknown'; - $host =~ s/ .*//; # Keep only the client's IP addr - $host =~ s/^::ffff://; - $host = gethostbyaddr(inet_aton($host),AF_INET) || $host; - printf LOG "%02d:%02d %-13s [%s]\n", $hh, $mm, $host, "@opts @args"; - close LOG; -} + command = os.environ.get('SSH_ORIGINAL_COMMAND', None) + if not command: + die("Not invoked via sshd") + command = command.split(' ', 2) + if command[0:1] != ['rsync']: + die("SSH_ORIGINAL_COMMAND does not run rsync") + if command[1:2] != ['--server']: + die("--server option is not the first arg") + command = '' if len(command) < 3 else command[2] -# Note: This assumes that the rsync protocol will not be maliciously hijacked. -exec(RSYNC, @opts, '--', '.', @args) or die "exec(rsync @opts -- . @args) failed: $? $!"; - -sub check_arg -{ - my($opt, $arg, $type) = @_; - $arg =~ s/\\(.)/$1/g; - if ($subdir ne '/' && ($type == 3 || ($type == 2 && !$am_sender))) { - $arg =~ s{//}{/}g; - die "Do not use .. in --$opt; anchor the path at the root of your restricted dir.\n" - if $arg =~ m{(^|/)\.\.(/|$)}; - $arg =~ s{^/}{$subdir/}; - } - $arg; -} + global am_sender + am_sender = command.startswith("--sender ") # Restrictive on purpose! + if args.ro and not am_sender: + die("sending to read-only server is not allowed") + if args.wo and am_sender: + die("reading from write-only server is not allowed") + + if args.wo or not am_sender: + long_opts['sender'] = -1 + if args.no_del: + for opt in long_opts: + if opt.startswith(('remove', 'delete')): + long_opts[opt] = -1 + if args.ro: + long_opts['log-file'] = -1 + + short_no_arg_re = short_no_arg + short_with_num_re = short_with_num + if short_disabled: + for ltr in short_disabled: + short_no_arg_re = short_no_arg_re.replace(ltr, '') + short_with_num_re = short_with_num_re.replace(ltr, '') + short_disabled_re = re.compile(r'^-[%s]*([%s])' % (short_no_arg_re, short_disabled)) + short_no_arg_re = re.compile(r'^-(?=.)[%s]*(e\d*\.\w*)?$' % short_no_arg_re) + short_with_num_re = re.compile(r'^-[%s]\d+$' % short_with_num_re) + + log_fh = open(LOGFILE, 'a') if os.path.isfile(LOGFILE) else None + + try: + os.chdir(args.dir) + except OSError as e: + die('unable to chdir to restricted dir:', str(e)) + + rsync_opts = [ '--server' ] + rsync_args = [ ] + saw_the_dot_arg = False + last_opt = check_type = None + + for arg in re.findall(r'(?:[^\s\\]+|\\.[^\s\\]*)+', command): + if check_type: + rsync_opts.append(validated_arg(last_opt, arg, check_type)) + check_type = None + elif saw_the_dot_arg: + # NOTE: an arg that starts with a '-' is safe due to our use of "--" in the cmd tuple. + try: + b_e = braceexpand(arg) # Also removes backslashes + except: # Handle errors such as unbalanced braces by just de-backslashing the arg: + b_e = [ DE_BACKSLASH_RE.sub(r'\1', arg) ] + for xarg in b_e: + rsync_args += validated_arg('arg', xarg, wild=True) + else: # parsing the option args + if arg == '.': + saw_the_dot_arg = True + continue + rsync_opts.append(arg) + if short_no_arg_re.match(arg) or short_with_num_re.match(arg): + continue + disabled = False + m = LONG_OPT_RE.match(arg) + if m: + opt = m.group(1) + opt_arg = m.group(2) + ct = long_opts.get(opt, None) + if ct is None: + break # Generate generic failure due to unfinished arg parsing + if ct == 0: + continue + opt = '--' + opt + if ct > 0: + if opt_arg is not None: + rsync_opts[-1] = opt + '=' + validated_arg(opt, opt_arg, ct) + else: + check_type = ct + last_opt = opt + continue + disabled = True + elif short_disabled: + m = short_disabled_re.match(arg) + if m: + disabled = True + opt = '-' + m.group(1) + + if disabled: + die("option", opt, "has been disabled on this server.") + break # Generate a generic failure + + if not saw_the_dot_arg: + die("invalid rsync-command syntax or options") + + if args.munge: + rsync_opts.append('--munge-links') + + if not rsync_args: + rsync_args = [ '.' ] + + cmd = (RSYNC, *rsync_opts, '--', '.', *rsync_args) + + if log_fh: + now = time.localtime() + host = os.environ.get('SSH_CONNECTION', 'unknown').split()[0] # Drop everything after the IP addr + if host.startswith('::ffff:'): + host = host[7:] + try: + host = socket.gethostbyaddr(socket.inet_aton(host)) + except: + pass + log_fh.write("%02d:%02d:%02d %-16s %s\n" % (now.tm_hour, now.tm_min, now.tm_sec, host, str(cmd))) + log_fh.close() + + # NOTE: This assumes that the rsync protocol will not be maliciously hijacked. + os.execlp(RSYNC, *cmd) + die("execlp(", RSYNC, *cmd, ') failed') + + +def validated_arg(opt, arg, typ=3, wild=False): + if opt != 'arg': # arg values already have their backslashes removed. + arg = DE_BACKSLASH_RE.sub(r'\1', arg) + + orig_arg = arg + if arg.startswith('./'): + arg = arg[1:] + arg = arg.replace('//', '/') + if args.dir != '/': + if HAS_DOT_DOT_RE.search(arg): + die("do not use .. in", opt, "(anchor the path at the root of your restricted dir)") + if arg.startswith('/'): + arg = args.dir + arg + + if wild: + got = glob.glob(arg) + if not got: + got = [ arg ] + else: + got = [ arg ] + + ret = [ ] + for arg in got: + if args.dir != '/' and arg != '.' and (typ == 3 or (typ == 2 and not am_sender)): + arg_has_trailing_slash = arg.endswith('/') + if arg_has_trailing_slash: + arg = arg[:-1] + else: + arg_has_trailing_slash_dot = arg.endswith('/.') + if arg_has_trailing_slash_dot: + arg = arg[:-2] + real_arg = os.path.realpath(arg) + if arg != real_arg and not real_arg.startswith(args.dir_slash): + die('unsafe arg:', orig_arg, [arg, real_arg]) + if arg_has_trailing_slash: + arg += '/' + elif arg_has_trailing_slash_dot: + arg += '/.' + if opt == 'arg' and arg.startswith(args.dir_slash): + arg = arg[args.dir_slash_len:] + if arg == '': + arg = '.' + ret.append(arg) + + return ret if wild else ret[0] + + +def die(*msg): + print(sys.argv[0], 'error:', *msg, file=sys.stderr) + if sys.stdin.isatty(): + arg_parser.print_help(sys.stderr) + sys.exit(1) + + +# This class displays the --help to the user on argparse error IFF they're running it interactively. +class OurArgParser(argparse.ArgumentParser): + def error(self, msg): + die(msg) + + +if __name__ == '__main__': + our_desc = """Use "man rrsync" to learn how to restrict ssh users to using a restricted rsync command.""" + arg_parser = OurArgParser(description=our_desc, add_help=False) + only_group = arg_parser.add_mutually_exclusive_group() + only_group.add_argument('-ro', action='store_true', help="Allow only reading from the DIR. Implies -no-del.") + only_group.add_argument('-wo', action='store_true', help="Allow only writing to the DIR.") + arg_parser.add_argument('-no-del', action='store_true', help="Disable rsync's --delete* and --remove* options.") + arg_parser.add_argument('-munge', action='store_true', help="Enable rsync's --munge-links on the server side.") + arg_parser.add_argument('-help', '-h', action='help', help="Output this help message and exit.") + arg_parser.add_argument('dir', metavar='DIR', help="The restricted directory to use.") + args = arg_parser.parse_args() + args.dir = os.path.realpath(args.dir) + args.dir_slash = args.dir + '/' + args.dir_slash_len = len(args.dir) + if args.ro: + args.no_del = True + main() -# vim: sw=2 +# vim: sw=4 et diff --git a/support/rrsync.1.md b/support/rrsync.1.md new file mode 100644 index 000000000..b945ecf07 --- /dev/null +++ b/support/rrsync.1.md @@ -0,0 +1,89 @@ +# NAME + +rrsync - a script to setup restricted rsync users via ssh logins + +# SYNOPSIS + +``` +rrsync [-ro|-rw] [-munge] [-no-del] DIR +``` + +# DESCRIPTION + +A user's ssh login can be restricted to only allow the running of an rsync +transfer in one of two easy ways: forcing the running of the rrsync script +or forcing the running of an rsync daemon-over-ssh command. + +To use the rrsync script, add a prefix like one of the following (followed by a +space) in front of each ssh-key line in the user's `~/.ssh/authorized_keys` +file that should be restricted: + +> ``` +> command="rrsync DIR" +> command="rrsync -ro DIR" +> command="rrsync -munge -no-del DIR" +> ``` + +Then, ensure that the rrsync script has your desired option restrictions. You +may want to copy the script to a local bin dir with a unique name if you want +to have multiple configurations. One or more rrsync options can be specified +prior to the `DIR` if you want to further restrict the transfer. + +To use an rsync daemon setup, add one of the following prefixes (followed by a +space) in front of each ssh-key line in the user's `~/.ssh/authorized_keys` +file that should be restricted: + +> ``` +> command="rsync --server --daemon ." +> command="rsync --server --daemon --config=/PATH/TO/rsyncd.conf ." +> ``` + +Then, ensure that the rsyncd.conf file is created with one or more module names +with the appropriate path and option restrictions. If the `--config` option is +omitted, it defaults to `~/rsyncd.conf`. See the `rsyncd.conf` man page for +details of how to configure an rsync daemon. + +The remainder of this man page is dedicated to using the rrsync script. + +# OPTION SUMMARY + +``` +-ro Allow only reading from the DIR. Implies -no-del. +-wo Allow only writing to the DIR. +-no-del Disable rsync's --delete* and --remove* options. +-munge Enable rsync's --munge-links on the server side. +-help, -h Output this help message and exit. +``` + +A single non-option argument specifies the restricted DIR to use. It can be +relative to the user's home directory or an absolute path. + +# SECURITY RESTRICTIONS + +The rrsync script validates the path arguments it is sent to try to restrict +them to staying within the specified DIR. + +The rrsync script rejects rsync's `--copy-links`` option (by default) so that a +copy cannot dereference a symlink within the DIR to get to a file outside the +DIR. + +The rrsync script rejects rsync's `--protect-args` (`-s`) option because it +would allow options to be sent to the server-side that the script could not +check. If you want to support `--protect-args`, use a daemon-over-ssh setup. + +The rrsync script accepts just a subset of rsync's options that the real rsync +uses when running the server command. A few extra convenience options are also +included to help it to interact with BackupPC and accept some convenient user +overrides. + +The script (or a copy of it) can be manually edited if you want it to customize +the option handling. + +# EXAMPLES + +The `.ssh/authorized_keys` file might have lines in it like this: + +> ``` +> command="rrsync client/logs" ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEAzG... +> command="rrsync -ro results" ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEAmk... +> ``` From 512acd125e450b748cd2c8f6b02f8e01e0e3a70e Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Sun, 26 Dec 2021 14:23:19 -0800 Subject: [PATCH 014/350] Use mallinfo2, when available, and use %zd for size_t values on C99. An exhanced version of pull request #265. --- configure.ac | 2 +- io.c | 106 +++++++++++++++++++++++++++++++-------------------- main.c | 49 +++++++++++------------- rsync.h | 11 ++++++ 4 files changed, 98 insertions(+), 70 deletions(-) diff --git a/configure.ac b/configure.ac index 84111de8b..b30eeb933 100644 --- a/configure.ac +++ b/configure.ac @@ -829,7 +829,7 @@ AC_CHECK_FUNCS(waitpid wait4 getcwd chown chmod lchmod mknod mkfifo \ fchmod fstat ftruncate strchr readlink link utime utimes lutimes strftime \ chflags getattrlist mktime innetgr linkat \ memmove lchown vsnprintf snprintf vasprintf asprintf setsid strpbrk \ - strlcat strlcpy strtol mallinfo getgroups setgroups geteuid getegid \ + strlcat strlcpy strtol mallinfo mallinfo2 getgroups setgroups geteuid getegid \ setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \ seteuid strerror putenv iconv_open locale_charset nl_langinfo getxattr \ extattr_get_link sigaction sigprocmask setattrlist getgrouplist \ diff --git a/io.c b/io.c index cc70ce887..abb0f7ab1 100644 --- a/io.c +++ b/io.c @@ -264,15 +264,18 @@ static size_t safe_read(int fd, char *buf, size_t len) rprintf(FINFO, "select exception on fd %d\n", fd); */ if (FD_ISSET(fd, &r_fds)) { - int n = read(fd, buf + got, len - got); - if (DEBUG_GTE(IO, 2)) - rprintf(FINFO, "[%s] safe_read(%d)=%ld\n", who_am_i(), fd, (long)n); + ssize_t n = read(fd, buf + got, len - got); + if (DEBUG_GTE(IO, 2)) { + rprintf(FINFO, "[%s] safe_read(%d)=%" SIZE_T_FMT_MOD "d\n", + who_am_i(), fd, (SIZE_T_FMT_CAST)n); + } if (n == 0) break; if (n < 0) { if (errno == EINTR) continue; - rsyserr(FERROR, errno, "safe_read failed to read %ld bytes", (long)len); + rsyserr(FERROR, errno, "safe_read failed to read %" SIZE_T_FMT_MOD "d bytes", + (SIZE_T_FMT_CAST)len); exit_cleanup(RERR_STREAMIO); } if ((got += (size_t)n) == len) @@ -304,7 +307,7 @@ static const char *what_fd_is(int fd) * is not used on the socket except very early in the transfer. */ static void safe_write(int fd, const char *buf, size_t len) { - int n; + ssize_t n; assert(fd != iobuf.out_fd); @@ -315,8 +318,8 @@ static void safe_write(int fd, const char *buf, size_t len) if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN) { write_failed: rsyserr(FERROR, errno, - "safe_write failed to write %ld bytes to %s", - (long)len, what_fd_is(fd)); + "safe_write failed to write %" SIZE_T_FMT_MOD "d bytes to %s", + (SIZE_T_FMT_CAST)len, what_fd_is(fd)); exit_cleanup(RERR_STREAMIO); } } else { @@ -362,7 +365,7 @@ static void safe_write(int fd, const char *buf, size_t len) * a chunk of data and put it into the output buffer. */ static void forward_filesfrom_data(void) { - int len; + ssize_t len; len = read(ff_forward_fd, ff_xb.buf + ff_xb.len, ff_xb.size - ff_xb.len); if (len <= 0) { @@ -377,8 +380,10 @@ static void forward_filesfrom_data(void) return; } - if (DEBUG_GTE(IO, 2)) - rprintf(FINFO, "[%s] files-from read=%ld\n", who_am_i(), (long)len); + if (DEBUG_GTE(IO, 2)) { + rprintf(FINFO, "[%s] files-from read=%" SIZE_T_FMT_MOD "d\n", + who_am_i(), (SIZE_T_FMT_CAST)len); + } #ifdef ICONV_OPTION len += ff_xb.len; @@ -562,52 +567,59 @@ static char *perform_io(size_t needed, int flags) case PIO_NEED_INPUT: /* We never resize the circular input buffer. */ if (iobuf.in.size < needed) { - rprintf(FERROR, "need to read %ld bytes, iobuf.in.buf is only %ld bytes.\n", - (long)needed, (long)iobuf.in.size); + rprintf(FERROR, "need to read %" SIZE_T_FMT_MOD "d bytes," + " iobuf.in.buf is only %" SIZE_T_FMT_MOD "d bytes.\n", + (SIZE_T_FMT_CAST)needed, (SIZE_T_FMT_CAST)iobuf.in.size); exit_cleanup(RERR_PROTOCOL); } if (msgs2stderr == 1 && DEBUG_GTE(IO, 3)) { - rprintf(FINFO, "[%s] perform_io(%ld, %sinput)\n", - who_am_i(), (long)needed, flags & PIO_CONSUME_INPUT ? "consume&" : ""); + rprintf(FINFO, "[%s] perform_io(%" SIZE_T_FMT_MOD "d, %sinput)\n", + who_am_i(), (SIZE_T_FMT_CAST)needed, flags & PIO_CONSUME_INPUT ? "consume&" : ""); } break; case PIO_NEED_OUTROOM: /* We never resize the circular output buffer. */ if (iobuf.out.size - iobuf.out_empty_len < needed) { - fprintf(stderr, "need to write %ld bytes, iobuf.out.buf is only %ld bytes.\n", - (long)needed, (long)(iobuf.out.size - iobuf.out_empty_len)); + fprintf(stderr, "need to write %" SIZE_T_FMT_MOD "d bytes," + " iobuf.out.buf is only %" SIZE_T_FMT_MOD "d bytes.\n", + (SIZE_T_FMT_CAST)needed, (SIZE_T_FMT_CAST)(iobuf.out.size - iobuf.out_empty_len)); exit_cleanup(RERR_PROTOCOL); } if (msgs2stderr == 1 && DEBUG_GTE(IO, 3)) { - rprintf(FINFO, "[%s] perform_io(%ld, outroom) needs to flush %ld\n", - who_am_i(), (long)needed, + rprintf(FINFO, "[%s] perform_io(%" SIZE_T_FMT_MOD "d," + " outroom) needs to flush %" SIZE_T_FMT_MOD "d\n", + who_am_i(), (SIZE_T_FMT_CAST)needed, iobuf.out.len + needed > iobuf.out.size - ? (long)(iobuf.out.len + needed - iobuf.out.size) : 0L); + ? (SIZE_T_FMT_CAST)(iobuf.out.len + needed - iobuf.out.size) : (SIZE_T_FMT_CAST)0); } break; case PIO_NEED_MSGROOM: /* We never resize the circular message buffer. */ if (iobuf.msg.size < needed) { - fprintf(stderr, "need to write %ld bytes, iobuf.msg.buf is only %ld bytes.\n", - (long)needed, (long)iobuf.msg.size); + fprintf(stderr, "need to write %" SIZE_T_FMT_MOD "d bytes," + " iobuf.msg.buf is only %" SIZE_T_FMT_MOD "d bytes.\n", + (SIZE_T_FMT_CAST)needed, (SIZE_T_FMT_CAST)iobuf.msg.size); exit_cleanup(RERR_PROTOCOL); } if (msgs2stderr == 1 && DEBUG_GTE(IO, 3)) { - rprintf(FINFO, "[%s] perform_io(%ld, msgroom) needs to flush %ld\n", - who_am_i(), (long)needed, + rprintf(FINFO, "[%s] perform_io(%" SIZE_T_FMT_MOD "d," + " msgroom) needs to flush %" SIZE_T_FMT_MOD "d\n", + who_am_i(), (SIZE_T_FMT_CAST)needed, iobuf.msg.len + needed > iobuf.msg.size - ? (long)(iobuf.msg.len + needed - iobuf.msg.size) : 0L); + ? (SIZE_T_FMT_CAST)(iobuf.msg.len + needed - iobuf.msg.size) : (SIZE_T_FMT_CAST)0); } break; case 0: - if (msgs2stderr == 1 && DEBUG_GTE(IO, 3)) - rprintf(FINFO, "[%s] perform_io(%ld, %d)\n", who_am_i(), (long)needed, flags); + if (msgs2stderr == 1 && DEBUG_GTE(IO, 3)) { + rprintf(FINFO, "[%s] perform_io(%" SIZE_T_FMT_MOD "d, %d)\n", + who_am_i(), (SIZE_T_FMT_CAST)needed, flags); + } break; default: @@ -665,8 +677,8 @@ static char *perform_io(size_t needed, int flags) ((MPLEX_BASE + (int)MSG_DATA)<<24) + iobuf.out.len - 4); if (msgs2stderr == 1 && DEBUG_GTE(IO, 1)) { - rprintf(FINFO, "[%s] send_msg(%d, %ld)\n", - who_am_i(), (int)MSG_DATA, (long)iobuf.out.len - 4); + rprintf(FINFO, "[%s] send_msg(%d, %" SIZE_T_FMT_MOD "d)\n", + who_am_i(), (int)MSG_DATA, (SIZE_T_FMT_CAST)iobuf.out.len - 4); } /* reserve room for the next MSG_DATA header */ @@ -757,7 +769,7 @@ static char *perform_io(size_t needed, int flags) if (iobuf.in_fd >= 0 && FD_ISSET(iobuf.in_fd, &r_fds)) { size_t len, pos = iobuf.in.pos + iobuf.in.len; - int n; + ssize_t n; if (pos >= iobuf.in.size) { pos -= iobuf.in.size; len = iobuf.in.size - iobuf.in.len; @@ -784,8 +796,10 @@ static char *perform_io(size_t needed, int flags) exit_cleanup(RERR_SOCKETIO); } } - if (msgs2stderr == 1 && DEBUG_GTE(IO, 2)) - rprintf(FINFO, "[%s] recv=%ld\n", who_am_i(), (long)n); + if (msgs2stderr == 1 && DEBUG_GTE(IO, 2)) { + rprintf(FINFO, "[%s] recv=%" SIZE_T_FMT_MOD "d\n", + who_am_i(), (SIZE_T_FMT_CAST)n); + } if (io_timeout || stop_at_utime) { last_io_in = time(NULL); @@ -803,7 +817,7 @@ static char *perform_io(size_t needed, int flags) if (out && FD_ISSET(iobuf.out_fd, &w_fds)) { size_t len = iobuf.raw_flushing_ends_before ? iobuf.raw_flushing_ends_before - out->pos : out->len; - int n; + ssize_t n; if (bwlimit_writemax && len > bwlimit_writemax) len = bwlimit_writemax; @@ -824,8 +838,8 @@ static char *perform_io(size_t needed, int flags) } } if (msgs2stderr == 1 && DEBUG_GTE(IO, 2)) { - rprintf(FINFO, "[%s] %s sent=%ld\n", - who_am_i(), out == &iobuf.out ? "out" : "msg", (long)n); + rprintf(FINFO, "[%s] %s sent=%" SIZE_T_FMT_MOD "d\n", + who_am_i(), out == &iobuf.out ? "out" : "msg", (SIZE_T_FMT_CAST)n); } if (io_timeout) @@ -945,8 +959,10 @@ int send_msg(enum msgcode code, const char *buf, size_t len, int convert) if (!OUT_MULTIPLEXED) return 0; - if (want_debug) - rprintf(FINFO, "[%s] send_msg(%d, %ld)\n", who_am_i(), (int)code, (long)len); + if (want_debug) { + rprintf(FINFO, "[%s] send_msg(%d, %" SIZE_T_FMT_MOD "d)\n", + who_am_i(), (int)code, (SIZE_T_FMT_CAST)len); + } /* When checking for enough free space for this message, we need to * make sure that there is space for the 4-byte header, plus we'll @@ -1021,8 +1037,10 @@ int send_msg(enum msgcode code, const char *buf, size_t len, int convert) SIVAL(hdr, 0, ((MPLEX_BASE + (int)code)<<24) + len); - if (want_debug && convert > 0) - rprintf(FINFO, "[%s] converted msg len=%ld\n", who_am_i(), (long)len); + if (want_debug && convert > 0) { + rprintf(FINFO, "[%s] converted msg len=%" SIZE_T_FMT_MOD "d\n", + who_am_i(), (SIZE_T_FMT_CAST)len); + } return 1; } @@ -1436,8 +1454,10 @@ static void read_a_msg(void) msg_bytes = tag & 0xFFFFFF; tag = (tag >> 24) - MPLEX_BASE; - if (msgs2stderr == 1 && DEBUG_GTE(IO, 1)) - rprintf(FINFO, "[%s] got msg=%d, len=%ld\n", who_am_i(), (int)tag, (long)msg_bytes); + if (msgs2stderr == 1 && DEBUG_GTE(IO, 1)) { + rprintf(FINFO, "[%s] got msg=%d, len=%" SIZE_T_FMT_MOD "d\n", + who_am_i(), (int)tag, (SIZE_T_FMT_CAST)msg_bytes); + } switch (tag) { case MSG_DATA: @@ -1613,8 +1633,10 @@ static void read_a_msg(void) else goto invalid_msg; iobuf.in_multiplexed = 1; - if (DEBUG_GTE(EXIT, 3)) - rprintf(FINFO, "[%s] got MSG_ERROR_EXIT with %ld bytes\n", who_am_i(), (long)msg_bytes); + if (DEBUG_GTE(EXIT, 3)) { + rprintf(FINFO, "[%s] got MSG_ERROR_EXIT with %" SIZE_T_FMT_MOD "d bytes\n", + who_am_i(), (SIZE_T_FMT_CAST)msg_bytes); + } if (msg_bytes == 0) { if (!am_sender && !am_generator) { if (DEBUG_GTE(EXIT, 3)) { diff --git a/main.c b/main.c index 013a05553..8b145867d 100644 --- a/main.c +++ b/main.c @@ -467,38 +467,33 @@ static void output_summary(void) **/ static void show_malloc_stats(void) { -#ifdef HAVE_MALLINFO - struct mallinfo mi; - - mi = mallinfo(); +#ifdef MEM_ALLOC_INFO + struct MEM_ALLOC_INFO mi = MEM_ALLOC_INFO(); /* mallinfo or mallinfo2 */ rprintf(FCLIENT, "\n"); rprintf(FINFO, RSYNC_NAME "[%d] (%s%s%s) heap statistics:\n", (int)getpid(), am_server ? "server " : "", am_daemon ? "daemon " : "", who_am_i()); - rprintf(FINFO, " arena: %10ld (bytes from sbrk)\n", - (long)mi.arena); - rprintf(FINFO, " ordblks: %10ld (chunks not in use)\n", - (long)mi.ordblks); - rprintf(FINFO, " smblks: %10ld\n", - (long)mi.smblks); - rprintf(FINFO, " hblks: %10ld (chunks from mmap)\n", - (long)mi.hblks); - rprintf(FINFO, " hblkhd: %10ld (bytes from mmap)\n", - (long)mi.hblkhd); - rprintf(FINFO, " allmem: %10ld (bytes from sbrk + mmap)\n", - (long)mi.arena + mi.hblkhd); - rprintf(FINFO, " usmblks: %10ld\n", - (long)mi.usmblks); - rprintf(FINFO, " fsmblks: %10ld\n", - (long)mi.fsmblks); - rprintf(FINFO, " uordblks: %10ld (bytes used)\n", - (long)mi.uordblks); - rprintf(FINFO, " fordblks: %10ld (bytes free)\n", - (long)mi.fordblks); - rprintf(FINFO, " keepcost: %10ld (bytes in releasable chunk)\n", - (long)mi.keepcost); -#endif /* HAVE_MALLINFO */ + +#define PRINT_ALLOC_NUM(title, descr, num) \ + rprintf(FINFO, " %-11s%10" SIZE_T_FMT_MOD "d (" descr ")\n", \ + title ":", (SIZE_T_FMT_CAST)(num)); + + PRINT_ALLOC_NUM("arena", "bytes from sbrk", mi.arena); + PRINT_ALLOC_NUM("ordblks", "chunks not in use", mi.ordblks); + PRINT_ALLOC_NUM("smblks", "free fastbin blocks", mi.smblks); + PRINT_ALLOC_NUM("hblks", "chunks from mmap", mi.hblks); + PRINT_ALLOC_NUM("hblkhd", "bytes from mmap", mi.hblkhd); + PRINT_ALLOC_NUM("allmem", "bytes from sbrk + mmap", mi.arena + mi.hblkhd); + PRINT_ALLOC_NUM("usmblks", "always 0", mi.usmblks); + PRINT_ALLOC_NUM("fsmblks", "bytes in freed fastbin blocks", mi.fsmblks); + PRINT_ALLOC_NUM("uordblks", "bytes used", mi.uordblks); + PRINT_ALLOC_NUM("fordblks", "bytes free", mi.fordblks); + PRINT_ALLOC_NUM("keepcost", "bytes in releasable chunk", mi.keepcost); + +#undef PRINT_ALLOC_NUM + +#endif /* MEM_ALLOC_INFO */ } diff --git a/rsync.h b/rsync.h index 86105afe3..ea1bfea6a 100644 --- a/rsync.h +++ b/rsync.h @@ -780,6 +780,11 @@ struct ht_int64_node { #if defined __STDC_VERSION__ && __STDC_VERSION__ >= 199901L #define USE_FLEXIBLE_ARRAY 1 +#define SIZE_T_FMT_MOD "z" /* printf supports %zd */ +#define SIZE_T_FMT_CAST size_t +#else +#define SIZE_T_FMT_MOD "l" /* printf supports %ld */ +#define SIZE_T_FMT_CAST long #endif union file_extras { @@ -1473,3 +1478,9 @@ const char *get_panic_action(void); exit_cleanup(RERR_UNSUPPORTED); \ } while (0) #endif /* AVX2_ASM */ + +#ifdef HAVE_MALLINFO2 +#define MEM_ALLOC_INFO mallinfo2 +#elif defined HAVE_MALLINFO +#define MEM_ALLOC_INFO mallinfo +#endif From a76e32f9491889c378e99d651ffb90cdc8fe20cf Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Sun, 26 Dec 2021 14:48:55 -0800 Subject: [PATCH 015/350] Test --with-rrsync configure option & put rrsync into the artifacts. --- .github/workflows/build.yml | 18 ++++++++++++------ Makefile.in | 3 +++ packaging/auto-Makefile | 2 +- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9e34b4c97..4f5cbf23d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,9 +22,9 @@ jobs: wget -O git-version.h https://gist.githubusercontent.com/WayneD/c11243fa374fc64d4e42f2855c8e3827/raw/rsync-git-version.h echo "/usr/local/bin" >>$GITHUB_PATH - name: configure - run: ./configure + run: ./configure --with-rrsync - name: make - run: make + run: make all rrsync - name: install run: sudo make install - name: info @@ -47,6 +47,8 @@ jobs: rsync.1 rsync-ssl.1 rsyncd.conf.5 + rrsync.1 + rrsync macos-build: runs-on: macos-latest @@ -59,9 +61,9 @@ jobs: wget -O git-version.h https://gist.githubusercontent.com/WayneD/c11243fa374fc64d4e42f2855c8e3827/raw/rsync-git-version.h echo "/usr/local/bin" >>$GITHUB_PATH - name: configure - run: CPPFLAGS=-I/usr/local/opt/openssl/include/ LDFLAGS=-L/usr/local/opt/openssl/lib/ ./configure + run: CPPFLAGS=-I/usr/local/opt/openssl/include/ LDFLAGS=-L/usr/local/opt/openssl/lib/ ./configure --with-rrsync - name: make - run: make + run: make all rrsync - name: install run: sudo make install - name: info @@ -80,6 +82,8 @@ jobs: rsync.1 rsync-ssl.1 rsyncd.conf.5 + rrsync.1 + rrsync cygwin-build: runs-on: windows-latest @@ -97,9 +101,9 @@ jobs: - name: commonmark run: bash -c 'python3 -mpip install --user commonmark' - name: configure - run: bash -c './configure' + run: bash -c './configure --with-rrsync' - name: make - run: bash -c 'make' + run: bash -c 'make all rrsync' - name: install run: bash -c 'make install' - name: info @@ -118,3 +122,5 @@ jobs: rsync.1 rsync-ssl.1 rsyncd.conf.5 + rrsync.1 + rrsync diff --git a/Makefile.in b/Makefile.in index 5eed339e3..98d5a7af8 100644 --- a/Makefile.in +++ b/Makefile.in @@ -101,6 +101,9 @@ install-strip: rsync$(EXEEXT): $(OBJS) $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(OBJS) $(LIBS) +rrsync: support/rrsync + cp -p $(srcdir)/support/rrsync rrsync + $(OBJS): $(HEADERS) $(CHECK_OBJS): $(HEADERS) tls.o xattrs.o: lib/sysxattrs.h diff --git a/packaging/auto-Makefile b/packaging/auto-Makefile index 29d2d6889..7f2e2585c 100644 --- a/packaging/auto-Makefile +++ b/packaging/auto-Makefile @@ -1,6 +1,6 @@ TARGETS := all install install-ssl-daemon install-all install-strip conf gen gensend reconfigure restatus \ proto man clean cleantests distclean test check check29 check30 installcheck splint \ - doxygen doxygen-upload finddead + doxygen doxygen-upload finddead rrsync .PHONY: $(TARGETS) auto-prep From b00e99c52953ca87bf2ac4523a1e04335af5bd4a Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Mon, 27 Dec 2021 12:10:31 -0800 Subject: [PATCH 016/350] Ignore the built rrsync man-page files. --- .gitignore | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 13c6db8dc..852901166 100644 --- a/.gitignore +++ b/.gitignore @@ -15,9 +15,10 @@ config.status aclocal.m4 /proto.h /proto.h-tstamp -/rsync*.1 -/rsync*.5 +/rsync*.[15] +/rrsync*.1 /rsync*.html +/rrsync*.html /help-rsync*.h /default-cvsignore.h /default-dont-compress.h From 7f8cf771b733f700259ce4c30ef7e3fe43b478b5 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Mon, 27 Dec 2021 13:11:23 -0800 Subject: [PATCH 017/350] Add more backticks. --- INSTALL.md | 8 ++++---- NEWS.md | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 1f373c691..6a95c451f 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -178,9 +178,9 @@ config.h, or just override them in your /etc/rsyncd.conf file. As of 2.4.7, rsync uses Eric Troan's popt option-parsing library. A cut-down copy of a recent release is included in the rsync distribution, and will be used if there is no popt library on your build host, or if -the --with-included-popt option is passed to ./configure. +the `--with-included-popt` option is passed to ./configure. -If you configure using --enable-maintainer-mode, then rsync will try +If you configure using `--enable-maintainer-mode`, then rsync will try to pop up an xterm on DISPLAY=:0 if it crashes. You might find this useful, but it should be turned off for production builds. @@ -194,7 +194,7 @@ This is helpful when using the branch-from-patch and patch-update scripts to maintain the official rsync patches. If you ever need to build from a "detached head" git position then you'll need to manually chdir into the build dir to run make. I also like to create 2 more symlinks in the -source dir: ln -s build/rsync . ; ln -s build/testtmp . +source dir: `ln -s build/rsync . ; ln -s build/testtmp .` ## Make compatibility @@ -228,7 +228,7 @@ Some versions of Mac OS X (Darwin) seem to have an IPv6 stack, but do not completely implement the "New Sockets" API. [This site][5] says that Apple started to support IPv6 in 10.2 (Jaguar). If -your build fails, try again after running configure with --disable-ipv6. +your build fails, try again after running configure with `--disable-ipv6`. [5]: http://www.ipv6.org/impl/mac.html diff --git a/NEWS.md b/NEWS.md index b3002e893..5994fff64 100644 --- a/NEWS.md +++ b/NEWS.md @@ -78,7 +78,7 @@ - The rsync daemon can now handle a client address with an implied "%scope" suffix. - - Added support for `--atimes` on macOS and fixed using using it without -t. + - Added support for `--atimes` on macOS and fixed using using it without `-t`. - Rsync can now update the xattrs on a read-only file when your user can temporarily add user-write permission to the file. (It always worked for a @@ -113,7 +113,7 @@ ### PACKAGING RELATED: - - Give configure the --with-rrsync option if you want `make install` to + - Give configure the `--with-rrsync` option if you want `make install` to install the (now python3) rrsync script and its (new) man page. - If the rrsync script is installed, make its package depend on python3 and @@ -1327,7 +1327,7 @@ that hasn't really been created. - Fixed a problem with `--compress` (`-z`) where the receiving side could - return the error "inflate (token) returned -5". + return the error "`inflate (token) returned -5`". - Fixed a bug where `--delete-during` could delete in a directory before it noticed that the sending side sent an I/O error for that directory (both @@ -1346,7 +1346,7 @@ - An absolute-path filter rule (i.e. with a '/' modifier) no longer loses its modifier when sending the filter rules to the remote rsync. - - Improved the "--delete does not work without -r or -d" message. + - Improved the "`--delete does not work without -r or -d`" message. - Improved rsync's handling of `--timeout` to avoid a weird timeout case where the sender could timeout even though it has recently written data to the @@ -1657,8 +1657,8 @@ of a proto.h-tstamp rule that could make the binaries get rebuild without cause. - - Improved the testsuite to work around a problem with some utilities (e.g. cp - -p & touch -r) rounding sub-second timestamps. + - Improved the testsuite to work around a problem with some utilities (e.g. + `cp -p` & `touch -r`) rounding sub-second timestamps. - Ensure that the early patches don't cause any generated-file hunks to bleed-over into patches that follow. @@ -4367,7 +4367,7 @@ - HP PA-RISC HP-UX 11.11 cc - IRIX 6.5 MIPS cc - IRIX 6.5 MIPS gcc - - Mac OS X PPC (--disable-ipv6) cc + - Mac OS X PPC (`--disable-ipv6`) cc - NetBSD 1.5 i386 gcc - NetBSD Current i386 cc - OpenBSD 2.5 Sparc gcc From 5b1baa7a2e58f51b575ce263dbc8c09beefca2d0 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Mon, 27 Dec 2021 13:42:19 -0800 Subject: [PATCH 018/350] Rename md2man. --- md2man => md-convert | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename md2man => md-convert (100%) diff --git a/md2man b/md-convert similarity index 100% rename from md2man rename to md-convert From a2b630c0bb586c9761fd5fc53dc4c212b6dd25df Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Mon, 27 Dec 2021 14:19:11 -0800 Subject: [PATCH 019/350] Unify md parsing scripts & improve non-man html conversions. --- Makefile.in | 8 +- NEWS.md | 2 + maybe-make-man | 4 +- md-convert | 222 +++++++++++++++++++++++++++------------- md2man | 1 + packaging/md2html | 104 ------------------- packaging/release-rsync | 2 +- 7 files changed, 159 insertions(+), 184 deletions(-) create mode 120000 md2man delete mode 100755 packaging/md2html diff --git a/Makefile.in b/Makefile.in index 98d5a7af8..14d95abed 100644 --- a/Makefile.in +++ b/Makefile.in @@ -257,16 +257,16 @@ proto.h-tstamp: $(srcdir)/*.c $(srcdir)/lib/compat.c daemon-parm.h .PHONY: man man: rsync.1 rsync-ssl.1 rsyncd.conf.5 rrsync.1 -rsync.1: rsync.1.md md2man version.h Makefile +rsync.1: rsync.1.md md-convert version.h Makefile @$(srcdir)/maybe-make-man $(srcdir) rsync.1.md -rsync-ssl.1: rsync-ssl.1.md md2man version.h Makefile +rsync-ssl.1: rsync-ssl.1.md md-convert version.h Makefile @$(srcdir)/maybe-make-man $(srcdir) rsync-ssl.1.md -rsyncd.conf.5: rsyncd.conf.5.md md2man version.h Makefile +rsyncd.conf.5: rsyncd.conf.5.md md-convert version.h Makefile @$(srcdir)/maybe-make-man $(srcdir) rsyncd.conf.5.md -rrsync.1: support/rrsync.1.md md2man Makefile +rrsync.1: support/rrsync.1.md md-convert Makefile @$(srcdir)/maybe-make-man $(srcdir) support/rrsync.1.md .PHONY: clean diff --git a/NEWS.md b/NEWS.md index 5994fff64..b88208c50 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4472,3 +4472,5 @@ \* DATE OF COMMIT is the date the protocol change was committed to version control. + +@USE_GFM_PARSER@ diff --git a/maybe-make-man b/maybe-make-man index 59f2dce42..99b8fb894 100755 --- a/maybe-make-man +++ b/maybe-make-man @@ -16,7 +16,7 @@ fi if [ ! -f "$flagfile" ]; then # We test our smallest manpage just to see if the python setup works. - if "$srcdir/md2man" --test "$srcdir/rsync-ssl.1.md" >/dev/null 2>&1; then + if "$srcdir/md-convert" --test "$srcdir/rsync-ssl.1.md" >/dev/null 2>&1; then touch $flagfile else outname=`echo "$inname" | sed 's/\.md$//'` @@ -37,4 +37,4 @@ if [ ! -f "$flagfile" ]; then fi fi -"$srcdir/md2man" -s "$srcdir" "$srcdir/$inname" +"$srcdir/md-convert" "$srcdir/$inname" diff --git a/md-convert b/md-convert index fd546f190..7780d06b5 100755 --- a/md-convert +++ b/md-convert @@ -1,28 +1,35 @@ #!/usr/bin/env python3 -# This script takes a manpage written in markdown and turns it into an html web -# page and a nroff man page. The input file must have the name of the program -# and the section in this format: NAME.NUM.md. The output files are written -# into the current directory named NAME.NUM.html and NAME.NUM. The input -# format has one extra extension: if a numbered list starts at 0, it is turned -# into a description list. The dl's dt tag is taken from the contents of the -# first tag inside the li, which is usually a p, code, or strong tag. The -# cmarkgfm or commonmark lib is used to transforms the input file into html. -# The html.parser is used as a state machine that both tweaks the html and -# outputs the nroff data based on the html tags. +# This script transforms markdown files into html and (optionally) nroff. The +# output files are written into the current directory named for the input file +# without the .md suffix and either the .html suffix or no suffix. # -# We normally grab the prefix from the generated Makefile, which is then used -# in the various other grabbed values (see the Makefile for its ${prefix} -# paths). However, the maintainer can choose to override this prefix by -# exporting RSYNC_OVERRIDE_PREFIX=/usr. This allows the man pages to refer to -# /usr paths (and are thus compatible with the release-rsync script) while -# still having the built rsync get installed into /usr/local for local testing. +# If the input .md file has a section number at the end of the name (e.g., +# rsync.1.md) a nroff file is also output (PROJ.NUM.md -> PROJ.NUM). # -# Copyright (C) 2020 Wayne Davison +# The markdown input format has one extra extension: if a numbered list starts +# at 0, it is turned into a description list. The dl's dt tag is taken from the +# contents of the first tag inside the li, which is usually a p, code, or +# strong tag. +# +# The cmarkgfm or commonmark lib is used to transforms the input file into +# html. Then, the html.parser is used as a state machine that lets us tweak +# the html and (optionally) output nroff data based on the html tags. +# +# If the string @USE_GFM_PARSER@ exists in the file, the string is removed and +# a github-flavored-markup parser is used to parse the file. +# +# The man-page .md files also get the vars @VERSION@, @BINDIR@, and @LIBDIR@ +# substituted. Some of these values depend on the Makefile $(prefix) (see the +# generated Makefile). If the maintainer wants to build files for /usr/local +# while creating release-ready man-page files for /usr, use the environment to +# set RSYNC_OVERRIDE_PREFIX=/usr. + +# Copyright (C) 2020 - 2021 Wayne Davison # # This program is freely redistributable. -import sys, os, re, argparse, subprocess, time +import os, sys, re, argparse, subprocess, time from html.parser import HTMLParser CONSUMES_TXT = set('h1 h2 p li pre'.split()) @@ -58,8 +65,30 @@ dd p:first-of-type { """ -HTML_END = """\ +TABLE_STYLE = """\ +table { + border-color: grey; + border-spacing: 0; +} +tr { + border-top: 1px solid grey; +} +tr:nth-child(2n) { + background-color: #f6f8fa; +} +th, td { + border: 1px solid #dfe2e5; + text-align: center; + padding-left: 1em; + padding-right: 1em; +} +""" + +MAN_HTML_END = """\

%s

+""" + +HTML_END = """\ """ @@ -78,41 +107,96 @@ NBR_DASH = ('\4', r"\-") NBR_SPACE = ('\xa0', r"\ ") md_parser = None +env_subs = { } def main(): - fi = re.match(r'^(?P(?P.+/)?(?P(?P[^/]+)\.(?P\d+))\.md)$', args.mdfile) + for mdfn in args.mdfiles: + parse_md_file(mdfn) + + if args.test: + print("The test was successful.") + + +def parse_md_file(mdfn): + fi = re.match(r'^(?P(?P.+/)?(?P(?P[^/]+?)(\.(?P\d+))?)\.md)$', mdfn) if not fi: - die('Failed to parse NAME.NUM.md out of input file:', args.mdfile) + die('Failed to parse a md input file name:', mdfn) fi = argparse.Namespace(**fi.groupdict()) + fi.want_manpage = not not fi.sect + if fi.want_manpage: + fi.title = fi.prog + '(' + fi.sect + ') man page' + else: + fi.title = fi.prog + + if fi.want_manpage: + if not env_subs: + find_man_substitutions() + prog_ver = 'rsync ' + env_subs['VERSION'] + if fi.prog != 'rsync': + prog_ver = fi.prog + ' from ' + prog_ver + fi.man_headings = (fi.prog, fi.sect, env_subs['date'], prog_ver, env_subs['prefix']) + + with open(mdfn, 'r', encoding='utf-8') as fh: + txt = fh.read() + + use_gfm_parser = '@USE_GFM_PARSER@' in txt + if use_gfm_parser: + txt = txt.replace('@USE_GFM_PARSER@', '') + + if fi.want_manpage: + txt = (txt.replace('@VERSION@', env_subs['VERSION']) + .replace('@BINDIR@', env_subs['bindir']) + .replace('@LIBDIR@', env_subs['libdir'])) + + if use_gfm_parser: + if not gfm_parser: + die('Input file requires cmarkgfm parser:', mdfn) + fi.html_in = gfm_parser(txt) + else: + fi.html_in = md_parser(txt) + txt = None + + TransformHtml(fi) + + if args.test: + return + + output_list = [ (fi.name + '.html', fi.html_out) ] + if fi.want_manpage: + output_list += [ (fi.name, fi.man_out) ] + for fn, txt in output_list: + if os.path.lexists(fn): + os.unlink(fn) + print("Wrote:", fn) + with open(fn, 'w', encoding='utf-8') as fh: + fh.write(txt) - if args.srcdir: - fi.srcdir = args.srcdir + '/' - elif not fi.srcdir: - fi.srcdir = './' - fi.title = fi.prog + '(' + fi.sect + ') man page' - fi.mtime = 0 +def find_man_substitutions(): + srcdir = os.path.dirname(sys.argv[0]) + '/' + mtime = 0 - git_dir = fi.srcdir + '.git' + git_dir = srcdir + '.git' if os.path.lexists(git_dir): - fi.mtime = int(subprocess.check_output(['git', '--git-dir', git_dir, 'log', '-1', '--format=%at'])) + mtime = int(subprocess.check_output(['git', '--git-dir', git_dir, 'log', '-1', '--format=%at'])) - env_subs = { 'prefix': os.environ.get('RSYNC_OVERRIDE_PREFIX', None) } + # Allow "prefix" to be overridden via the environment: + env_subs['prefix'] = os.environ.get('RSYNC_OVERRIDE_PREFIX', None) if args.test: env_subs['VERSION'] = '1.0.0' env_subs['bindir'] = '/usr/bin' env_subs['libdir'] = '/usr/lib/rsync' else: - for fn in (fi.srcdir + 'version.h', 'Makefile'): + for fn in (srcdir + 'version.h', 'Makefile'): try: st = os.lstat(fn) except OSError: - die('Failed to find', fi.srcdir + fn) - if not fi.mtime: - fi.mtime = st.st_mtime + die('Failed to find', srcdir + fn) + if not mtime: + mtime = st.st_mtime - with open(fi.srcdir + 'version.h', 'r', encoding='utf-8') as fh: + with open(srcdir + 'version.h', 'r', encoding='utf-8') as fh: txt = fh.read() m = re.search(r'"(.+?)"', txt) env_subs['VERSION'] = m.group(1) @@ -131,40 +215,14 @@ def main(): if var == 'srcdir': break - fi.prog_ver = 'rsync ' + env_subs['VERSION'] - if fi.prog != 'rsync': - fi.prog_ver = fi.prog + ' from ' + fi.prog_ver - - with open(fi.fn, 'r', encoding='utf-8') as fh: - txt = fh.read() - - txt = re.sub(r'@VERSION@', env_subs['VERSION'], txt) - txt = re.sub(r'@BINDIR@', env_subs['bindir'], txt) - txt = re.sub(r'@LIBDIR@', env_subs['libdir'], txt) - - fi.html_in = md_parser(txt) - txt = None - - fi.date = time.strftime('%d %b %Y', time.localtime(fi.mtime)) - fi.man_headings = (fi.prog, fi.sect, fi.date, fi.prog_ver, env_subs['prefix']) - - HtmlToManPage(fi) - - if args.test: - print("The test was successful.") - return - - for fn, txt in ((fi.name + '.html', fi.html_out), (fi.name, fi.man_out)): - print("Wrote:", fn) - with open(fn, 'w', encoding='utf-8') as fh: - fh.write(txt) + env_subs['date'] = time.strftime('%d %b %Y', time.localtime(mtime)) def html_via_commonmark(txt): return commonmark.HtmlRenderer().render(commonmark.Parser().parse(txt)) -class HtmlToManPage(HTMLParser): +class TransformHtml(HTMLParser): def __init__(self, fi): HTMLParser.__init__(self, convert_charrefs=True) @@ -177,14 +235,23 @@ class HtmlToManPage(HTMLParser): in_pre = False, in_code = False, html_out = [ HTML_START % fi.title ], - man_out = [ MAN_START % fi.man_headings ], + man_out = [ ], txt = '', + want_manpage = fi.want_manpage, ) + if st.want_manpage: + st.man_out.append(MAN_START % fi.man_headings) + + if '' in fi.html_in: + st.html_out[0] = st.html_out[0].replace('', TABLE_STYLE + '') + self.feed(fi.html_in) fi.html_in = None - st.html_out.append(HTML_END % fi.date) + if st.want_manpage: + st.html_out.append(MAN_HTML_END % env_subs['date']) + st.html_out.append(HTML_END) st.man_out.append(MAN_END) fi.html_out = ''.join(st.html_out) @@ -232,8 +299,9 @@ class HtmlToManPage(HTMLParser): elif tag == 'strong' or tag == 'b': st.txt += BOLD_FONT[0] elif tag == 'em' or tag == 'i': - tag = 'u' # Change it into underline to be more like the man page - st.txt += UNDR_FONT[0] + if st.want_manpage: + tag = 'u' # Change it into underline to be more like the man page + st.txt += UNDR_FONT[0] elif tag == 'ol': start = 1 for var, val in attrs_list: @@ -256,6 +324,10 @@ class HtmlToManPage(HTMLParser): st.man_out.append(".RS\n") st.p_macro = ".IP\n" st.list_state.append('o') + elif tag == 'hr': + st.man_out.append(".l\n") + st.html_out.append("
") + return st.html_out.append('<' + tag + ''.join(' ' + var + '="' + htmlify(val) + '"' for var, val in attrs_list) + '>') st.at_first_tag_in_dd = False @@ -300,8 +372,9 @@ class HtmlToManPage(HTMLParser): elif tag == 'strong' or tag == 'b': add_to_txt = NORM_FONT[0] elif tag == 'em' or tag == 'i': - tag = 'u' # Change it into underline to be more like the man page - add_to_txt = NORM_FONT[0] + if st.want_manpage: + tag = 'u' # Change it into underline to be more like the man page + add_to_txt = NORM_FONT[0] elif tag == 'ol' or tag == 'ul': if st.list_state.pop() == 'dl': tag = 'dl' @@ -310,6 +383,8 @@ class HtmlToManPage(HTMLParser): else: st.p_macro = ".P\n" st.at_first_tag_in_dd = False + elif tag == 'hr': + return st.html_out.append('') if add_to_txt: if txt is None: @@ -379,22 +454,23 @@ def die(*msg): if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Transform a NAME.NUM.md markdown file into a NAME.NUM.html web page & a NAME.NUM man page.', add_help=False) - parser.add_argument('--srcdir', '-s', help='Specify the source dir if the input file is not in it.') - parser.add_argument('--test', action='store_true', help='Test if we can parse the input w/o updating any files.') + parser = argparse.ArgumentParser(description="Output html and (optionally) nroff for markdown pages.", add_help=False) + parser.add_argument('--test', action='store_true', help="Just test the parsing without outputting any files.") parser.add_argument('--debug', '-D', action='count', default=0, help='Output copious info on the html parsing. Repeat for even more.') parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.") - parser.add_argument('mdfile', help="The NAME.NUM.md file to parse.") + parser.add_argument("mdfiles", nargs='+', help="The source .md files to convert.") args = parser.parse_args() try: import cmarkgfm md_parser = cmarkgfm.markdown_to_html + gfm_parser = cmarkgfm.github_flavored_markdown_to_html except: try: import commonmark md_parser = html_via_commonmark except: die("Failed to find cmarkgfm or commonmark for python3.") + gfm_parser = None main() diff --git a/md2man b/md2man new file mode 120000 index 000000000..5d1a8fcd9 --- /dev/null +++ b/md2man @@ -0,0 +1 @@ +md-convert \ No newline at end of file diff --git a/packaging/md2html b/packaging/md2html deleted file mode 100755 index 21e42c66a..000000000 --- a/packaging/md2html +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (C) 2020 Wayne Davison -# -# This program is freely redistributable. - -import os, re, argparse - -HTML_START = """\ - -%s - - - -""" - -HTML_END = """\ - -""" - -md_parser = None - -def main(): - for mdfn in args.mdfiles: - if not mdfn.endswith('.md'): - print('Ignoring non-md input file:', mdfn) - continue - title = re.sub(r'.*/', '', mdfn).replace('.md', '') - htfn = mdfn.replace('.md', '.html') - - print("Parsing", mdfn, '->', htfn) - - with open(mdfn, 'r', encoding='utf-8') as fh: - txt = fh.read() - - txt = re.sub(r'\s--\s', '\xa0-- ', txt) - - html = md_parser(txt) - - html = re.sub(r'(?)()([\s\S]*?)()', lambda m: m[1] + re.sub(r'\s', '\xa0', m[2]) + m[3], html) - html = html.replace('--', '‑‑').replace("\xa0-", ' ‑').replace("\xa0", ' ') - html = re.sub(r'(\W)-', r'\1‑', html) - - if os.path.lexists(htfn): - os.unlink(htfn) - - with open(htfn, 'w', encoding='utf-8') as fh: - fh.write(HTML_START % title) - fh.write(html) - fh.write(HTML_END) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Output html for md pages.', add_help=False) - parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.") - parser.add_argument("mdfiles", nargs='+', help="The .md files to turn into .html files.") - args = parser.parse_args() - - try: - import cmarkgfm - # Our NEWS.md file has a gfm table in it. - md_parser = cmarkgfm.github_flavored_markdown_to_html - except: - die("Failed to find cmarkgfm for python3.") - - main() diff --git a/packaging/release-rsync b/packaging/release-rsync index fa1da2349..0ffc10950 100755 --- a/packaging/release-rsync +++ b/packaging/release-rsync @@ -341,7 +341,7 @@ About to: md_files = 'README.md NEWS.md INSTALL.md'.split() html_files = [ fn for fn in gen_pathnames if fn.endswith('.html') ] cmd_chk(['rsync', '-a', *md_files, *html_files, dest]) - cmd_chk(["packaging/md2html"] + [ dest +'/'+ fn for fn in md_files ]) + cmd_chk(["./md-convert"] + [ dest +'/'+ fn for fn in md_files ]) cmd_chk(f"git log --name-status | gzip -9 >{dest}/ChangeLog.gz") From 3008e7c2269936672d796d95b85b285fc07adc01 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Mon, 27 Dec 2021 15:44:51 -0800 Subject: [PATCH 020/350] Include "rrsync" in "all" target when `--with-rrsync` was used. --- .github/workflows/build.yml | 6 +++--- Makefile.in | 8 ++++---- configure.ac | 6 +++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4f5cbf23d..a44cd2ab9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,7 +24,7 @@ jobs: - name: configure run: ./configure --with-rrsync - name: make - run: make all rrsync + run: make - name: install run: sudo make install - name: info @@ -63,7 +63,7 @@ jobs: - name: configure run: CPPFLAGS=-I/usr/local/opt/openssl/include/ LDFLAGS=-L/usr/local/opt/openssl/lib/ ./configure --with-rrsync - name: make - run: make all rrsync + run: make - name: install run: sudo make install - name: info @@ -103,7 +103,7 @@ jobs: - name: configure run: bash -c './configure --with-rrsync' - name: make - run: bash -c 'make all rrsync' + run: bash -c 'make' - name: install run: bash -c 'make install' - name: info diff --git a/Makefile.in b/Makefile.in index 14d95abed..334f1bd76 100644 --- a/Makefile.in +++ b/Makefile.in @@ -45,11 +45,11 @@ OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \ util1.o util2.o main.o checksum.o match.o syscall.o log.o backup.o delete.o OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \ usage.o fileio.o batch.o clientname.o chmod.o acls.o xattrs.o -OBJS3=progress.o pipe.o @ASM@ +OBJS3=progress.o pipe.o @ASM@ @SIMD@ DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o popt_OBJS=popt/findme.o popt/popt.o popt/poptconfig.o \ popt/popthelp.o popt/poptparse.o -OBJS=$(OBJS1) $(OBJS2) $(OBJS3) @SIMD@ $(DAEMON_OBJ) $(LIBOBJ) @BUILD_ZLIB@ @BUILD_POPT@ +OBJS=$(OBJS1) $(OBJS2) $(OBJS3) $(DAEMON_OBJ) $(LIBOBJ) @BUILD_ZLIB@ @BUILD_POPT@ TLS_OBJ = tls.o syscall.o util2.o t_stub.o lib/compat.o lib/snprintf.o lib/permstring.o lib/sysxattrs.o @BUILD_POPT@ @@ -68,7 +68,7 @@ CHECK_OBJS=tls.o testrun.o getgroups.o getfsdev.o t_stub.o t_unsafe.o trimslash. $(CC) -I. -I$(srcdir) $(CFLAGS) $(CPPFLAGS) -c $< @CC_SHOBJ_FLAG@ @OBJ_RESTORE@ -all: Makefile rsync$(EXEEXT) stunnel-rsyncd.conf @MAKE_MAN@ +all: Makefile rsync$(EXEEXT) stunnel-rsyncd.conf @MAKE_RRSYNC@ @MAKE_MAN@ .PHONY: all .PHONY: install @@ -255,7 +255,7 @@ proto.h-tstamp: $(srcdir)/*.c $(srcdir)/lib/compat.c daemon-parm.h $(AWK) -f $(srcdir)/mkproto.awk $(srcdir)/*.c $(srcdir)/lib/compat.c daemon-parm.h .PHONY: man -man: rsync.1 rsync-ssl.1 rsyncd.conf.5 rrsync.1 +man: rsync.1 rsync-ssl.1 rsyncd.conf.5 @MAKE_RRSYNC_1@ rsync.1: rsync.1.md md-convert version.h Makefile @$(srcdir)/maybe-make-man $(srcdir) rsync.1.md diff --git a/configure.ac b/configure.ac index b30eeb933..3caf5e395 100644 --- a/configure.ac +++ b/configure.ac @@ -117,7 +117,6 @@ if test x"$enable_md2man" != x"no"; then MAKE_MAN=man else AC_MSG_RESULT(no) - MAKE_MAN='' fi # Specifically, this turns on panic_action handling. @@ -140,6 +139,9 @@ AC_ARG_WITH(rrsync, AS_HELP_STRING([--with-rrsync],[also install the rrsync script and its man page])) if test x"$with_rrsync" != x"yes"; then with_rrsync=no +else + MAKE_RRSYNC='rrsync' + MAKE_RRSYNC_1='rrsync.1' fi AC_SUBST(with_rrsync) @@ -1253,6 +1255,8 @@ AC_SUBST(OBJ_RESTORE) AC_SUBST(CC_SHOBJ_FLAG) AC_SUBST(BUILD_POPT) AC_SUBST(BUILD_ZLIB) +AC_SUBST(MAKE_RRSYNC) +AC_SUBST(MAKE_RRSYNC_1) AC_SUBST(MAKE_MAN) AC_CHECK_FUNCS(_acl __acl _facl __facl) From 8cf9dbb742ce4c89c649cd4f4a14ed9bd254a075 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Mon, 27 Dec 2021 17:40:31 -0800 Subject: [PATCH 021/350] Change args to maybe-make-man. --- Makefile.in | 10 +++++----- maybe-make-man | 13 ++++--------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/Makefile.in b/Makefile.in index 334f1bd76..8a16f651f 100644 --- a/Makefile.in +++ b/Makefile.in @@ -82,7 +82,7 @@ install: all if test -f rsync-ssl.1; then $(INSTALLMAN) -m 644 rsync-ssl.1 $(DESTDIR)$(mandir)/man1; fi if test -f rsyncd.conf.5; then $(INSTALLMAN) -m 644 rsyncd.conf.5 $(DESTDIR)$(mandir)/man5; fi if test "$(with_rrsync)" = yes; then \ - $(INSTALLCMD) -m 755 $(srcdir)/support/rrsync $(DESTDIR)$(bindir); \ + $(INSTALLCMD) -m 755 rrsync $(DESTDIR)$(bindir); \ if test -f rrsync.1; then $(INSTALLMAN) -m 644 rrsync.1 $(DESTDIR)$(mandir)/man1; fi; \ fi @@ -258,16 +258,16 @@ proto.h-tstamp: $(srcdir)/*.c $(srcdir)/lib/compat.c daemon-parm.h man: rsync.1 rsync-ssl.1 rsyncd.conf.5 @MAKE_RRSYNC_1@ rsync.1: rsync.1.md md-convert version.h Makefile - @$(srcdir)/maybe-make-man $(srcdir) rsync.1.md + @$(srcdir)/maybe-make-man rsync.1.md rsync-ssl.1: rsync-ssl.1.md md-convert version.h Makefile - @$(srcdir)/maybe-make-man $(srcdir) rsync-ssl.1.md + @$(srcdir)/maybe-make-man rsync-ssl.1.md rsyncd.conf.5: rsyncd.conf.5.md md-convert version.h Makefile - @$(srcdir)/maybe-make-man $(srcdir) rsyncd.conf.5.md + @$(srcdir)/maybe-make-man rsyncd.conf.5.md rrsync.1: support/rrsync.1.md md-convert Makefile - @$(srcdir)/maybe-make-man $(srcdir) support/rrsync.1.md + @$(srcdir)/maybe-make-man support/rrsync.1.md .PHONY: clean clean: cleantests diff --git a/maybe-make-man b/maybe-make-man index 99b8fb894..0dc173059 100755 --- a/maybe-make-man +++ b/maybe-make-man @@ -1,19 +1,14 @@ #!/bin/sh -if [ x"$2" = x ]; then - echo "Usage: $0 SRC_DIR NAME.NUM.md" 1>&2 +if [ $# != 1 ]; then + echo "Usage: $0 NAME.NUM.md" 1>&2 exit 1 fi -srcdir="$1" -inname="$2" +inname="$1" +srcdir=`dirname "$0"` flagfile="$srcdir/.md2man-works" -if [ ! -d "$srcdir" ]; then - echo "The specified SRC_DIR is not a directory: $srcdir" 1>&2 - exit 1 -fi - if [ ! -f "$flagfile" ]; then # We test our smallest manpage just to see if the python setup works. if "$srcdir/md-convert" --test "$srcdir/rsync-ssl.1.md" >/dev/null 2>&1; then From e07f8fb86329d9370edb47d5c057def16108892e Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Mon, 27 Dec 2021 17:44:32 -0800 Subject: [PATCH 022/350] Add a default single-access lock. --- NEWS.md | 23 ++++++++++++++--------- packaging/cull_options | 5 ++++- support/lsh | 19 +++++++++---------- support/rrsync | 35 ++++++++++++++++++++++++++++++----- 4 files changed, 57 insertions(+), 25 deletions(-) diff --git a/NEWS.md b/NEWS.md index b88208c50..735dec1d6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -97,15 +97,20 @@ - More ASM optimizations from Shark64. - - Transformed rrsync into a python script with improvements: security has been - beefed up; the known rsync options were updated to include recent additions; - rrsync rejects `-L` (`--copy-links`) by default to make it harder to exploit - any out-of-subdir symlinks; a new rrsync option of `-munge` tells rrsync to - always enable the `--munge-links` rsync option on the server side; a new - rrsync option of `-no-del` disables all `--remove*` and `--delete*` rsync - options on the server side; the log format has been tweaked slightly to add - seconds to the timestamp and output the command executed as a tuple; an - rrsync.1 manpage is now created. + - Transformed rrsync into a python script with improvements: + - Security has been beefed up. + - The known rsync options were updated to include recent additions. + - Make rrsync reject `-L`, `-K`, & `-k` by default to make it harder to + exploit any out-of-subdir symlinks. + - A new rrsync option of `-munge` tells rrsync to always enable rsync's + `--munge-links` option on the server side. + - A new rrsync option of `-no-lock` disables a new single-use locking idiom + that is the default when `-ro` is not used (useful with `-munge`). + - A new rrsync option of `-no-del` disables all `--remove*` and `--delete*` + options on the server side. + - The log format has been tweaked slightly to add seconds to the timestamp + and to output the command executed as a tuple (making the args clearer). + - An rrsync.1 manpage was added. - Work around a glibc bug where lchmod() breaks in a chroot w/o /proc mounted. diff --git a/packaging/cull_options b/packaging/cull_options index d4e1c626d..ca0611211 100755 --- a/packaging/cull_options +++ b/packaging/cull_options @@ -86,7 +86,10 @@ def main(): # To disable a short-named option, add its letter to this string: """ - txt += str_assign('short_disabled', 'Ls') + "\n" + txt += str_assign('short_disabled', 's') + "\n" + txt += '# These are also disabled when the restricted dir is not "/":\n' + txt += str_assign('short_disabled_subdir', 'KLk') + "\n" + txt += '# These are all possible short options that we will accept (when not disabled above):\n' txt += str_assign('short_no_arg', ''.join(sorted(short_no_arg)), 'DO NOT REMOVE ANY') txt += str_assign('short_with_num', ''.join(sorted(short_with_num)), 'DO NOT REMOVE ANY') diff --git a/support/lsh b/support/lsh index 40fe3d738..7b3c0656d 100755 --- a/support/lsh +++ b/support/lsh @@ -16,10 +16,7 @@ GetOptions( 'no-cd' => \( my $no_chdir ), 'sudo' => \( my $use_sudo ), 'rrsync=s' => \( my $rrsync_dir ), - 'ro' => \( my $rrsync_ro = '' ), - 'wo' => \( my $rrsync_wo = '' ), - 'munge' => \( my $rrsync_munge = '' ), - 'no-del' => \( my $rrsync_no_del = '' ), + 'rropts=s' => \( my $rrsync_opts ), ) or &usage; &usage unless @ARGV > 1; @@ -75,10 +72,12 @@ unless ($no_chdir) { if ($rrsync_dir) { $ENV{SSH_ORIGINAL_COMMAND} = join(' ', @ARGV); push @cmd, 'rrsync'; - push @cmd, '-ro' if $rrsync_ro; - push @cmd, '-wo' if $rrsync_wo; - push @cmd, '-munge' if $rrsync_munge; - push @cmd, '-no-del' if $rrsync_no_del; + if ($rrsync_opts) { + foreach my $opt (split(/[ ,]+/, $rrsync_opts)) { + $opt = "-$opt" unless $opt =~ /^-/; + push @cmd, $opt; + } + } push @cmd, $rrsync_dir; } else { push @cmd, '/bin/sh', '-c', "@ARGV"; @@ -101,8 +100,8 @@ Options: --no-cd Skip the chdir \$HOME (the default with hostname "lh") --sudo Use sudo -H -l USER to become root or the specified USER. --rrsync=DIR Test rrsync restricted copying without using ssh. ---ro Passes -ro to rrsync (when --rrsync is specified). ---wo Passes -wo to rrsync (when --rrsync is specified. +--rropts=STR The string "munge,no-del,no-lock" would pass 3 options to + rrsync (must be combined with --rrsync=DIR). The script also ignores a bunch of single-letter ssh options. EOT diff --git a/support/rrsync b/support/rrsync index 5b43a8198..469288b9f 100755 --- a/support/rrsync +++ b/support/rrsync @@ -24,8 +24,12 @@ LOGFILE = 'rrsync.log' # NOTE: the file must exist for a line to be appended! ### START of options data produced by the cull_options script. ### # To disable a short-named option, add its letter to this string: -short_disabled = 'Ls' +short_disabled = 's' +# These are also disabled when the restricted dir is not "/": +short_disabled_subdir = 'KLk' + +# These are all possible short options that we will accept (when not disabled above): short_no_arg = 'ACDEHIJKLNORSUWXbcdgklmnopqrstuvxyz' # DO NOT REMOVE ANY short_with_num = '@B' # DO NOT REMOVE ANY @@ -125,7 +129,7 @@ long_opts = { ### END of options data produced by the cull_options script. ### -import os, sys, re, argparse, glob, socket, time +import os, sys, re, argparse, glob, socket, time, subprocess from argparse import RawTextHelpFormatter try: @@ -174,6 +178,10 @@ def main(): if args.ro: long_opts['log-file'] = -1 + if args.dir != '/': + global short_disabled + short_disabled += short_disabled_subdir + short_no_arg_re = short_no_arg short_with_num_re = short_with_num if short_disabled: @@ -268,8 +276,12 @@ def main(): log_fh.close() # NOTE: This assumes that the rsync protocol will not be maliciously hijacked. - os.execlp(RSYNC, *cmd) - die("execlp(", RSYNC, *cmd, ') failed') + if args.no_lock: + os.execlp(RSYNC, *cmd) + die("execlp(", RSYNC, *cmd, ') failed') + child = subprocess.run(cmd) + if child.returncode != 0: + sys.exit(child.returncode) def validated_arg(opt, arg, typ=3, wild=False): @@ -319,6 +331,16 @@ def validated_arg(opt, arg, typ=3, wild=False): return ret if wild else ret[0] +def lock_or_die(dirname): + import fcntl + global lock_handle + lock_handle = os.open(dirname, os.O_RDONLY) + try: + fcntl.flock(lock_handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + except: + die('Another instance of rrsync is already accessing this directory.') + + def die(*msg): print(sys.argv[0], 'error:', *msg, file=sys.stderr) if sys.stdin.isatty(): @@ -336,9 +358,10 @@ if __name__ == '__main__': our_desc = """Use "man rrsync" to learn how to restrict ssh users to using a restricted rsync command.""" arg_parser = OurArgParser(description=our_desc, add_help=False) only_group = arg_parser.add_mutually_exclusive_group() - only_group.add_argument('-ro', action='store_true', help="Allow only reading from the DIR. Implies -no-del.") + only_group.add_argument('-ro', action='store_true', help="Allow only reading from the DIR. Implies -no-del and -no-lock.") only_group.add_argument('-wo', action='store_true', help="Allow only writing to the DIR.") arg_parser.add_argument('-no-del', action='store_true', help="Disable rsync's --delete* and --remove* options.") + arg_parser.add_argument('-no-lock', action='store_true', help="Avoid the single-run (per-user) lock check.") arg_parser.add_argument('-munge', action='store_true', help="Enable rsync's --munge-links on the server side.") arg_parser.add_argument('-help', '-h', action='help', help="Output this help message and exit.") arg_parser.add_argument('dir', metavar='DIR', help="The restricted directory to use.") @@ -348,6 +371,8 @@ if __name__ == '__main__': args.dir_slash_len = len(args.dir) if args.ro: args.no_del = True + elif not args.no_lock: + lock_or_die(args.dir) main() # vim: sw=4 et From ffec7fe109efda650988ff993c989c2bc9f24dff Mon Sep 17 00:00:00 2001 From: Marco Nenciarini Date: Thu, 30 Dec 2021 17:59:17 +0100 Subject: [PATCH 023/350] Fix rrsync directory normalization (#268) Fix an off-by-one in the `args.dir_slash_len` variable that leads to base every absolute path on `/` --- support/rrsync | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/support/rrsync b/support/rrsync index 469288b9f..fe1bc2507 100755 --- a/support/rrsync +++ b/support/rrsync @@ -368,7 +368,7 @@ if __name__ == '__main__': args = arg_parser.parse_args() args.dir = os.path.realpath(args.dir) args.dir_slash = args.dir + '/' - args.dir_slash_len = len(args.dir) + args.dir_slash_len = len(args.dir_slash) if args.ro: args.no_del = True elif not args.no_lock: From 8e77ece0eec633126f546f75c67f62837427e9c1 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Thu, 30 Dec 2021 12:28:16 -0800 Subject: [PATCH 024/350] Tweak the rrsync man page. --- support/rrsync.1.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/support/rrsync.1.md b/support/rrsync.1.md index b945ecf07..4aedfae94 100644 --- a/support/rrsync.1.md +++ b/support/rrsync.1.md @@ -14,9 +14,9 @@ A user's ssh login can be restricted to only allow the running of an rsync transfer in one of two easy ways: forcing the running of the rrsync script or forcing the running of an rsync daemon-over-ssh command. -To use the rrsync script, add a prefix like one of the following (followed by a -space) in front of each ssh-key line in the user's `~/.ssh/authorized_keys` -file that should be restricted: +To use the rrsync script, edit the user's `~/.ssh/authorized_keys` file and add +a prefix like one of the following (followed by a space) in front of each +ssh-key line that should be restricted: > ``` > command="rrsync DIR" @@ -29,9 +29,9 @@ may want to copy the script to a local bin dir with a unique name if you want to have multiple configurations. One or more rrsync options can be specified prior to the `DIR` if you want to further restrict the transfer. -To use an rsync daemon setup, add one of the following prefixes (followed by a -space) in front of each ssh-key line in the user's `~/.ssh/authorized_keys` -file that should be restricted: +To use an rsync daemon setup, edit the user's `~/.ssh/authorized_keys` file and +add a prefix like one of the following (followed by a space) in front of each +ssh-key line that should be restricted: > ``` > command="rsync --server --daemon ." @@ -43,6 +43,10 @@ with the appropriate path and option restrictions. If the `--config` option is omitted, it defaults to `~/rsyncd.conf`. See the `rsyncd.conf` man page for details of how to configure an rsync daemon. +When using rrsync, there can be just one restricted dir per authorized key. A +daemon setup, on the other hand, allows multiple module names inside the config +file, each one with its own path setting. + The remainder of this man page is dedicated to using the rrsync script. # OPTION SUMMARY From 13cfe6406fbf73d20fff9213656d94e21ac0289a Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Thu, 30 Dec 2021 12:29:14 -0800 Subject: [PATCH 025/350] Add error-code ignoring options to atomic-rsync. --- NEWS.md | 13 ++++++++++- support/atomic-rsync | 53 +++++++++++++++++++++++++++----------------- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/NEWS.md b/NEWS.md index 735dec1d6..aed80b59a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -97,7 +97,7 @@ - More ASM optimizations from Shark64. - - Transformed rrsync into a python script with improvements: + - Transformed support/rrsync into a python script with improvements: - Security has been beefed up. - The known rsync options were updated to include recent additions. - Make rrsync reject `-L`, `-K`, & `-k` by default to make it harder to @@ -112,6 +112,17 @@ and to output the command executed as a tuple (making the args clearer). - An rrsync.1 manpage was added. + - Added options to support/lsh to allow the rrsync script to be easily tested. + + - Transformed support/atomic-rsync into a python script and added the ability + to ignore one or more non-zero exit codes. By default, it now ignores code + 24 (file vanished). + + - Improved support/rsync-no-vanished wrapper script to not join stdout & + stderr together. + + - Transformed support/munge-symlinks into a python script. + - Work around a glibc bug where lchmod() breaks in a chroot w/o /proc mounted. - Some manpage improvements. diff --git a/support/atomic-rsync b/support/atomic-rsync index 37363e435..1964090da 100755 --- a/support/atomic-rsync +++ b/support/atomic-rsync @@ -29,6 +29,13 @@ def main(): if bad_args: die("You cannot use the", ' or '.join(bad_args), "option with atomic-rsync.\nUse --help for help.") + # We ignore exit-code 24 (file vanished) by default. + allowed_exit_codes = '0 ' + os.environ.get('ATOMIC_RSYNC_OK_CODES', '24') + try: + allowed_exit_codes = set(int(num) for num in re.split(r'[, ]+', allowed_exit_codes) if num != '') + except ValueError: + die('Invalid integer in ATOMIC_RSYNC_OK_CODES:', allowed_exit_codes[2:]) + symlink_content = os.readlink(dest_dir) if os.path.islink(dest_dir) else None dest_arg = dest_dir @@ -55,14 +62,18 @@ def main(): if os.path.isdir(new_dir): shutil.rmtree(new_dir) - subprocess.run([RSYNC_PROG, '--link-dest=' + dest_dir, *cmd_args], check=True) + child = subprocess.run([RSYNC_PROG, '--link-dest=' + dest_dir, *cmd_args]) + if child.returncode not in allowed_exit_codes: + die('The rsync copy failed with code', child.returncode, exitcode=child.returncode) + + if not os.path.isdir(new_dir): + die('The rsync copy failed to create:', new_dir) if old_dir is None: atomic_symlink(symlink_content, dest_arg) - return - - os.rename(dest_dir, old_dir) - os.rename(new_dir, dest_dir) + else: + os.rename(dest_dir, old_dir) + os.rename(new_dir, dest_dir) def atomic_symlink(target, link): @@ -89,22 +100,24 @@ to a local directory, and that directory must already exist. For example: ln -s files-1 /local/files atomic-rsync -aiv host:/remote/files/ /local/files/ -If /local/files is a symlink to a directory that ends in -1 or -2, the -copy will go to the alternate suffix and the symlink will be changed to -point to the new dir. This is a fully atomic update. If the destination -is not a symlink (or not a symlink to a *-1 or a *-2 directory), this -will instead create a directory with "~new~" suffixed, move the current -directory to a name with "~old~" suffixed, and then move the ~new~ -directory to the original destination name (this double rename is not -fully atomic, but is rapid). In both cases, the prior destintaion -directory will be preserved until the next update, at which point it -will be deleted. +If /local/files is a symlink to a directory that ends in -1 or -2, the copy +will go to the alternate suffix and the symlink will be changed to point to +the new dir. This is a fully atomic update. If the destination is not a +symlink (or not a symlink to a *-1 or a *-2 directory), this will instead +create a directory with "~new~" suffixed, move the current directory to a +name with "~old~" suffixed, and then move the ~new~ directory to the original +destination name (this double rename is not fully atomic, but is rapid). In +both cases, the prior destintaion directory will be preserved until the next +update, at which point it will be deleted. -In all likelihood, you do NOT want to specify this command: +By default, rsync exit-code 24 (file vanished) is allowed without halting the +atomic update. If you want to change that, specify the environment variable +ATOMIC_RSYNC_OK_CODES with numeric values separated by spaces and/or commas. +Specify an empty string to only allow a successful copy. An override example: - atomic-rsync -aiv host:/remote/files /local/ + ATOMIC_RSYNC_OK_CODES='23 24' atomic-rsync -aiv host:src/ dest/ -... UNLESS you want the entire /local dir to be swapped out! +See the errcode.h file for a list of all the exit codes. See the "rsync" command for its list of options. You may not use the --link-dest, --compare-dest, or --copy-dest options (since this script @@ -114,9 +127,9 @@ uses --link-dest to make the transfer efficient). sys.exit(1 if use_stderr else 0) -def die(*args): +def die(*args, exitcode=1): print(*args, file=sys.stderr) - sys.exit(1) + sys.exit(exitcode) if __name__ == '__main__': From c11467af36845e153b415d41bff7325ff40095ff Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Fri, 31 Dec 2021 10:58:19 -0800 Subject: [PATCH 026/350] Some compression improvements. The compression level of the first file in the transfer no longer sets the level for all files that follow it. Document that per-file level switching has no current effect (except for a global "dont compress = *" rule in the daemon). --- NEWS.md | 7 ++++++- rsync.1.md | 20 +++++++++----------- rsyncd.conf.5.md | 25 +++++++++++++++---------- token.c | 9 +++++++-- 4 files changed, 37 insertions(+), 24 deletions(-) diff --git a/NEWS.md b/NEWS.md index aed80b59a..6635ccab2 100644 --- a/NEWS.md +++ b/NEWS.md @@ -17,7 +17,7 @@ export LC_ALL=C.UTF-8 ``` - or maybe: + or if iconv translations are needed: ```shell if [ "${LC_ALL:-}" ]; then @@ -60,6 +60,11 @@ - Avoid a weird failure if you run a local copy with a (useless) `--rsh` option that contains a `V`. + - Fixed a long-standing compression bug where the compression level of the + first file transferred affected the level for all future files. Also, the + per-file compression skipping has apparently not worked in a very long time + (I checked back to 2.6.4), so it is now documented as being ineffective. + ### ENHANCEMENTS: - Use openssl's `-verify_hostname` option in the rsync-ssl script. diff --git a/rsync.1.md b/rsync.1.md index da2271712..c46fe2bad 100644 --- a/rsync.1.md +++ b/rsync.1.md @@ -2395,9 +2395,6 @@ your home directory (remove the '=' for that). ignore this weirdness unless the rsync server complains and tells you to specify `-zz`. - See also the `--skip-compress` option for the default list of file suffixes - that will be transferred with no (or minimal) compression. - 0. `--compress-choice=STR`, `--zc=STR` This option can be used to override the automatic negotiation of the @@ -2442,8 +2439,8 @@ your home directory (remove the '=' for that). > rsync -aiv --zc=zstd --zl=22 host:src/ dest/ For zlib & zlibx compression the valid values are from 1 to 9 with 6 being - the default. Specifying 0 turns compression off, and specifying -1 chooses - the default of 6. + the default. Specifying `--zl=0` turns compression off, and specifying + `--zl=-1` chooses the default level of 6. For zstd compression the valid values are from -131072 to 22 with 3 being the default. Specifying 0 chooses the default of 3. @@ -2462,14 +2459,15 @@ your home directory (remove the '=' for that). 0. `--skip-compress=LIST` + **NOTE:** no compression method currently supports per-file compression + changes, so this option has no effect. + Override the list of file suffixes that will be compressed as little as possible. Rsync sets the compression level on a per-file basis based on - the file's suffix. If the compression algorithm has an "off" level (such - as zlib/zlibx) then no compression occurs for those files. Other - algorithms that support changing the streaming level on-the-fly will have - the level minimized to reduces the CPU usage as much as possible for a - matching file. At this time, only zlib & zlibx compression support this - changing of levels on a per-file basis. + the file's suffix. If the compression algorithm has an "off" level, then + no compression occurs for those files. Other algorithms that support + changing the streaming level on-the-fly will have the level minimized to + reduces the CPU usage as much as possible for a matching file. The **LIST** should be one or more file suffixes (without the dot) separated by slashes (`/`). You may specify an empty string to indicate that no files diff --git a/rsyncd.conf.5.md b/rsyncd.conf.5.md index 730ef71e1..258322bdd 100644 --- a/rsyncd.conf.5.md +++ b/rsyncd.conf.5.md @@ -922,13 +922,14 @@ the values of parameters. See the GLOBAL PARAMETERS section for more details. > refuse options = * !a !delete* delete-after - A note on refusing "compress" -- it is better to set the "dont compress" - daemon parameter to "`*`" because that disables compression silently + A note on refusing "compress": it may be better to set the "dont compress" + daemon parameter to "`*`" and ensure that `RSYNC_COMPRESS_LIST=zlib` is set + in the environment of the daemon in order to disable compression silently instead of returning an error that forces the client to remove the `-z` option. - If you are un-refusing the compress option, you probably want to match - "`!compress*`" so that you also accept the `--compress-level` option. + If you are un-refusing the compress option, you may want to match + "`!compress*`" if you also want to allow the `--compress-level` option. Note that the "write-devices" option is refused by default, but can be explicitly accepted with "`!write-devices`". The options "log-file" and @@ -954,6 +955,10 @@ the values of parameters. See the GLOBAL PARAMETERS section for more details. 0. `dont compress` + **NOTE:** This parameter currently has no effect except in one instance: if + it is set to "`*`" then it minimizes or disables compression for all files + (for those that don't want to refuse the `--compress` option completely). + This parameter allows you to select filenames based on wildcard patterns that should not be compressed when pulling files from the daemon (no analogous parameter exists to govern the pushing of files to a daemon). @@ -964,14 +969,14 @@ the values of parameters. See the GLOBAL PARAMETERS section for more details. The "dont compress" parameter takes a space-separated list of case-insensitive wildcard patterns. Any source filename matching one of the patterns will be compressed as little as possible during the transfer. If - the compression algorithm has an "off" level (such as zlib/zlibx) then no - compression occurs for those files. Other algorithms have the level - minimized to reduces the CPU usage as much as possible. + the compression algorithm has an "off" level, then no compression occurs + for those files. If an algorithms has the ability to change the level in + mid-stream, it will be minimized to reduce the CPU usage as much as + possible. See the `--skip-compress` parameter in the **rsync**(1) manpage for the - list of file suffixes that are not compressed by default. Specifying a - value for the "dont compress" parameter changes the default when the daemon - is the sender. + list of file suffixes that are skipped by default if this parameter is not + set. 0. `early exec`, `pre-xfer exec`, `post-xfer exec` diff --git a/token.c b/token.c index 3a6d069ed..20d10bc5b 100644 --- a/token.c +++ b/token.c @@ -39,7 +39,6 @@ extern char *skip_compress; #define Z_INSERT_ONLY Z_SYNC_FLUSH #endif -static int compression_level; /* The compression level for the current file. */ static int skip_compression_level; /* The least possible compressing for handling skip-compress files. */ static int per_file_default_level; /* The default level that each new file gets prior to checking its suffix. */ @@ -224,9 +223,11 @@ static void init_set_compression(void) /* determine the compression level based on a wildcard filename list */ void set_compression(const char *fname) { +#if 0 /* No compression algorithms currently allow mid-stream changing of the level. */ const struct suffix_tree *node; const char *s; char ltr; +#endif if (!do_compression) return; @@ -234,6 +235,7 @@ void set_compression(const char *fname) if (!match_list) init_set_compression(); +#if 0 compression_level = per_file_default_level; if (!*match_list && !suftree) @@ -270,6 +272,9 @@ void set_compression(const char *fname) if (!(node = node->child)) return; } +#else + (void)fname; +#endif } /* non-compressing recv token */ @@ -361,7 +366,7 @@ send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset, in tx_strm.next_in = NULL; tx_strm.zalloc = NULL; tx_strm.zfree = NULL; - if (deflateInit2(&tx_strm, compression_level, + if (deflateInit2(&tx_strm, per_file_default_level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY) != Z_OK) { rprintf(FERROR, "compression init failed\n"); From ffbca80ca2347beaa1004db56db439cdfa59f59e Mon Sep 17 00:00:00 2001 From: Rodrigo Osorio Date: Sun, 2 Jan 2022 23:37:27 +0100 Subject: [PATCH 027/350] Time-limit options are not being checked enough (#179) The `--stop-at`, `--stop-after`, and `--time-limit`` options should have their limit checked when receiving and sending data, not just when receiving. Fixes #177. --- io.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/io.c b/io.c index abb0f7ab1..8b5c690fa 100644 --- a/io.c +++ b/io.c @@ -801,12 +801,8 @@ static char *perform_io(size_t needed, int flags) who_am_i(), (SIZE_T_FMT_CAST)n); } - if (io_timeout || stop_at_utime) { + if (io_timeout) { last_io_in = time(NULL); - if (stop_at_utime && last_io_in >= stop_at_utime) { - rprintf(FERROR, "stopping at requested limit\n"); - exit_cleanup(RERR_TIMEOUT); - } if (io_timeout && flags & PIO_NEED_INPUT) maybe_send_keepalive(last_io_in, 0); } @@ -815,6 +811,11 @@ static char *perform_io(size_t needed, int flags) iobuf.in.len += n; } + if (stop_at_utime && time(NULL) >= stop_at_utime) { + rprintf(FERROR, "stopping at requested limit\n"); + exit_cleanup(RERR_TIMEOUT); + } + if (out && FD_ISSET(iobuf.out_fd, &w_fds)) { size_t len = iobuf.raw_flushing_ends_before ? iobuf.raw_flushing_ends_before - out->pos : out->len; ssize_t n; From 4a7ba3cfaf99f2ad42a7b810fead3cee75d9205b Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Sun, 2 Jan 2022 14:43:30 -0800 Subject: [PATCH 028/350] A couple man page improvements. --- rsync.1.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/rsync.1.md b/rsync.1.md index c46fe2bad..37e2f9b01 100644 --- a/rsync.1.md +++ b/rsync.1.md @@ -535,6 +535,18 @@ your home directory (remove the '=' for that). being skipped and slightly more information at the end. More than two `-v` options should only be used if you are debugging rsync. + The end-of-run summary tells you the number of bytes sent to the remote + rsync (which is the receiving side on a local copy), the number of bytes + received from the remote host, and the average bytes per second of the + transferred data computed over the entire length of the rsync run. The + second line shows the total size (in bytes), which is the sum of all the + file sizes that rsync considered transferring. It also shows a "speedup" + value, which is a ratio of the total file size divided by the sum of the + sent and received bytes (which is really just a feel-good bigger-is-better + number). Note that these byte values can be made more (or less) + human-readable by using the `--human-readable` (or `--no-human-readable`) + options. + In a modern rsync, the `-v` option is equivalent to the setting of groups of `--info` and `--debug` options. You can choose to use these newer options in addition to, or in place of using `--verbose`, as any @@ -899,6 +911,13 @@ your home directory (remove the '=' for that). data that goes into the file-lists, and thus it doesn't affect deletions. It just limits the files that the receiver requests to be transferred. + A caution for those that choose to combine `--inplace` with `--update`: an + interrupted transfer will leave behind a partial file on the receiving side + that has a very recent modified time, so re-running the transfer will + probably **not** continue the interrutped file. As such, it is usually + best to avoid combining this with `--inplace` unless you have implemented + manual steps to handle any interrutped in-progress files. + 0. `--inplace` This option changes how rsync transfers a file when its data needs to be From 4adfdaaf12db26c348b4d6150119b377f9b622c8 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Sun, 2 Jan 2022 14:46:27 -0800 Subject: [PATCH 029/350] Tweak stderr handling for older BackupPC versions This makes the default for a protocol-28 server process be --stderr=client instead of --stderr=errors. See rsync's github issue #95. --- NEWS.md | 3 +++ compat.c | 5 +++++ options.c | 5 +++++ 3 files changed, 13 insertions(+) diff --git a/NEWS.md b/NEWS.md index 6635ccab2..05215c08a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -130,6 +130,9 @@ - Work around a glibc bug where lchmod() breaks in a chroot w/o /proc mounted. + - Try to support a client that sent a remote rsync a wacko stderr file handle + (such as an older File::RsyncP perl library used by BackupPC). + - Some manpage improvements. ### PACKAGING RELATED: diff --git a/compat.c b/compat.c index 0a882cda0..e84cc8c03 100644 --- a/compat.c +++ b/compat.c @@ -52,6 +52,8 @@ extern int need_messages_from_generator; extern int delete_mode, delete_before, delete_during, delete_after; extern int do_compression; extern int do_compression_level; +extern int saw_stderr_opt; +extern int msgs2stderr; extern char *shell_cmd; extern char *partial_dir; extern char *files_from; @@ -622,6 +624,9 @@ void setup_protocol(int f_out,int f_in) if (read_batch) check_batch_flags(); + if (!saw_stderr_opt && protocol_version <= 28 && am_server) + msgs2stderr = 0; /* The client side may not have stderr setup for us. */ + #ifndef SUPPORT_PREALLOCATION if (preallocate_files && !am_sender) { rprintf(FERROR, "preallocation is not supported on this %s\n", diff --git a/options.c b/options.c index eb5744a12..eb16c4f6d 100644 --- a/options.c +++ b/options.c @@ -91,6 +91,7 @@ int implied_dirs = 1; int missing_args = 0; /* 0 = FERROR_XFER, 1 = ignore, 2 = delete */ int numeric_ids = 0; int msgs2stderr = 2; /* Default: send errors to stderr for local & remote-shell transfers */ +int saw_stderr_opt = 0; int allow_8bit_chars = 0; int force_delete = 0; int io_timeout = 0; @@ -1882,6 +1883,7 @@ int parse_arguments(int *argc_p, const char ***argv_p) "--stderr mode \"%s\" is not one of errors, all, or client\n", arg); return 0; } + saw_stderr_opt = 1; break; } @@ -1900,6 +1902,9 @@ int parse_arguments(int *argc_p, const char ***argv_p) } } + if (msgs2stderr != 2) + saw_stderr_opt = 1; + if (version_opt_cnt) { print_rsync_version(FINFO); exit_cleanup(0); From 3e44bbd3132c22f7ce1ae48c0ce262f237473038 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Sun, 2 Jan 2022 15:13:19 -0800 Subject: [PATCH 030/350] Preparing for release of 3.2.4pre1 --- access.c | 2 +- acls.c | 2 +- backup.c | 2 +- clientname.c | 2 +- clientserver.c | 2 +- compat.c | 2 +- flist.c | 2 +- generator.c | 2 +- ifuncs.h | 2 +- io.c | 2 +- itypes.h | 2 +- latest-year.h | 2 +- lib/sysxattrs.c | 2 +- log.c | 2 +- main.c | 2 +- options.c | 2 +- packaging/lsb/rsync.spec | 12 ++++++------ receiver.c | 2 +- rsync.c | 2 +- rsync.h | 2 +- runtests.sh | 2 +- sender.c | 2 +- syscall.c | 2 +- t_stub.c | 2 +- testsuite/backup.test | 2 +- testsuite/chmod-temp-dir.test | 2 +- testsuite/chmod.test | 2 +- testsuite/compare-dest.test | 2 +- testsuite/delete.test | 2 +- testsuite/exclude.test | 2 +- testsuite/fuzzy.test | 2 +- testsuite/itemize.test | 2 +- testsuite/merge.test | 2 +- testsuite/wildmatch.test | 2 +- tls.c | 2 +- token.c | 2 +- util1.c | 2 +- version.h | 2 +- xattrs.c | 2 +- 39 files changed, 44 insertions(+), 44 deletions(-) diff --git a/access.c b/access.c index f6d6e272f..0e076b2d5 100644 --- a/access.c +++ b/access.c @@ -2,7 +2,7 @@ * Routines to authenticate access to a daemon (hosts allow/deny). * * Copyright (C) 1998 Andrew Tridgell - * Copyright (C) 2004-2020 Wayne Davison + * Copyright (C) 2004-2021 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/acls.c b/acls.c index c98f7b40b..5658274dc 100644 --- a/acls.c +++ b/acls.c @@ -3,7 +3,7 @@ * * Copyright (C) 1996 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras - * Copyright (C) 2006-2020 Wayne Davison + * Copyright (C) 2006-2021 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/backup.c b/backup.c index 5036c23a4..9d266ef45 100644 --- a/backup.c +++ b/backup.c @@ -2,7 +2,7 @@ * Backup handling code. * * Copyright (C) 1999 Andrew Tridgell - * Copyright (C) 2003-2020 Wayne Davison + * Copyright (C) 2003-2021 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/clientname.c b/clientname.c index 1572f3ef0..e0f76203f 100644 --- a/clientname.c +++ b/clientname.c @@ -3,7 +3,7 @@ * * Copyright (C) 1992-2001 Andrew Tridgell * Copyright (C) 2001, 2002 Martin Pool - * Copyright (C) 2002-2020 Wayne Davison + * Copyright (C) 2002-2021 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/clientserver.c b/clientserver.c index 14e3c544a..8852076b0 100644 --- a/clientserver.c +++ b/clientserver.c @@ -3,7 +3,7 @@ * * Copyright (C) 1998-2001 Andrew Tridgell * Copyright (C) 2001-2002 Martin Pool - * Copyright (C) 2002-2020 Wayne Davison + * Copyright (C) 2002-2021 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/compat.c b/compat.c index e84cc8c03..55d76df8f 100644 --- a/compat.c +++ b/compat.c @@ -3,7 +3,7 @@ * * Copyright (C) Andrew Tridgell 1996 * Copyright (C) Paul Mackerras 1996 - * Copyright (C) 2004-2020 Wayne Davison + * Copyright (C) 2004-2022 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/flist.c b/flist.c index 803490bda..4f749bca5 100644 --- a/flist.c +++ b/flist.c @@ -4,7 +4,7 @@ * Copyright (C) 1996 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras * Copyright (C) 2001, 2002 Martin Pool - * Copyright (C) 2002-2020 Wayne Davison + * Copyright (C) 2002-2021 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/generator.c b/generator.c index 6e1cbe916..1abe036b4 100644 --- a/generator.c +++ b/generator.c @@ -4,7 +4,7 @@ * Copyright (C) 1996-2000 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras * Copyright (C) 2002 Martin Pool - * Copyright (C) 2003-2020 Wayne Davison + * Copyright (C) 2003-2021 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/ifuncs.h b/ifuncs.h index 491f0807e..b5823b78a 100644 --- a/ifuncs.h +++ b/ifuncs.h @@ -1,6 +1,6 @@ /* Inline functions for rsync. * - * Copyright (C) 2007-2020 Wayne Davison + * Copyright (C) 2007-2021 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/io.c b/io.c index 8b5c690fa..8ee48f935 100644 --- a/io.c +++ b/io.c @@ -4,7 +4,7 @@ * Copyright (C) 1996-2001 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras * Copyright (C) 2001, 2002 Martin Pool - * Copyright (C) 2003-2020 Wayne Davison + * Copyright (C) 2003-2022 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/itypes.h b/itypes.h index a78300b3b..0e23c85bc 100644 --- a/itypes.h +++ b/itypes.h @@ -1,6 +1,6 @@ /* Inline functions for rsync. * - * Copyright (C) 2007-2020 Wayne Davison + * Copyright (C) 2007-2021 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/latest-year.h b/latest-year.h index ddbf2b5ef..37e8efbb8 100644 --- a/latest-year.h +++ b/latest-year.h @@ -1 +1 @@ -#define LATEST_YEAR "2020" +#define LATEST_YEAR "2022" diff --git a/lib/sysxattrs.c b/lib/sysxattrs.c index d403caf57..e747de37f 100644 --- a/lib/sysxattrs.c +++ b/lib/sysxattrs.c @@ -2,7 +2,7 @@ * Extended attribute support for rsync. * * Copyright (C) 2004 Red Hat, Inc. - * Copyright (C) 2003-2019 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * Written by Jay Fenlason. * * This program is free software; you can redistribute it and/or modify diff --git a/log.c b/log.c index 45ae5c1bb..6c50d0e00 100644 --- a/log.c +++ b/log.c @@ -3,7 +3,7 @@ * * Copyright (C) 1998-2001 Andrew Tridgell * Copyright (C) 2000-2001 Martin Pool - * Copyright (C) 2003-2020 Wayne Davison + * Copyright (C) 2003-2021 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/main.c b/main.c index 8b145867d..ef7a3010c 100644 --- a/main.c +++ b/main.c @@ -4,7 +4,7 @@ * Copyright (C) 1996-2001 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras * Copyright (C) 2001, 2002 Martin Pool - * Copyright (C) 2003-2020 Wayne Davison + * Copyright (C) 2003-2021 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/options.c b/options.c index eb16c4f6d..f4c159f36 100644 --- a/options.c +++ b/options.c @@ -3,7 +3,7 @@ * * Copyright (C) 1998-2001 Andrew Tridgell * Copyright (C) 2000, 2001, 2002 Martin Pool - * Copyright (C) 2002-2020 Wayne Davison + * Copyright (C) 2002-2022 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/packaging/lsb/rsync.spec b/packaging/lsb/rsync.spec index 04163c72b..f6f9135d7 100644 --- a/packaging/lsb/rsync.spec +++ b/packaging/lsb/rsync.spec @@ -1,9 +1,9 @@ Summary: A fast, versatile, remote (and local) file-copying tool Name: rsync -Version: 3.2.3 -%define fullversion %{version} -Release: 1 -%define srcdir src +Version: 3.2.4 +%define fullversion %{version}pre1 +Release: 0.1.pre1 +%define srcdir src-previews Group: Applications/Internet License: GPL Source0: https://rsync.samba.org/ftp/rsync/%{srcdir}/rsync-%{fullversion}.tar.gz @@ -79,8 +79,8 @@ rm -rf $RPM_BUILD_ROOT %dir /etc/rsync-ssl/certs %changelog -* Thu Aug 06 2020 Wayne Davison -Released 3.2.3. +* Sun Jan 02 2022 Wayne Davison +Released 3.2.4pre1. * Fri Mar 21 2008 Wayne Davison Added installation of /etc/xinetd.d/rsync file and some commented-out diff --git a/receiver.c b/receiver.c index 2cd843517..48d31d8d2 100644 --- a/receiver.c +++ b/receiver.c @@ -3,7 +3,7 @@ * * Copyright (C) 1996-2000 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras - * Copyright (C) 2003-2020 Wayne Davison + * Copyright (C) 2003-2021 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/rsync.c b/rsync.c index 60029c2d8..2bcf8dc67 100644 --- a/rsync.c +++ b/rsync.c @@ -3,7 +3,7 @@ * * Copyright (C) 1996 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras - * Copyright (C) 2003-2020 Wayne Davison + * Copyright (C) 2003-2021 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/rsync.h b/rsync.h index ea1bfea6a..5670d3348 100644 --- a/rsync.h +++ b/rsync.h @@ -2,7 +2,7 @@ * Copyright (C) 1996, 2000 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras * Copyright (C) 2001, 2002 Martin Pool - * Copyright (C) 2003-2020 Wayne Davison + * Copyright (C) 2003-2021 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/runtests.sh b/runtests.sh index 8c573041d..e0236c9ee 100755 --- a/runtests.sh +++ b/runtests.sh @@ -1,7 +1,7 @@ #! /bin/sh # Copyright (C) 2001, 2002 by Martin Pool -# Copyright (C) 2003-2020 Wayne Davison +# Copyright (C) 2003-2021 Wayne Davison # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version diff --git a/sender.c b/sender.c index 83603b995..837a83bd3 100644 --- a/sender.c +++ b/sender.c @@ -3,7 +3,7 @@ * * Copyright (C) 1996 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras - * Copyright (C) 2003-2020 Wayne Davison + * Copyright (C) 2003-2021 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/syscall.c b/syscall.c index 56948a832..390abb36a 100644 --- a/syscall.c +++ b/syscall.c @@ -4,7 +4,7 @@ * * Copyright (C) 1998 Andrew Tridgell * Copyright (C) 2002 Martin Pool - * Copyright (C) 2003-2020 Wayne Davison + * Copyright (C) 2003-2021 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/t_stub.c b/t_stub.c index 12f7b622c..e4816b8fe 100644 --- a/t_stub.c +++ b/t_stub.c @@ -3,7 +3,7 @@ * functions, so that module test harnesses can run standalone. * * Copyright (C) 2001, 2002 Martin Pool - * Copyright (C) 2003-2020 Wayne Davison + * Copyright (C) 2003-2021 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/testsuite/backup.test b/testsuite/backup.test index 925c9fe0b..c479b460c 100644 --- a/testsuite/backup.test +++ b/testsuite/backup.test @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (C) 2004-2020 Wayne Davison +# Copyright (C) 2004-2021 Wayne Davison # This program is distributable under the terms of the GNU GPL (see # COPYING). diff --git a/testsuite/chmod-temp-dir.test b/testsuite/chmod-temp-dir.test index 22b2df816..3f0251ff3 100644 --- a/testsuite/chmod-temp-dir.test +++ b/testsuite/chmod-temp-dir.test @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (C) 2004-2020 Wayne Davison +# Copyright (C) 2004-2021 Wayne Davison # This program is distributable under the terms of the GNU GPL (see # COPYING). diff --git a/testsuite/chmod.test b/testsuite/chmod.test index 63258df7c..6cc589c5a 100644 --- a/testsuite/chmod.test +++ b/testsuite/chmod.test @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (C) 2004-2020 Wayne Davison +# Copyright (C) 2004-2021 Wayne Davison # This program is distributable under the terms of the GNU GPL (see # COPYING). diff --git a/testsuite/compare-dest.test b/testsuite/compare-dest.test index 3c6348576..ff2af33c0 100644 --- a/testsuite/compare-dest.test +++ b/testsuite/compare-dest.test @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (C) 2004-2020 Wayne Davison +# Copyright (C) 2004-2021 Wayne Davison # This program is distributable under the terms of the GNU GPL (see # COPYING). diff --git a/testsuite/delete.test b/testsuite/delete.test index f54214cd2..c720bbcb0 100644 --- a/testsuite/delete.test +++ b/testsuite/delete.test @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (C) 2005-2020 Wayne Davison +# Copyright (C) 2005-2021 Wayne Davison # This program is distributable under the terms of the GNU GPL (see # COPYING). diff --git a/testsuite/exclude.test b/testsuite/exclude.test index bd21c51cc..ef4ba3556 100644 --- a/testsuite/exclude.test +++ b/testsuite/exclude.test @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (C) 2003-2020 Wayne Davison +# Copyright (C) 2003-2021 Wayne Davison # This program is distributable under the terms of the GNU GPL (see # COPYING). diff --git a/testsuite/fuzzy.test b/testsuite/fuzzy.test index 2415173de..f8372aa3c 100644 --- a/testsuite/fuzzy.test +++ b/testsuite/fuzzy.test @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (C) 2005-2020 Wayne Davison +# Copyright (C) 2005-2021 Wayne Davison # This program is distributable under the terms of the GNU GPL (see # COPYING). diff --git a/testsuite/itemize.test b/testsuite/itemize.test index 923272389..1b66d2d76 100644 --- a/testsuite/itemize.test +++ b/testsuite/itemize.test @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (C) 2005-2020 Wayne Davison +# Copyright (C) 2005-2021 Wayne Davison # This program is distributable under the terms of the GNU GPL (see # COPYING). diff --git a/testsuite/merge.test b/testsuite/merge.test index 5f9a6da66..7fccc40db 100644 --- a/testsuite/merge.test +++ b/testsuite/merge.test @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (C) 2004-2020 Wayne Davison +# Copyright (C) 2004-2021 Wayne Davison # This program is distributable under the terms of the GNU GPL (see # COPYING). diff --git a/testsuite/wildmatch.test b/testsuite/wildmatch.test index fa225d90f..6673364c5 100644 --- a/testsuite/wildmatch.test +++ b/testsuite/wildmatch.test @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (C) 2003-2020 Wayne Davison +# Copyright (C) 2003-2021 Wayne Davison # This program is distributable under the terms of the GNU GPL (see # COPYING). diff --git a/tls.c b/tls.c index 966986745..0ddb579d1 100644 --- a/tls.c +++ b/tls.c @@ -2,7 +2,7 @@ * Trivial ls for comparing two directories after running an rsync. * * Copyright (C) 2001, 2002 Martin Pool - * Copyright (C) 2003-2020 Wayne Davison + * Copyright (C) 2003-2021 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/token.c b/token.c index 20d10bc5b..d65869d49 100644 --- a/token.c +++ b/token.c @@ -3,7 +3,7 @@ * * Copyright (C) 1996 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras - * Copyright (C) 2003-2020 Wayne Davison + * Copyright (C) 2003-2021 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/util1.c b/util1.c index 3f337d077..a9e0fd244 100644 --- a/util1.c +++ b/util1.c @@ -4,7 +4,7 @@ * Copyright (C) 1996-2000 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras * Copyright (C) 2001, 2002 Martin Pool - * Copyright (C) 2003-2020 Wayne Davison + * Copyright (C) 2003-2021 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/version.h b/version.h index da21e0fdc..4f6a2cfdf 100644 --- a/version.h +++ b/version.h @@ -1 +1 @@ -#define RSYNC_VERSION "3.2.4dev" +#define RSYNC_VERSION "3.2.4pre1" diff --git a/xattrs.c b/xattrs.c index 3c5491920..e7d98660a 100644 --- a/xattrs.c +++ b/xattrs.c @@ -3,7 +3,7 @@ * Written by Jay Fenlason, vaguely based on the ACLs patch. * * Copyright (C) 2004 Red Hat, Inc. - * Copyright (C) 2006-2020 Wayne Davison + * Copyright (C) 2006-2021 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by From f1a6998df29f0f41f882edf11cb957df78aeb267 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Sun, 2 Jan 2022 23:51:04 -0800 Subject: [PATCH 031/350] Only send the `--no-W` kluge to a receiver. --- options.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/options.c b/options.c index f4c159f36..75165adf9 100644 --- a/options.c +++ b/options.c @@ -2834,7 +2834,7 @@ void server_options(char **args, int *argc_p) } else if (inplace) { args[ac++] = "--inplace"; /* Work around a bug in older rsync versions (on the remote side) for --inplace --sparse */ - if (sparse_files && !whole_file) + if (sparse_files && !whole_file && am_sender) args[ac++] = "--no-W"; } From ee9199b54274aed05776f55806dc18b3e2f20a32 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Mon, 3 Jan 2022 00:02:28 -0800 Subject: [PATCH 032/350] More NEWS improvements. --- NEWS.md | 113 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 65 insertions(+), 48 deletions(-) diff --git a/NEWS.md b/NEWS.md index 05215c08a..339b31ee9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -30,11 +30,14 @@ ### BUG FIXES: - - Fixed a bug with `--inplace` + `--sparse` where the destination file could - get reconstructed with bogus data. This bug can be worked-around in older - rsync versions by also specifying `--no-W -M--no-W`. When running 3.2.4 or - newer for your copy, rsync now sends `--no-W` to the remote rsync in such a - scenario (just in case the remote rsync is a version with this bug). + - Fixed a bug with `--inplace` + `--sparse` (and a lack of `--whole-file`) + where the destination file could get reconstructed with bogus data. Since + the bug can also be avoided by using (the seemingly redundant) `--no-W` on + the receiving side, the latest rsync will now send `--no-W` to a remote + receiver when this option combination occurs. If your client rsync is not + new enough to do this for you (or if you're just paranoid), you can manually + specify `--no-W -M--no-W` (when not using `--whole-file`) to make sure the + bug is avoided. - Fixed a bug with `--mkpath` if a single-file copy specifies an existing destination dir with a non-existing destination filename. @@ -47,9 +50,9 @@ or it is skipped. Fixes a crash that could occur when the size changes to 0 in the middle of the send negotiations. - - When dealing with a special file in an alt-dest hierarchy, rsync now checks - the non-permissions mode bits to ensure that the 2 special files are really - the same. + - When dealing with special files (see `--specials`) in an alt-dest hierarchy, + rsync now checks the non-permission mode bits to ensure that the 2 special + files are really the same before hard-linking them together. - Fixed a bug where `--delay-updates` with stale partial data could cause a file to fail to update. @@ -58,12 +61,16 @@ should only have been output given `--verbose` or `--itemize-changes`. - Avoid a weird failure if you run a local copy with a (useless) `--rsh` - option that contains a `V`. + option that contains a `V` in the command. - Fixed a long-standing compression bug where the compression level of the first file transferred affected the level for all future files. Also, the - per-file compression skipping has apparently not worked in a very long time - (I checked back to 2.6.4), so it is now documented as being ineffective. + per-file compression skipping has apparently never worked, so it is now + documented as being ineffective. + + - Improved how the `--stop-at`, `--stop-after`, and `--time-limit` options + check to see if the allowed time is over, which should make rsync exit more + consistently. ### ENHANCEMENTS: @@ -83,7 +90,8 @@ - The rsync daemon can now handle a client address with an implied "%scope" suffix. - - Added support for `--atimes` on macOS and fixed using using it without `-t`. + - Added support for `--atimes` on macOS and fixed a bug where it wouldn't work + without `--times`. - Rsync can now update the xattrs on a read-only file when your user can temporarily add user-write permission to the file. (It always worked for a @@ -92,8 +100,8 @@ - Rsync can now work around an `--inplace` update of a file that is being refused due to the Linux fs.protected_regular sysctl setting. - - When `--chown`, `--usermap`, or `--groupmap` is used, rsync now implies - the appropriate `--owner` and/or `--group` option. + - When `--chown`, `--usermap`, or `--groupmap` is specified, rsync now makes + sure that the appropriate `--owner` and/or `--group` option is enabled. - Added the `--info=NONREG` setting to control if rsync should warn about non-regular files in the transfer. This is enabled by default (keeping the @@ -102,7 +110,7 @@ - More ASM optimizations from Shark64. - - Transformed support/rrsync into a python script with improvements: + - Transformed rrsync (in support dir) into a python script with improvements: - Security has been beefed up. - The known rsync options were updated to include recent additions. - Make rrsync reject `-L`, `-K`, & `-k` by default to make it harder to @@ -115,33 +123,34 @@ options on the server side. - The log format has been tweaked slightly to add seconds to the timestamp and to output the command executed as a tuple (making the args clearer). - - An rrsync.1 manpage was added. + - An rrsync.1 man page was added. - - Added options to support/lsh to allow the rrsync script to be easily tested. + - Added options to the lsh script (in the support dir) to facilitate rrsync + testing. - - Transformed support/atomic-rsync into a python script and added the ability - to ignore one or more non-zero exit codes. By default, it now ignores code - 24 (file vanished). + - Transformed the atomic-rsync script (in the support dir) into a python + script and added the ability to ignore one or more non-zero exit codes. + By default, it now ignores code 24 (file vanished). - - Improved support/rsync-no-vanished wrapper script to not join stdout & - stderr together. + - Transformed the munge-symlinks script (in the support dir) into python. - - Transformed support/munge-symlinks into a python script. + - Improved the rsync-no-vanished script (in the support dir) to not join + stdout & stderr together. - Work around a glibc bug where lchmod() breaks in a chroot w/o /proc mounted. - Try to support a client that sent a remote rsync a wacko stderr file handle (such as an older File::RsyncP perl library used by BackupPC). - - Some manpage improvements. + - Some man page improvements. ### PACKAGING RELATED: - Give configure the `--with-rrsync` option if you want `make install` to - install the (now python3) rrsync script and its (new) man page. + install the (now python3) rrsync script and its new man page. - - If the rrsync script is installed, make its package depend on python3 and - (suggested but not required) the python3 braceexpand lib. + - If the rrsync script is installed, its package should be changed to depend + on python3 and the (suggested but not mandatory) python3 braceexpand lib. - When creating a package from a non-release version (w/o a git checkout), the packager can elect to create git-version.h and define RSYNC_GITVER to the @@ -153,19 +162,21 @@ - Made SIMD & ASM configure default to "no" on non-Linux hosts due to various reports of problems on NetBSD & macOS hosts. These tests were also tweaked - to support a host_cpu of amd64 in addition to x86_64. + to allow enabling the feature on a host_cpu of amd64 (was only x86_64). - Fixed configure to not fail at the SIMD check when cross-compiling. - - Compile the C files with `-pedantic-errors` when possible so that we get - warned about an overflowed static initialization (among other things). + - Compile the C files with `-pedantic-errors` (when possible) so that we will + get warned if a static initialization overflows in the future (among other + things). - Added a SECURITY.md file. ### DEVELOPER RELATED: - Made it easier to write rsync tests that diff the output while also checking - the status code, and used the idiom to improve the existing tests. + the status code, and used the idiom to improve the existing tests. (See the + `checkdiff` and `checkdiff2` idioms in the `testsuite/*.test` files. - The packaging scripts & related python lib got some minor enhancements. @@ -176,7 +187,12 @@ - Improve the logic in compat.c so that we don't need to try to remember to sprinkle `!local_server` exceptions throughout the protocol logic. - - One more C99 Flexible Array improvement (started in the last release). + - One more C99 Flexible Array improvement (started in the last release) and + make use of the C99 `%zd` format string when printing size_t values (when + possible). + + - Use mallinfo2() instead of mallinfo(), when available. + ------------------------------------------------------------------------------ @@ -649,7 +665,8 @@ - Don't output about a new backup dir without appropriate info verbosity. - - Fixed some issues with the sort functions in support/rsyncstats script. + - Fixed some issues with the sort functions in the rsyncstats script (in the + support dir). - Added a way to specify daemon config lists (e.g. users, groups, etc) that contain spaces (see `auth users` in the latest rsyncd.conf manpage). @@ -860,7 +877,7 @@ non-bundled zlib. See the `--new-compress` and `--old-compress` options in the manpage. - - Added the support/rsync-no-vanished wrapper script. + - Added the rsync-no-vanished shell script (in the support dir). - Made configure more prominently mention when we failed to find yodl (in case the user wants to be able to generate manpages from `*.yo` files). @@ -1193,7 +1210,7 @@ - Fix some issues with the post-processing of the man pages. - - Fixed the user home-dir handling in the support/lsh script. + - Fixed the user home-dir handling in the lsh script (in the support dir). - Some minor manpage improvements. @@ -1311,10 +1328,10 @@ reject an attempt to supply one (can configure `--with-included-popt` if your system's popt library doesn't yet have this fix). - - A couple minor option tweaks to the support/rrsync script, and also some - regex changes that make vim highlighting happier. + - A couple minor option tweaks to the rrsync script (in the support dir), and + also some regex changes that make vim highlighting happier. - - Fixed some issues in the support/mnt-excl script. + - Fixed some issues in the mnt-excl script (in the support dir). - Various manpage improvements. @@ -1524,9 +1541,9 @@ ### ENHANCEMENTS: - - Made the support/atomic-rsync script able to perform a fully atomic update - of the copied hierarchy when the destination is setup using a particular - symlink idiom. + - Made the atomic-rsync script (in the support dir) able to perform a fully + atomic update of the copied hierarchy when the destination is setup using a + particular symlink idiom. ------------------------------------------------------------------------------ @@ -1794,8 +1811,8 @@ - Fixed the inclusion of per-dir merge files from implied dirs. - - Fixed the support/rrsync script to work with the latest options that rsync - sends (including its flag-specifying use of `-e` to the server). + - Fixed the rrsync script (in the support dir) to work with the latest options + that rsync sends (including its flag-specifying use of `-e` to the server). ### ENHANCEMENTS: @@ -2799,10 +2816,10 @@ - Made the `max verbosity` setting in the rsyncd.conf file settable on a per-module basis (which now matches the documentation). - - The support/rrsync script has been upgraded to verify the args of options - that take args (instead of rejecting any such options). The script was also - changed to try to be more secure and to fix a problem in the parsing of a - pull operation that has multiple sources. + - The rrsync script (in the support dir) has been upgraded to verify the args + of options that take args (instead of rejecting any such options). It was + also changed to try to be more secure and to fix a problem in the parsing + of a pull operation that has multiple source args. - Improved the documentation that explains the difference between a normal daemon transfer and a daemon-over remote-shell transfer. @@ -4425,7 +4442,7 @@ | RELEASE DATE | VER. | DATE OF COMMIT\* | PROTOCOL | |--------------|--------|------------------|-------------| -| ?? Sep 2020 | 3.2.4 | | 31 | +| ?? Jan 2022 | 3.2.4 | | 31 | | 06 Aug 2020 | 3.2.3 | | 31 | | 04 Jul 2020 | 3.2.2 | | 31 | | 22 Jun 2020 | 3.2.1 | | 31 | From c98327902086cb6e7f525aa263169344c6306771 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Mon, 3 Jan 2022 00:37:57 -0800 Subject: [PATCH 033/350] Improve rrsync usage and some more NEWS tweaks. --- NEWS.md | 49 +++++++++++++++++++++++---------------------- support/rrsync | 2 +- support/rrsync.1.md | 7 ++++--- 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/NEWS.md b/NEWS.md index 339b31ee9..dc5231368 100644 --- a/NEWS.md +++ b/NEWS.md @@ -110,7 +110,7 @@ - More ASM optimizations from Shark64. - - Transformed rrsync (in support dir) into a python script with improvements: + - Transformed rrsync into a python script with improvements: - Security has been beefed up. - The known rsync options were updated to include recent additions. - Make rrsync reject `-L`, `-K`, & `-k` by default to make it harder to @@ -123,19 +123,19 @@ options on the server side. - The log format has been tweaked slightly to add seconds to the timestamp and to output the command executed as a tuple (making the args clearer). - - An rrsync.1 man page was added. + - An rrsync.1 man page was added (in the support dir with rrsync). - - Added options to the lsh script (in the support dir) to facilitate rrsync - testing. + - Added options to the lsh script to facilitate rrsync testing. (See the + support dir.) - - Transformed the atomic-rsync script (in the support dir) into a python - script and added the ability to ignore one or more non-zero exit codes. - By default, it now ignores code 24 (file vanished). + - Transformed the atomic-rsync script into a python script and added the + ability to ignore one or more non-zero exit codes. By default, it now + ignores code 24, the file-vanished exit code. (See the support dir.) - - Transformed the munge-symlinks script (in the support dir) into python. + - Transformed the munge-symlinks script into python. (See the support dir.) - - Improved the rsync-no-vanished script (in the support dir) to not join - stdout & stderr together. + - Improved the rsync-no-vanished script to not join stdout & stderr together. + (See the support dir.) - Work around a glibc bug where lchmod() breaks in a chroot w/o /proc mounted. @@ -877,7 +877,7 @@ non-bundled zlib. See the `--new-compress` and `--old-compress` options in the manpage. - - Added the rsync-no-vanished shell script (in the support dir). + - Added the rsync-no-vanished shell script. (See the support dir.) - Made configure more prominently mention when we failed to find yodl (in case the user wants to be able to generate manpages from `*.yo` files). @@ -1210,7 +1210,7 @@ - Fix some issues with the post-processing of the man pages. - - Fixed the user home-dir handling in the lsh script (in the support dir). + - Fixed the user home-dir handling in the lsh script. (See the support dir.) - Some minor manpage improvements. @@ -1328,10 +1328,10 @@ reject an attempt to supply one (can configure `--with-included-popt` if your system's popt library doesn't yet have this fix). - - A couple minor option tweaks to the rrsync script (in the support dir), and - also some regex changes that make vim highlighting happier. + - A couple minor option tweaks to the rrsync script, and also some regex + changes that make vim highlighting happier. (See the support dir.) - - Fixed some issues in the mnt-excl script (in the support dir). + - Fixed some issues in the mnt-excl script. (See the support dir.) - Various manpage improvements. @@ -1541,9 +1541,9 @@ ### ENHANCEMENTS: - - Made the atomic-rsync script (in the support dir) able to perform a fully - atomic update of the copied hierarchy when the destination is setup using a - particular symlink idiom. + - Made the atomic-rsync script able to perform a fully atomic update of the + copied hierarchy when the destination is setup using a particular symlink + idiom. (See the support dir.) ------------------------------------------------------------------------------ @@ -1811,8 +1811,9 @@ - Fixed the inclusion of per-dir merge files from implied dirs. - - Fixed the rrsync script (in the support dir) to work with the latest options - that rsync sends (including its flag-specifying use of `-e` to the server). + - Fixed the rrsync script to work with the latest options that rsync sends, + including its flag-specifying use of `-e` to the server. (See the support + dir.) ### ENHANCEMENTS: @@ -2816,10 +2817,10 @@ - Made the `max verbosity` setting in the rsyncd.conf file settable on a per-module basis (which now matches the documentation). - - The rrsync script (in the support dir) has been upgraded to verify the args - of options that take args (instead of rejecting any such options). It was - also changed to try to be more secure and to fix a problem in the parsing - of a pull operation that has multiple source args. + - The rrsync script has been upgraded to verify the args of options that take + args (instead of rejecting any such options). It was also changed to try to + be more secure and to fix a problem in the parsing of a pull operation that + has multiple source args. (See the support dir.) - Improved the documentation that explains the difference between a normal daemon transfer and a daemon-over remote-shell transfer. diff --git a/support/rrsync b/support/rrsync index fe1bc2507..8ea76e053 100755 --- a/support/rrsync +++ b/support/rrsync @@ -360,9 +360,9 @@ if __name__ == '__main__': only_group = arg_parser.add_mutually_exclusive_group() only_group.add_argument('-ro', action='store_true', help="Allow only reading from the DIR. Implies -no-del and -no-lock.") only_group.add_argument('-wo', action='store_true', help="Allow only writing to the DIR.") + arg_parser.add_argument('-munge', action='store_true', help="Enable rsync's --munge-links on the server side.") arg_parser.add_argument('-no-del', action='store_true', help="Disable rsync's --delete* and --remove* options.") arg_parser.add_argument('-no-lock', action='store_true', help="Avoid the single-run (per-user) lock check.") - arg_parser.add_argument('-munge', action='store_true', help="Enable rsync's --munge-links on the server side.") arg_parser.add_argument('-help', '-h', action='help', help="Output this help message and exit.") arg_parser.add_argument('dir', metavar='DIR', help="The restricted directory to use.") args = arg_parser.parse_args() diff --git a/support/rrsync.1.md b/support/rrsync.1.md index 4aedfae94..cbb76ef89 100644 --- a/support/rrsync.1.md +++ b/support/rrsync.1.md @@ -5,7 +5,7 @@ rrsync - a script to setup restricted rsync users via ssh logins # SYNOPSIS ``` -rrsync [-ro|-rw] [-munge] [-no-del] DIR +rrsync [-ro|-rw] [-munge] [-no-del] [-no-lock] DIR ``` # DESCRIPTION @@ -52,10 +52,11 @@ The remainder of this man page is dedicated to using the rrsync script. # OPTION SUMMARY ``` --ro Allow only reading from the DIR. Implies -no-del. +-ro Allow only reading from the DIR. Implies -no-del and -no-lock. -wo Allow only writing to the DIR. --no-del Disable rsync's --delete* and --remove* options. -munge Enable rsync's --munge-links on the server side. +-no-del Disable rsync's --delete* and --remove* options. +-no-lock Avoid the single-run (per-user) lock check. -help, -h Output this help message and exit. ``` From b985123d2ebae4d9950585a717067beb10fd4b5f Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Sun, 9 Jan 2022 11:40:41 -0800 Subject: [PATCH 034/350] Allow someone to specify scratchbase=FOO for runtests.sh. --- runtests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtests.sh b/runtests.sh index e0236c9ee..5c204dc9a 100755 --- a/runtests.sh +++ b/runtests.sh @@ -238,7 +238,7 @@ failed=0 # failure to aid investigation. We don't remove the testtmp subdir at # the end so that it can be configured as a symlink to a filesystem that # has ACLs and xattr support enabled (if desired). -scratchbase="$TOOLDIR"/testtmp +scratchbase="${scratchbase:-$TOOLDIR}"/testtmp echo " scratchbase=$scratchbase" [ -d "$scratchbase" ] || mkdir "$scratchbase" From ff1792edf14e718934de271219f6e95f6497363a Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Sun, 9 Jan 2022 12:20:46 -0800 Subject: [PATCH 035/350] Improve `--copy-links` description. --- rsync.1.md | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/rsync.1.md b/rsync.1.md index 37e2f9b01..2ad467c84 100644 --- a/rsync.1.md +++ b/rsync.1.md @@ -1044,14 +1044,21 @@ your home directory (remove the '=' for that). 0. `--copy-links`, `-L` - When symlinks are encountered, the item that they point to (the referent) - is copied, rather than the symlink. In older versions of rsync, this - option also had the side-effect of telling the receiving side to follow - symlinks, such as symlinks to directories. In a modern rsync such as this - one, you'll need to specify `--keep-dirlinks` (`-K`) to get this extra - behavior. The only exception is when sending files to an rsync that is too - old to understand `-K` -- in that case, the `-L` option will still have the - side-effect of `-K` on that older receiving rsync. + The sender transforms each symlink encountered in the transfer into the + referent item, following the symlink chain to the file or directory that it + references. If a symlink chain is broken, an error is output and the file + is dropped from the transfer. On the receiving side, any existing symlinks + in the destination directories are replaced with the non-symlinks that the + sender specifies (though any destination filenames that do not match a name + in the transfer can remain as symlinks if rsync is not deleting files). + + In versions of rsync prior to 2.6.3, this option also had the side-effect + of telling the receiving side to follow symlinks, such as a symlink to a + directory. A modern rsync does not do this, though you can choose to + specify `--keep-dirlinks` (`-K`) if you want rsync to treat a symlink to a + directory on the receiving side as if it were a real directory. Remember + that it's the version of rsync on the receiving side that determines how it + reacts to existing destination symlinks when this option is in effect. 0. `--copy-unsafe-links` From 3b2804c8158a32e3d3b232430e48955a7dfc9178 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Sun, 9 Jan 2022 14:03:31 -0800 Subject: [PATCH 036/350] Tweak a comment. --- support/lsh.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/support/lsh.sh b/support/lsh.sh index f74a8ef48..db034228b 100755 --- a/support/lsh.sh +++ b/support/lsh.sh @@ -4,7 +4,8 @@ # for testing or for running a local copy where the sender and the # receiver needs to use different options (e.g. --fake-super). If # we get a -l USER option, we try to use "sudo -u USER" to run the -# command. +# command. Supports only the hostnames "localhost" and "lh", with +# the latter implying the --no-cd option. user='' do_cd=y # Default path is user's home dir (just like ssh) unless host is "lh". From 6b8db0f6440b28d26ef807d17517715c47e62bd9 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Sun, 9 Jan 2022 17:35:39 -0800 Subject: [PATCH 037/350] Add an arg-protection idiom using backslash-escapes The new default is to protect args and options from unintended shell interpretation using backslash escapes. See the new `--old-args` option for a way to get the old-style splitting. This idiom was chosen over making `--protect-args` enabled by default because it is more backward compatible (e.g. it works with rrsync). Fixes #272. --- NEWS.md | 18 ++- main.c | 6 +- options.c | 143 ++++++++++++++--------- packaging/{cull_options => cull-options} | 14 ++- rsync.1.md | 87 ++++++++------ support/rrsync | 6 +- 6 files changed, 175 insertions(+), 99 deletions(-) rename packaging/{cull_options => cull-options} (89%) diff --git a/NEWS.md b/NEWS.md index dc5231368..8bab61cfb 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,7 +4,23 @@ ## Changes in this version: -### OUTPUT CHANGES: +### BEHAVIOR CHANGES: + + - A new form of arg protection was added that works similarly to the older + `--protect-args` (`-s`) option but in a way that avoids breaking things like + rrsync (the restricted rsync script): rsync now uses backslash escaping for + sending "shell-active" characters to the remote shell. This includes spaces, + so fetching a remote file via a simple quoted filename value now works by + default without any extra quoting: + + ```shell + rsync -aiv host:'a simple file.pdf' . + ``` + + Wildcards are not escaped in filename args, but they are escaped in options + like the `--suffix` and `--usermap` values. If your rsync script depends on + the old arg-splitting behavior, either run it with the `--old-args` option + or `export RSYNC_OLD_ARGS=1` in the script's environment. - A long-standing bug was preventing rsync from figuring out the current locale's decimal point character, which made rsync always output numbers diff --git a/main.c b/main.c index ef7a3010c..0378f3f3a 100644 --- a/main.c +++ b/main.c @@ -607,11 +607,7 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in rprintf(FERROR, "internal: args[] overflowed in do_cmd()\n"); exit_cleanup(RERR_SYNTAX); } - if (**remote_argv == '-') { - if (asprintf(args + argc++, "./%s", *remote_argv++) < 0) - out_of_memory("do_cmd"); - } else - args[argc++] = *remote_argv++; + args[argc++] = safe_arg(NULL, *remote_argv++); remote_argc--; } } diff --git a/options.c b/options.c index 75165adf9..2dba06e3c 100644 --- a/options.c +++ b/options.c @@ -102,6 +102,7 @@ int filesfrom_fd = -1; char *filesfrom_host = NULL; int eol_nulls = 0; int protect_args = -1; +int old_style_args = -1; int human_readable = 1; int recurse = 0; int mkpath_dest_arg = 0; @@ -780,6 +781,8 @@ static struct poptOption long_options[] = { {"files-from", 0, POPT_ARG_STRING, &files_from, 0, 0, 0 }, {"from0", '0', POPT_ARG_VAL, &eol_nulls, 1, 0, 0}, {"no-from0", 0, POPT_ARG_VAL, &eol_nulls, 0, 0, 0}, + {"old-args", 0, POPT_ARG_VAL, &old_style_args, 1, 0, 0}, + {"no-old-args", 0, POPT_ARG_VAL, &old_style_args, 0, 0, 0}, {"protect-args", 's', POPT_ARG_VAL, &protect_args, 1, 0, 0}, {"no-protect-args", 0, POPT_ARG_VAL, &protect_args, 0, 0, 0}, {"no-s", 0, POPT_ARG_VAL, &protect_args, 0, 0, 0}, @@ -1922,6 +1925,13 @@ int parse_arguments(int *argc_p, const char ***argv_p) max_alloc = size; } + if (old_style_args < 0) { + if (!am_server && (arg = getenv("RSYNC_OLD_ARGS")) != NULL && *arg) + old_style_args = atoi(arg) ? 1 : 0; + else + old_style_args = 0; + } + if (protect_args < 0) { if (am_server) protect_args = 0; @@ -2447,6 +2457,61 @@ int parse_arguments(int *argc_p, const char ***argv_p) } +static char SPLIT_ARG_WHEN_OLD[1]; + +/** + * Do backslash quoting of any weird chars in "arg", append the resulting + * string to the end of the "opt" (which gets a "=" appended if it is not + * an empty or NULL string), and return the (perhaps malloced) result. + * If opt is NULL, arg is considered a filename arg that allows wildcards. + * If it is "" or any other value, it is considered an option. + **/ +char *safe_arg(const char *opt, const char *arg) +{ +#define SHELL_CHARS "!#$&;|<>(){}\"' \t\\" +#define WILD_CHARS "*?[]" /* We don't allow remote brace expansion */ + BOOL is_filename_arg = !opt; + char *escapes = is_filename_arg ? SHELL_CHARS : WILD_CHARS SHELL_CHARS; + BOOL escape_leading_dash = is_filename_arg && *arg == '-'; + int len1 = opt && *opt ? strlen(opt) + 1 : 0; + int len2 = strlen(arg); + int extras = escape_leading_dash ? 2 : 0; + char *ret; + if (!protect_args && (!old_style_args || (!is_filename_arg && opt != SPLIT_ARG_WHEN_OLD))) { + const char *f; + for (f = arg; *f; f++) { + if (strchr(escapes, *f)) + extras++; + } + } + if (!len1 && !extras) + return (char*)arg; + ret = new_array(char, len1 + len2 + extras + 1); + if (len1) { + memcpy(ret, opt, len1-1); + ret[len1-1] = '='; + } + if (escape_leading_dash) { + ret[len1++] = '.'; + ret[len1++] = '/'; + extras -= 2; + } + if (!extras) + memcpy(ret + len1, arg, len2); + else { + const char *f = arg; + char *t = ret + len1; + while (*f) { + if (strchr(escapes, *f)) + *t++ = '\\'; + *t++ = *f++; + } + } + ret[len1+len2+extras] = '\0'; + return ret; +} + + /** * Construct a filtered list of options to pass through from the * client to the server. @@ -2590,9 +2655,7 @@ void server_options(char **args, int *argc_p) set++; else set = iconv_opt; - if (asprintf(&arg, "--iconv=%s", set) < 0) - goto oom; - args[ac++] = arg; + args[ac++] = safe_arg("--iconv", set); } #endif @@ -2658,33 +2721,24 @@ void server_options(char **args, int *argc_p) } if (backup_dir) { + /* This split idiom allows for ~/path expansion via the shell. */ args[ac++] = "--backup-dir"; - args[ac++] = backup_dir; + args[ac++] = safe_arg("", backup_dir); } /* Only send --suffix if it specifies a non-default value. */ - if (strcmp(backup_suffix, backup_dir ? "" : BACKUP_SUFFIX) != 0) { - /* We use the following syntax to avoid weirdness with '~'. */ - if (asprintf(&arg, "--suffix=%s", backup_suffix) < 0) - goto oom; - args[ac++] = arg; - } + if (strcmp(backup_suffix, backup_dir ? "" : BACKUP_SUFFIX) != 0) + args[ac++] = safe_arg("--suffix", backup_suffix); - if (checksum_choice) { - if (asprintf(&arg, "--checksum-choice=%s", checksum_choice) < 0) - goto oom; - args[ac++] = arg; - } + if (checksum_choice) + args[ac++] = safe_arg("--checksum-choice", checksum_choice); if (do_compression == CPRES_ZLIBX) args[ac++] = "--new-compress"; else if (compress_choice && do_compression == CPRES_ZLIB) args[ac++] = "--old-compress"; - else if (compress_choice) { - if (asprintf(&arg, "--compress-choice=%s", compress_choice) < 0) - goto oom; - args[ac++] = arg; - } + else if (compress_choice) + args[ac++] = safe_arg("--compress-choice", compress_choice); if (am_sender) { if (max_delete > 0) { @@ -2693,14 +2747,10 @@ void server_options(char **args, int *argc_p) args[ac++] = arg; } else if (max_delete == 0) args[ac++] = "--max-delete=-1"; - if (min_size >= 0) { - args[ac++] = "--min-size"; - args[ac++] = min_size_arg; - } - if (max_size >= 0) { - args[ac++] = "--max-size"; - args[ac++] = max_size_arg; - } + if (min_size >= 0) + args[ac++] = safe_arg("--min-size", min_size_arg); + if (max_size >= 0) + args[ac++] = safe_arg("--max-size", max_size_arg); if (delete_before) args[ac++] = "--delete-before"; else if (delete_during == 2) @@ -2724,17 +2774,12 @@ void server_options(char **args, int *argc_p) if (do_stats) args[ac++] = "--stats"; } else { - if (skip_compress) { - if (asprintf(&arg, "--skip-compress=%s", skip_compress) < 0) - goto oom; - args[ac++] = arg; - } + if (skip_compress) + args[ac++] = safe_arg("--skip-compress", skip_compress); } - if (max_alloc_arg && max_alloc != DEFAULT_MAX_ALLOC) { - args[ac++] = "--max-alloc"; - args[ac++] = max_alloc_arg; - } + if (max_alloc_arg && max_alloc != DEFAULT_MAX_ALLOC) + args[ac++] = safe_arg("--max-alloc", max_alloc_arg); /* --delete-missing-args needs the cooperation of both sides, but * the sender can handle --ignore-missing-args by itself. */ @@ -2759,7 +2804,7 @@ void server_options(char **args, int *argc_p) if (partial_dir && am_sender) { if (partial_dir != tmp_partialdir) { args[ac++] = "--partial-dir"; - args[ac++] = partial_dir; + args[ac++] = safe_arg("", partial_dir); } if (delay_updates) args[ac++] = "--delay-updates"; @@ -2782,17 +2827,11 @@ void server_options(char **args, int *argc_p) args[ac++] = "--use-qsort"; if (am_sender) { - if (usermap) { - if (asprintf(&arg, "--usermap=%s", usermap) < 0) - goto oom; - args[ac++] = arg; - } + if (usermap) + args[ac++] = safe_arg("--usermap", usermap); - if (groupmap) { - if (asprintf(&arg, "--groupmap=%s", groupmap) < 0) - goto oom; - args[ac++] = arg; - } + if (groupmap) + args[ac++] = safe_arg("--groupmap", groupmap); if (ignore_existing) args[ac++] = "--ignore-existing"; @@ -2803,7 +2842,7 @@ void server_options(char **args, int *argc_p) if (tmpdir) { args[ac++] = "--temp-dir"; - args[ac++] = tmpdir; + args[ac++] = safe_arg("", tmpdir); } if (do_fsync) @@ -2816,7 +2855,7 @@ void server_options(char **args, int *argc_p) */ for (i = 0; i < basis_dir_cnt; i++) { args[ac++] = alt_dest_opt(0); - args[ac++] = basis_dir[i]; + args[ac++] = safe_arg("", basis_dir[i]); } } } @@ -2841,7 +2880,7 @@ void server_options(char **args, int *argc_p) if (files_from && (!am_sender || filesfrom_host)) { if (filesfrom_host) { args[ac++] = "--files-from"; - args[ac++] = files_from; + args[ac++] = safe_arg("", files_from); if (eol_nulls) args[ac++] = "--from0"; } else { @@ -2884,7 +2923,7 @@ void server_options(char **args, int *argc_p) exit_cleanup(RERR_SYNTAX); } for (j = 1; j <= remote_option_cnt; j++) - args[ac++] = (char*)remote_options[j]; + args[ac++] = safe_arg(SPLIT_ARG_WHEN_OLD, remote_options[j]); } *argc_p = ac; diff --git a/packaging/cull_options b/packaging/cull-options similarity index 89% rename from packaging/cull_options rename to packaging/cull-options index ca0611211..955c21f1b 100755 --- a/packaging/cull_options +++ b/packaging/cull-options @@ -6,7 +6,7 @@ import re, argparse short_no_arg = { } -short_with_num = { '@': 1 }; +short_with_num = { '@': 1 } long_opts = { # These include some extra long-args that BackupPC uses: 'block-size': 1, 'daemon': -1, @@ -57,11 +57,13 @@ def main(): continue if last_long_opt: - m = re.search(r'args\[ac\+\+\] = ([^["\s]+);', line) + m = re.search(r'args\[ac\+\+\] = safe_arg\("", ([^[("\s]+)\);', line) if m: long_opts[last_long_opt] = 2 last_long_opt = None continue + if 'args[ac++] = ' in line: + last_long_opt = None m = re.search(r'return "--([^"]+-dest)";', line) if m: @@ -73,7 +75,9 @@ def main(): if not m: m = re.search(r'args\[ac\+\+\] = "--([^"=]+)=', line) if not m: - m = re.search(r'fmt = .*: "--([^"=]+)=', line) + m = re.search(r'args\[ac\+\+\] = safe_arg\("--([^"=]+)"', line) + if not m: + m = re.search(r'fmt = .*: "--([^"=]+)=', line) if m: long_opts[m.group(1)] = 1 last_long_opt = None @@ -81,7 +85,7 @@ def main(): long_opts['files-from'] = 3 txt = """\ -### START of options data produced by the cull_options script. ### +### START of options data produced by the cull-options script. ### # To disable a short-named option, add its letter to this string: """ @@ -119,7 +123,7 @@ def main(): print("}") else: print(");") - print("\n### END of options data produced by the cull_options script. ###") + print("\n### END of options data produced by the cull-options script. ###") def str_assign(name, val, comment=None): diff --git a/rsync.1.md b/rsync.1.md index 2ad467c84..f94e468c5 100644 --- a/rsync.1.md +++ b/rsync.1.md @@ -159,22 +159,16 @@ the hostname omitted. For instance, all these work: > rsync -av host:file1 :file2 host:file{3,4} /dest/ > rsync -av host::modname/file{1,2} host::modname/file3 /dest/ -> rsync -av host::modname/file1 ::modname/file{3,4} +> rsync -av host::modname/file1 ::modname/file{3,4} /dest/ -Older versions of rsync required using quoted spaces in the SRC, like these +**Older versions of rsync** required using quoted spaces in the SRC, like these examples: > rsync -av host:'dir1/file1 dir2/file2' /dest > rsync host::'modname/dir1/file1 modname/dir2/file2' /dest -This word-splitting still works (by default) in the latest rsync, but is not as -easy to use as the first method. - -If you need to transfer a filename that contains whitespace, you can either -specify the `--protect-args` (`-s`) option, or you'll need to escape the -whitespace in a way that the remote shell will understand. For instance: - -> rsync -av host:'file\ name\ with\ spaces' /dest +This word-splitting only works in a modern rsync by using `--old-args` (or its +environment variable) and making sure that `--protect-args` is not enabled. # CONNECTING TO AN RSYNC DAEMON @@ -351,6 +345,7 @@ detailed description below for a complete description. --append append data onto shorter files --append-verify --append w/old data in file checksum --dirs, -d transfer directories without recursing +--old-dirs, --old-d works like --dirs when talking to old rsync --mkpath create the destination's path component --links, -l copy symlinks as symlinks --copy-links, -L transform symlink into referent file/dir @@ -438,6 +433,7 @@ detailed description below for a complete description. --include-from=FILE read include patterns from FILE --files-from=FILE read list of source-file names from FILE --from0, -0 all *-from/filter files are delimited by 0s +--old-args disable the modern arg-protection idiom --protect-args, -s no space-splitting; wildcard chars only --copy-as=USER[:GROUP] specify user & optional group for the copy --address=ADDRESS bind address for outgoing socket to daemon @@ -1962,11 +1958,10 @@ your home directory (remove the '=' for that). cause rsync to have a different idea about what data to expect next over the socket, and that will make it fail in a cryptic fashion. - Note that it is best to use a separate `--remote-option` for each option - you want to pass. This makes your usage compatible with the - `--protect-args` option. If that option is off, any spaces in your remote - options will be split by the remote shell unless you take steps to protect - them. + Note that you should use a separate `-M` option for each remote option you + want to pass. On older rsync versions, the presence of any spaces in the + remote-option arg could cause it to be split into separate remote args, but + this requires the use of `--old-args` in a modern rsync. When performing a local transfer, the "local" side is the sender and the "remote" side is the receiver. @@ -2182,36 +2177,52 @@ your home directory (remove the '=' for that). files specified in a `--filter` rule. It does not affect `--cvs-exclude` (since all names read from a .cvsignore file are split on whitespace). +0. `--old-args` + + This option tells rsync to stop trying to protect the arg values from + unintended word-splitting or other misinterpretation by using its new + backslash-escape idiom. The newest default is for remote filenames to only + allow wildcards characters to be interpretated by the shell while + protecting other shell-interpreted characters (and the args of options get + even wildcards escaped). The only active wildcard characters on the remote + side are: `*`, `?`, `[`, & `]`. + + If you have a script that wants to use old-style arg splitting, this option + should get it going. + + You may also control this setting via the RSYNC_OLD_ARGS environment + variable. If this variable has a non-zero value, this setting will be + enabled by default, otherwise it will be disabled by default. Either state + is overridden by a manually specified positive or negative version of this + option (the negative is `--no-old-args`). + 0. `--protect-args`, `-s` This option sends all filenames and most options to the remote rsync - without allowing the remote shell to interpret them. This means that - spaces are not split in names, and any non-wildcard special characters are - not translated (such as `~`, `$`, `;`, `&`, etc.). Wildcards are expanded - on the remote host by rsync (instead of the shell doing it). + without allowing the remote shell to interpret them. Wildcards are + expanded on the remote host by rsync instead of the shell doing it. + + This is similar to the new-style backslash-escaping of args that was added + in 3.2.4, but supports some extra features and doesn't rely on backslash + escaping in the remote shell. If you use this option with `--iconv`, the args related to the remote side will also be translated from the local to the remote character-set. The translation happens before wild-cards are expanded. See also the `--files-from` option. - You may also control this option via the RSYNC_PROTECT_ARGS environment - variable. If this variable has a non-zero value, this option will be + You may also control this setting via the RSYNC_PROTECT_ARGS environment + variable. If this variable has a non-zero value, this setting will be enabled by default, otherwise it will be disabled by default. Either state is overridden by a manually specified positive or negative version of this option (note that `--no-s` and `--no-protect-args` are the negative - versions). Since this option was first introduced in 3.0.0, you'll need to - make sure it's disabled if you ever need to interact with a remote rsync - that is older than that. + versions). - Rsync can also be configured (at build time) to have this option enabled by - default (with is overridden by both the environment and the command-line). - Run `rsync --version` to check if this is the case, as it will display - "default protect-args" or "optional protect-args" depending on how it was - compiled. + You may need to disable this option when interacting with an older rsync + (one prior to 3.0.0). - This option will eventually become a new default setting at some - as-yet-undetermined point in the future. + Note that this option is incompatible with the use of the restricted rsync + script (`rrsync`) since it hides options from the script's inspection. 0. `--copy-as=USER[:GROUP]` @@ -2679,7 +2690,9 @@ your home directory (remove the '=' for that). The `--usermap` option implies the `--owner` option while the `--groupmap` option implies the `--group` option. - If your shell complains about the wildcards, use `--protect-args` (`-s`). + An older rsync client may need to use `--protect-args` (`-s`) to avoid a + complaint about wildcard characters, but a modern rsync handles this + automatically. 0. `--chown=USER:GROUP` @@ -2694,7 +2707,9 @@ your home directory (remove the '=' for that). "`--usermap=*:foo --groupmap=*:bar`", only easier (with the same implied `--owner` and/or `--group` option). - If your shell complains about the wildcards, use `--protect-args` (`-s`). + An older rsync client may need to use `--protect-args` (`-s`) to avoid a + complaint about wildcard characters, but a modern rsync handles this + automatically. 0. `--timeout=SECONDS` @@ -4145,6 +4160,12 @@ file is included or excluded. Specify a default `--iconv` setting using this environment variable. (First supported in 3.0.0.) +0. `RSYNC_OLD_ARGS` + + Specify a non-zero numeric value if you want the `--old-args` option to be + enabled by default, or a zero value to make sure that it is disabled by + default. (First supported in 3.2.4.) + 0. `RSYNC_PROTECT_ARGS` Specify a non-zero numeric value if you want the `--protect-args` option to diff --git a/support/rrsync b/support/rrsync index 8ea76e053..629aa1828 100755 --- a/support/rrsync +++ b/support/rrsync @@ -21,7 +21,7 @@ LOGFILE = 'rrsync.log' # NOTE: the file must exist for a line to be appended! # NOTE when disabling: check for both a short & long version of the option! -### START of options data produced by the cull_options script. ### +### START of options data produced by the cull-options script. ### # To disable a short-named option, add its letter to this string: short_disabled = 's' @@ -63,7 +63,7 @@ long_opts = { 'files-from': 3, 'force': 0, 'from0': 0, - 'fsync': 2, + 'fsync': 0, 'fuzzy': 0, 'group': 0, 'groupmap': 1, @@ -127,7 +127,7 @@ long_opts = { 'write-devices': -1, } -### END of options data produced by the cull_options script. ### +### END of options data produced by the cull-options script. ### import os, sys, re, argparse, glob, socket, time, subprocess from argparse import RawTextHelpFormatter From 635d8c0632fa9549441ed468ff2cc167800f7fab Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Sun, 9 Jan 2022 18:20:20 -0800 Subject: [PATCH 038/350] A repeated `--old-args` does more escape disabling. --- options.c | 15 +++++++++++---- rsync.1.md | 23 +++++++++++++---------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/options.c b/options.c index 2dba06e3c..0a7b4cc79 100644 --- a/options.c +++ b/options.c @@ -578,7 +578,7 @@ enum {OPT_SERVER = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM, OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_MAX_SIZE, OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG, OPT_BLOCK_SIZE, OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT, OPT_STDERR, - OPT_OLD_COMPRESS, OPT_NEW_COMPRESS, OPT_NO_COMPRESS, + OPT_OLD_COMPRESS, OPT_NEW_COMPRESS, OPT_NO_COMPRESS, OPT_OLD_ARGS, OPT_STOP_AFTER, OPT_STOP_AT, OPT_REFUSED_BASE = 9000}; @@ -781,7 +781,7 @@ static struct poptOption long_options[] = { {"files-from", 0, POPT_ARG_STRING, &files_from, 0, 0, 0 }, {"from0", '0', POPT_ARG_VAL, &eol_nulls, 1, 0, 0}, {"no-from0", 0, POPT_ARG_VAL, &eol_nulls, 0, 0, 0}, - {"old-args", 0, POPT_ARG_VAL, &old_style_args, 1, 0, 0}, + {"old-args", 0, POPT_ARG_NONE, 0, OPT_OLD_ARGS, 0, 0}, {"no-old-args", 0, POPT_ARG_VAL, &old_style_args, 0, 0, 0}, {"protect-args", 's', POPT_ARG_VAL, &protect_args, 1, 0, 0}, {"no-protect-args", 0, POPT_ARG_VAL, &protect_args, 0, 0, 0}, @@ -1608,6 +1608,13 @@ int parse_arguments(int *argc_p, const char ***argv_p) compress_choice = NULL; break; + case OPT_OLD_ARGS: + if (old_style_args <= 0) + old_style_args = 1; + else + old_style_args++; + break; + case 'M': arg = poptGetOptArg(pc); if (*arg != '-') { @@ -1927,7 +1934,7 @@ int parse_arguments(int *argc_p, const char ***argv_p) if (old_style_args < 0) { if (!am_server && (arg = getenv("RSYNC_OLD_ARGS")) != NULL && *arg) - old_style_args = atoi(arg) ? 1 : 0; + old_style_args = atoi(arg); else old_style_args = 0; } @@ -2477,7 +2484,7 @@ char *safe_arg(const char *opt, const char *arg) int len2 = strlen(arg); int extras = escape_leading_dash ? 2 : 0; char *ret; - if (!protect_args && (!old_style_args || (!is_filename_arg && opt != SPLIT_ARG_WHEN_OLD))) { + if (!protect_args && old_style_args < 2 && (!old_style_args || (!is_filename_arg && opt != SPLIT_ARG_WHEN_OLD))) { const char *f; for (f = arg; *f; f++) { if (strchr(escapes, *f)) diff --git a/rsync.1.md b/rsync.1.md index f94e468c5..735325735 100644 --- a/rsync.1.md +++ b/rsync.1.md @@ -2187,14 +2187,16 @@ your home directory (remove the '=' for that). even wildcards escaped). The only active wildcard characters on the remote side are: `*`, `?`, `[`, & `]`. - If you have a script that wants to use old-style arg splitting, this option - should get it going. + If you have a script that wants to use old-style arg splitting in the + filenames, specify this option once. If the remote shell has a problem + with any backslash escapes, specify the option twice. You may also control this setting via the RSYNC_OLD_ARGS environment - variable. If this variable has a non-zero value, this setting will be - enabled by default, otherwise it will be disabled by default. Either state - is overridden by a manually specified positive or negative version of this - option (the negative is `--no-old-args`). + variable. If it has the value "1", rsync will default to a single-option + setting. If it has the value "2" (or more), rsync will default to a + repeated-option setting. If it is "0", you'll get the default escaping + behavior. The environment is always overridden by manually specified + positive or negative options (the negative is `--no-old-args`). 0. `--protect-args`, `-s` @@ -2212,7 +2214,7 @@ your home directory (remove the '=' for that). `--files-from` option. You may also control this setting via the RSYNC_PROTECT_ARGS environment - variable. If this variable has a non-zero value, this setting will be + variable. If it has a non-zero value, this setting will be enabled by default, otherwise it will be disabled by default. Either state is overridden by a manually specified positive or negative version of this option (note that `--no-s` and `--no-protect-args` are the negative @@ -4162,9 +4164,10 @@ file is included or excluded. 0. `RSYNC_OLD_ARGS` - Specify a non-zero numeric value if you want the `--old-args` option to be - enabled by default, or a zero value to make sure that it is disabled by - default. (First supported in 3.2.4.) + Specify a "1" if you want the `--old-args` option to be enabled by default, + a "2" (or more) if you want it to be enabled in the option-repeated state, + or a "0" to make sure that it is disabled by default. (First supported in + 3.2.4.) 0. `RSYNC_PROTECT_ARGS` From e841944b47613c242a016184255b6855c8181d32 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Wed, 12 Jan 2022 16:41:36 -0800 Subject: [PATCH 039/350] Change manpage headings in html to use h2 tags with an id target. --- md-convert | 14 ++++++++-- rsync-ssl.1.md | 24 ++++++++-------- rsync.1.md | 68 ++++++++++++++++++++++----------------------- rsyncd.conf.5.md | 36 ++++++++++++------------ support/rrsync.1.md | 12 ++++---- 5 files changed, 81 insertions(+), 73 deletions(-) diff --git a/md-convert b/md-convert index 7780d06b5..d1266f45b 100755 --- a/md-convert +++ b/md-convert @@ -32,7 +32,7 @@ import os, sys, re, argparse, subprocess, time from html.parser import HTMLParser -CONSUMES_TXT = set('h1 h2 p li pre'.split()) +CONSUMES_TXT = set('h1 h2 h3 p li pre'.split()) HTML_START = """\ @@ -332,6 +332,12 @@ class TransformHtml(HTMLParser): st.at_first_tag_in_dd = False + def add_anchor(self, txt): + st = self.state + txt = txt.lower().replace(' ', '-') + st.html_out.append('') + + def handle_endtag(self, tag): st = self.state if args.debug: @@ -342,10 +348,12 @@ class TransformHtml(HTMLParser): else: txt = None add_to_txt = None - if tag == 'h1': + if tag == 'h1' or tag == 'h2': st.man_out.append(st.p_macro + '.SH "' + manify(txt) + '"\n') - elif tag == 'h2': + self.add_anchor(txt) + elif tag == 'h3': st.man_out.append(st.p_macro + '.SS "' + manify(txt) + '"\n') + self.add_anchor(txt) elif tag == 'p': if st.dt_from == 'p': tag = 'dt' diff --git a/rsync-ssl.1.md b/rsync-ssl.1.md index 32ab31e2c..c09603f98 100644 --- a/rsync-ssl.1.md +++ b/rsync-ssl.1.md @@ -1,14 +1,14 @@ -# NAME +## NAME rsync-ssl - a helper script for connecting to an ssl rsync daemon -# SYNOPSIS +## SYNOPSIS ``` rsync-ssl [--type=SSL_TYPE] RSYNC_ARGS ``` -# DESCRIPTION +## DESCRIPTION The rsync-ssl script helps you to run an rsync copy to/from an rsync daemon that requires ssl connections. @@ -20,7 +20,7 @@ environment. You can specify an overriding port via `--port` or by including it in the normal spot in the URL format, though both of those require your rsync version to be at least 3.2.0. -# OPTIONS +## OPTIONS If the **first** arg is a `--type=SSL_TYPE` option, the script will only use that particular program to open an ssl connection instead of trying to find an @@ -32,7 +32,7 @@ required for this particular option. All the other options are passed through to the rsync command, so consult the **rsync**(1) manpage for more information on how it works. -# ENVIRONMENT VARIABLES +## ENVIRONMENT VARIABLES The ssl helper scripts are affected by the following environment variables: @@ -60,7 +60,7 @@ The ssl helper scripts are affected by the following environment variables: connection type is set to stunnel. If unspecified, the $PATH is searched first for "stunnel4" and then for "stunnel". -# EXAMPLES +## EXAMPLES > rsync-ssl -aiv example.com::mod/ dest @@ -70,11 +70,11 @@ The ssl helper scripts are affected by the following environment variables: > rsync-ssl -aiv rsync://example.com:9874/mod/ dest -# SEE ALSO +## SEE ALSO **rsync**(1), **rsyncd.conf**(5) -# CAVEATS +## CAVEATS Note that using an stunnel connection requires at least version 4 of stunnel, which should be the case on modern systems. Also, it does not verify a @@ -87,15 +87,15 @@ release the gnutls-cli command was dropping output, making it unusable. If that bug has been fixed in your version, feel free to put gnutls into an exported RSYNC_SSL_TYPE environment variable to make its use the default. -# BUGS +## BUGS Please report bugs! See the web site at . -# VERSION +## VERSION This man page is current for version @VERSION@ of rsync. -# CREDITS +## CREDITS rsync is distributed under the GNU General Public License. See the file COPYING for details. @@ -103,7 +103,7 @@ COPYING for details. A web site is available at . The site includes an FAQ-O-Matic which may cover questions unanswered by this manual page. -# AUTHOR +## AUTHOR This manpage was written by Wayne Davison. diff --git a/rsync.1.md b/rsync.1.md index 735325735..0a4d58684 100644 --- a/rsync.1.md +++ b/rsync.1.md @@ -1,8 +1,8 @@ -# NAME +## NAME rsync - a fast, versatile, remote (and local) file-copying tool -# SYNOPSIS +## SYNOPSIS ``` Local: @@ -26,7 +26,7 @@ Access via rsync daemon: Usages with just one SRC arg and no DEST arg will list the source files instead of copying. -# DESCRIPTION +## DESCRIPTION Rsync is a fast and extraordinarily versatile file copying tool. It can copy locally, to/from another host over any remote shell, or to/from a remote rsync @@ -54,7 +54,7 @@ Some of the additional features of rsync are: - pipelining of file transfers to minimize latency costs - support for anonymous or authenticated rsync daemons (ideal for mirroring) -# GENERAL +## GENERAL Rsync copies files either to or from a remote host, or locally on the current host (it does not support copying files between two remote hosts). @@ -79,7 +79,7 @@ Rsync refers to the local side as the client and the remote side as the server. Don't confuse server with an rsync daemon. A daemon is always a server, but a server can be either a daemon or a remote-shell spawned process. -# SETUP +## SETUP See the file README.md for installation instructions. @@ -94,7 +94,7 @@ command line option, or by setting the RSYNC_RSH environment variable. Note that rsync must be installed on both the source and destination machines. -# USAGE +## USAGE You use rsync in the same way you use rcp. You must specify a source and a destination, one of which may be remote. @@ -151,7 +151,7 @@ rsync daemon by leaving off the module name: See the following section for more details. -# ADVANCED USAGE +## ADVANCED USAGE The syntax for requesting multiple files from a remote host is done by specifying additional remote-host args in the same style as the first, or with @@ -170,7 +170,7 @@ examples: This word-splitting only works in a modern rsync by using `--old-args` (or its environment variable) and making sure that `--protect-args` is not enabled. -# CONNECTING TO AN RSYNC DAEMON +## CONNECTING TO AN RSYNC DAEMON It is also possible to use rsync without a remote shell as the transport. In this case you will directly connect to a remote rsync daemon, typically using @@ -227,7 +227,7 @@ Note also that if the RSYNC_SHELL environment variable is set, that program will be used to run the RSYNC_CONNECT_PROG command instead of using the default shell of the **system()** call. -# USING RSYNC-DAEMON FEATURES VIA A REMOTE-SHELL CONNECTION +## USING RSYNC-DAEMON FEATURES VIA A REMOTE-SHELL CONNECTION It is sometimes useful to use various features of an rsync daemon (such as named modules) without actually allowing any new socket connections into a @@ -260,7 +260,7 @@ example that uses the short version of the `--rsh` option: The "ssh-user" will be used at the ssh level; the "rsync-user" will be used to log-in to the "module". -# STARTING AN RSYNC DAEMON TO ACCEPT CONNECTIONS +## STARTING AN RSYNC DAEMON TO ACCEPT CONNECTIONS In order to connect to an rsync daemon, the remote system needs to have a daemon already running (or it needs to have configured something like inetd to @@ -273,7 +273,7 @@ the daemon, and it contains the full details for how to run the daemon If you're using one of the remote-shell transports for the transfer, there is no need to manually start an rsync daemon. -# SORTED TRANSFER ORDER +## SORTED TRANSFER ORDER Rsync always sorts the specified filenames into its internal transfer list. This handles the merging together of the contents of identically named @@ -286,7 +286,7 @@ separate the files into different rsync calls, or consider using `--delay-updates` (which doesn't affect the sorted transfer order, but does make the final file-updating phase happen much more rapidly). -# EXAMPLES +## EXAMPLES Here are some examples of how I use rsync. @@ -316,7 +316,7 @@ I mirror a directory between my "old" and "new" ftp sites with the command: This is launched from cron every few hours. -# OPTION SUMMARY +## OPTION SUMMARY Here is a short summary of the options available in rsync. Please refer to the detailed description below for a complete description. @@ -492,7 +492,7 @@ accepted: --help, -h show this help (when used with --daemon) ``` -# OPTIONS +## OPTIONS Rsync accepts both long (double-dash + word) and short (single-dash + letter) options. The full list of the available options are described below. If an @@ -3429,7 +3429,7 @@ your home directory (remove the '=' for that). user wants a more random checksum seed. Setting NUM to 0 causes rsync to use the default of **time**() for checksum seed. -# DAEMON OPTIONS +## DAEMON OPTIONS The options allowed when starting an rsync daemon are as follows: @@ -3537,7 +3537,7 @@ The options allowed when starting an rsync daemon are as follows: When specified after `--daemon`, print a short help page describing the options available for starting an rsync daemon. -# FILTER RULES +## FILTER RULES The filter rules allow for flexible selection of which files to transfer (include) and which files to skip (exclude). The rules either directly specify @@ -3594,7 +3594,7 @@ rule/pattern each. To add multiple ones, you can repeat the options on the command-line, use the merge-file syntax of the `--filter` option, or the `--include-from` / `--exclude-from` options. -# INCLUDE/EXCLUDE PATTERN RULES +## INCLUDE/EXCLUDE PATTERN RULES You can include and exclude files by specifying patterns using the "+", "-", etc. filter rules (as introduced in the FILTER RULES section above). The @@ -3722,7 +3722,7 @@ The following modifiers are accepted after a "`+`" or "`-`": xattr-matching rules are specified, a default xattr filtering rule is used (see the `--xattrs` option). -# MERGE-FILE FILTER RULES +## MERGE-FILE FILTER RULES You can merge whole files into your filter rules by specifying either a merge (.) or a dir-merge (:) filter rule (as introduced in the FILTER RULES section @@ -3856,7 +3856,7 @@ $HOME/.cvsignore, and the value of $CVSIGNORE) you should omit the `-C` command-line option and instead insert a "-C" rule into your filter rules; e.g. "`--filter=-C`". -# LIST-CLEARING FILTER RULE +## LIST-CLEARING FILTER RULE You can clear the current include/exclude list by using the "!" filter rule (as introduced in the FILTER RULES section above). The "current" list is either @@ -3864,7 +3864,7 @@ the global list of rules (if the rule is encountered while parsing the filter options) or a set of per-directory rules (which are inherited in their own sub-list, so a subdirectory can use this to clear out the parent's rules). -# ANCHORING INCLUDE/EXCLUDE PATTERNS +## ANCHORING INCLUDE/EXCLUDE PATTERNS As mentioned earlier, global include/exclude patterns are anchored at the "root of the transfer" (as opposed to per-directory patterns, which are anchored at @@ -3919,7 +3919,7 @@ The easiest way to see what name you should filter is to just look at the output when using `--verbose` and put a / in front of the name (use the `--dry-run` option if you're not yet ready to copy any files). -# PER-DIRECTORY RULES AND DELETE +## PER-DIRECTORY RULES AND DELETE Without a delete option, per-directory rules are only relevant on the sending side, so you can feel free to exclude the merge files themselves without @@ -3965,7 +3965,7 @@ one of these commands: > rsync -avFF --delete host:src/dir /dest > ``` -# BATCH MODE +## BATCH MODE Batch mode can be used to apply the same set of updates to many identical systems. Suppose one has a tree which is replicated on a number of hosts. Now @@ -4063,7 +4063,7 @@ this detail and just use the shell script as an easy way to run the appropriate The original batch mode in rsync was based on "rsync+", but the latest version uses a new implementation. -# SYMBOLIC LINKS +## SYMBOLIC LINKS Three basic behaviors are possible when rsync encounters a symbolic link in the source directory. @@ -4102,7 +4102,7 @@ first line that is a complete subset of your options: 0. `--links --safe-links` Duplicate safe symlinks and skip unsafe ones. 0. `--links` Duplicate all symlinks. -# DIAGNOSTICS +## DIAGNOSTICS rsync occasionally produces error messages that may seem a little cryptic. The one that seems to cause the most confusion is "protocol version mismatch -- is @@ -4125,7 +4125,7 @@ If you are having trouble debugging filter patterns, then try specifying the `-vv` option. At this level of verbosity rsync will show why each individual file is included or excluded. -# EXIT VALUES +## EXIT VALUES 0. **0** Success 0. **1** Syntax or usage error @@ -4150,7 +4150,7 @@ file is included or excluded. 0. **30** Timeout in data send/receive 0. **35** Timeout waiting for daemon connection -# ENVIRONMENT VARIABLES +## ENVIRONMENT VARIABLES 0. `CVSIGNORE` @@ -4206,15 +4206,15 @@ file is included or excluded. The HOME environment variable is used to find the user's default .cvsignore file. -# FILES +## FILES /etc/rsyncd.conf or rsyncd.conf -# SEE ALSO +## SEE ALSO **rsync-ssl**(1), **rsyncd.conf**(5) -# BUGS +## BUGS times are transferred as \*nix time_t values @@ -4229,11 +4229,11 @@ see also the comments on the `--delete` option Please report bugs! See the web site at . -# VERSION +## VERSION This man page is current for version @VERSION@ of rsync. -# INTERNAL OPTIONS +## INTERNAL OPTIONS The options `--server` and `--sender` are used internally by rsync, and should never be typed by a user under normal circumstances. Some awareness of these @@ -4242,7 +4242,7 @@ that can only run an rsync command. For instance, the support directory of the rsync distribution has an example script named rrsync (for restricted rsync) that can be used with a restricted ssh login. -# CREDITS +## CREDITS rsync is distributed under the GNU General Public License. See the file COPYING for details. @@ -4256,7 +4256,7 @@ contact the mailing-list at . This program uses the excellent zlib compression library written by Jean-loup Gailly and Mark Adler. -# THANKS +## THANKS Special thanks go out to: John Van Essen, Matt McCutchen, Wesley W. Terpstra, David Dykstra, Jos Backus, Sebastian Krahmer, Martin Pool, and our @@ -4265,7 +4265,7 @@ gone-but-not-forgotten compadre, J.W. Schultz. Thanks also to Richard Brent, Brendan Mackay, Bill Waite, Stephen Rothwell and David Bell. I've probably missed some people, my apologies if I have. -# AUTHOR +## AUTHOR rsync was originally written by Andrew Tridgell and Paul Mackerras. Many people have later contributed to it. It is currently maintained by Wayne diff --git a/rsyncd.conf.5.md b/rsyncd.conf.5.md index 258322bdd..9d1ad14c1 100644 --- a/rsyncd.conf.5.md +++ b/rsyncd.conf.5.md @@ -1,12 +1,12 @@ -# NAME +## NAME rsyncd.conf - configuration file for rsync in daemon mode -# SYNOPSIS +## SYNOPSIS rsyncd.conf -# DESCRIPTION +## DESCRIPTION The rsyncd.conf file is the runtime configuration file for rsync when run as an rsync daemon. @@ -14,7 +14,7 @@ rsync daemon. The rsyncd.conf file controls authentication, access, logging and available modules. -# FILE FORMAT +## FILE FORMAT The file consists of modules and parameters. A module begins with the name of the module in square brackets and continues until the next module begins. @@ -40,7 +40,7 @@ The values following the equals sign in parameters are all either a string (no quotes needed) or a boolean, which may be given as yes/no, 0/1 or true/false. Case is not significant in boolean values, but is preserved in string values. -# LAUNCHING THE RSYNC DAEMON +## LAUNCHING THE RSYNC DAEMON The rsync daemon is launched by specifying the `--daemon` option to rsync. @@ -69,7 +69,7 @@ reread its config file. Note that you should **not** send the rsync daemon a HUP signal to force it to reread the `rsyncd.conf` file. The file is re-read on each client connection. -# GLOBAL PARAMETERS +## GLOBAL PARAMETERS The first parameters in the file (before a [module] header) are the global parameters. Rsync also allows for the use of a "[global]" module name to @@ -136,7 +136,7 @@ a literal % into a value is to use %%. You can override the default backlog value when the daemon listens for connections. It defaults to 5. -# MODULE PARAMETERS +## MODULE PARAMETERS After the global parameters you should define a number of modules, each module exports a directory tree as a symbolic name. Modules are exported by specifying @@ -1038,7 +1038,7 @@ the values of parameters. See the GLOBAL PARAMETERS section for more details. **system()** call's default shell), and use RSYNC_NO_XFER_EXEC to disable both options completely. -# CONFIG DIRECTIVES +## CONFIG DIRECTIVES There are currently two config directives available that allow a config file to incorporate the contents of other files: `&include` and `&merge`. Both allow @@ -1093,7 +1093,7 @@ This would merge any `/etc/rsyncd.d/*.inc` files (for global values that should stay in effect), and then include any `/etc/rsyncd.d/*.conf` files (defining modules without any global-value cross-talk). -# AUTHENTICATION STRENGTH +## AUTHENTICATION STRENGTH The authentication protocol used in rsync is a 128 bit MD4 based challenge response system. This is fairly weak protection, though (with at least one @@ -1108,7 +1108,7 @@ authentication is provided. Use ssh as the transport if you want encryption. You can also make use of SSL/TLS encryption if you put rsync behind an SSL proxy. -# SSL/TLS Daemon Setup +## SSL/TLS Daemon Setup When setting up an rsync daemon for access via SSL/TLS, you will need to configure a proxy (such as haproxy or nginx) as the front-end that handles the @@ -1153,7 +1153,7 @@ An example nginx proxy setup is as follows: > } > ``` -# EXAMPLES +## EXAMPLES A simple rsyncd.conf file that allow anonymous rsync to a ftp area at `/home/ftp` would be: @@ -1202,24 +1202,24 @@ The /etc/rsyncd.secrets file would look something like this: > tridge:mypass > susan:herpass -# FILES +## FILES /etc/rsyncd.conf or rsyncd.conf -# SEE ALSO +## SEE ALSO **rsync**(1), **rsync-ssl**(1) -# BUGS +## BUGS Please report bugs! The rsync bug tracking system is online at . -# VERSION +## VERSION This man page is current for version @VERSION@ of rsync. -# CREDITS +## CREDITS rsync is distributed under the GNU General Public License. See the file COPYING for details. @@ -1233,12 +1233,12 @@ We would be delighted to hear from you if you like this program. This program uses the zlib compression library written by Jean-loup Gailly and Mark Adler. -# THANKS +## THANKS Thanks to Warren Stanley for his original idea and patch for the rsync daemon. Thanks to Karsten Thygesen for his many suggestions and documentation! -# AUTHOR +## AUTHOR rsync was written by Andrew Tridgell and Paul Mackerras. Many people have later contributed to it. diff --git a/support/rrsync.1.md b/support/rrsync.1.md index cbb76ef89..8108bbf4d 100644 --- a/support/rrsync.1.md +++ b/support/rrsync.1.md @@ -1,14 +1,14 @@ -# NAME +## NAME rrsync - a script to setup restricted rsync users via ssh logins -# SYNOPSIS +## SYNOPSIS ``` rrsync [-ro|-rw] [-munge] [-no-del] [-no-lock] DIR ``` -# DESCRIPTION +## DESCRIPTION A user's ssh login can be restricted to only allow the running of an rsync transfer in one of two easy ways: forcing the running of the rrsync script @@ -49,7 +49,7 @@ file, each one with its own path setting. The remainder of this man page is dedicated to using the rrsync script. -# OPTION SUMMARY +## OPTION SUMMARY ``` -ro Allow only reading from the DIR. Implies -no-del and -no-lock. @@ -63,7 +63,7 @@ The remainder of this man page is dedicated to using the rrsync script. A single non-option argument specifies the restricted DIR to use. It can be relative to the user's home directory or an absolute path. -# SECURITY RESTRICTIONS +## SECURITY RESTRICTIONS The rrsync script validates the path arguments it is sent to try to restrict them to staying within the specified DIR. @@ -84,7 +84,7 @@ overrides. The script (or a copy of it) can be manually edited if you want it to customize the option handling. -# EXAMPLES +## EXAMPLES The `.ssh/authorized_keys` file might have lines in it like this: From 30a590954416fa0ab2a2b27cd8098b310794446d Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Wed, 12 Jan 2022 16:42:40 -0800 Subject: [PATCH 040/350] Some symlink improvements to the man page. --- rsync.1.md | 163 ++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 111 insertions(+), 52 deletions(-) diff --git a/rsync.1.md b/rsync.1.md index 0a4d58684..9b9b6996b 100644 --- a/rsync.1.md +++ b/rsync.1.md @@ -1032,37 +1032,44 @@ your home directory (remove the '=' for that). 0. `--links`, `-l` - When symlinks are encountered, recreate the symlink on the destination. + Add symlinks to the transferred files instead of noisily ignoring them with + a "non-regular file" warning for each symlink encountered. You can + alternately silence the warning by specifying ``--info=nonreg0``. - By default, rsync generates a "non-regular file" warning for each symlink - encountered when this option is not set. You can silence the warning by - specifying ``--info=nonreg0``. + The default handling of symlinks is to recreate each symlink's unchanged + value on the receiving side. + + See the [SYMBOLIC LINKS](#SYMBOLIC-LINKS) section for multi-option info. 0. `--copy-links`, `-L` The sender transforms each symlink encountered in the transfer into the referent item, following the symlink chain to the file or directory that it references. If a symlink chain is broken, an error is output and the file - is dropped from the transfer. On the receiving side, any existing symlinks - in the destination directories are replaced with the non-symlinks that the - sender specifies (though any destination filenames that do not match a name - in the transfer can remain as symlinks if rsync is not deleting files). - - In versions of rsync prior to 2.6.3, this option also had the side-effect - of telling the receiving side to follow symlinks, such as a symlink to a - directory. A modern rsync does not do this, though you can choose to - specify `--keep-dirlinks` (`-K`) if you want rsync to treat a symlink to a - directory on the receiving side as if it were a real directory. Remember - that it's the version of rsync on the receiving side that determines how it - reacts to existing destination symlinks when this option is in effect. + is dropped from the transfer. + + This option supersedes any other options that affect symlinks in the + transfer, since there are no symlinks left in the transfer. + + This option does not change the handling of existing symlinks on the + receiving side, unlike versions of rsync prior to 2.6.3 which had the + side-effect of telling the receiving side to also follow symlinks. A + modern rsync won't forward this option to a remote receiver (since only the + sender needs to know about it), so this caveat should only affect someone + using an rsync client older than 2.6.7 (which is when `-L` stopped being + forwarded to the receiver). + + See the `--keep-dirlinks` (`-K`) if you need a symlink to a directory to be + treated as a real directory on the receiving side. + + See the [SYMBOLIC LINKS](#SYMBOLIC-LINKS) section for multi-option info. 0. `--copy-unsafe-links` This tells rsync to copy the referent of symbolic links that point outside the copied tree. Absolute symlinks are also treated like ordinary files, and so are any symlinks in the source path itself when `--relative` is - used. This option has no additional effect if `--copy-links` was also - specified. + used. Note that the cut-off point is the top of the transfer, which is the part of the path that rsync isn't mentioning in the verbose output. If you copy @@ -1073,32 +1080,68 @@ your home directory (remove the '=' for that). slash) to "/dest/subdir" that would not allow symlinks to any files outside of "subdir". + This option has no additional effect if `--copy-links` was also specified. + The safe symlinks are only copied if `--links` was also specified or + implied by `--archive` (`-a`). + + See the [SYMBOLIC LINKS](#SYMBOLIC-LINKS) section for multi-option info. + 0. `--safe-links` - This tells rsync to ignore any symbolic links which point outside the - copied tree. All absolute symlinks are also ignored. Using this option in - conjunction with `--relative` may give unexpected results. + This tells the receiving rsync to ignore any symbolic links in the transfer + which point outside the copied tree. All absolute symlinks are also + ignored. -0. `--munge-links` + Since this ignoring is happening on the receiving side, it will still be + effective even when the sending side has munged symlinks (when it is using + `--munge-links`). It also affects deletions, since the file being present + in the transfer prevents any matching file on the receiver from being + deleted when the symlink is deemed to be unsafe and is skipped. + + This option must be combined with `--links` (or `--archive`) to have any + symlinks in the transfer to conditionally ignore. Its effect is superseded + by `--copy-unsafe-links`. - This option tells rsync to (1) modify all symlinks on the receiving side in - a way that makes them unusable but recoverable (see below), or (2) to - unmunge symlinks on the sending side that had been stored in a munged - state. This is useful if you don't quite trust the source of the data to - not try to slip in a symlink to a unexpected place. + Using this option in conjunction with `--relative` may give unexpected + results. - The way rsync disables the use of symlinks is to prefix each one with the - string "/rsyncd-munged/". This prevents the links from being used as long - as that directory does not exist. When this option is enabled, rsync will - refuse to run if that path is a directory or a symlink to a directory. + See the [SYMBOLIC LINKS](#SYMBOLIC-LINKS) section for multi-option info. - The option only affects the client side of the transfer, so if you need it - to affect the server, specify it via `--remote-option`. (Note that in a - local transfer, the client side is the sender.) +0. `--munge-links` - This option has no affect on a daemon, since the daemon configures whether - it wants munged symlinks via its "`munge symlinks`" parameter. See also the - "munge-symlinks" perl script in the support directory of the source code. + This option affects just one side of the transfer and tells rsync to munge + symlink values when it is receiving files or unmunge symlink values when it + is sending files. The munged values make the symlinks unusable on disk but + allows the original contents of the symlinks to be recovered. + + The server-side rsync often enables this option without the client's + knowledge, such as in an rsync daemon's configuration file or by an option + given to the rrsync (restricted rsync) script. When specified on the + client side, specify the option normally if it is the client side that + has/needs the munged symlinks, or use `-M--munge-links` to give the option + to the server when it has/needs the munged symlinks. Note that on a local + transfer, the client is the sender, so specifying the option directly + unmunges symlinks while specifying it as a remote option munges symlinks. + + This option has no affect when sent to a daemon via `--remote-option` + because the daemon configures whether it wants munged symlinks via its + "`munge symlinks`" parameter. + + The symlink value is munged/unmunged once it is in the transfer, so any + option that transforms symlinks into non-symlinks occurs prior to the + munging/unmunging **except** for `--safe-links`, which is a choice that the + receiver makes, so it bases its decision on the munged/unmunged value. + This does mean that if a receiver has munging enabled, that using + `--safe-links` will cause all symlinks to be ignored (since they are all + absolute). + + The method that rsync uses to munge the symlinks is to prefix each one's + value with the string "/rsyncd-munged/". This prevents the links from + being used as long as the directory does not exist. When this option is + enabled, rsync will refuse to run if that path is a directory or a symlink + to a directory (though it only checks at startup). See also the + "munge-symlinks" python script in the support directory of the source code + for a way to munge/unmunge one or more symlinks in-place. 0. `--copy-dirlinks`, `-k` @@ -1125,6 +1168,8 @@ your home directory (remove the '=' for that). directory in the file-list which overrides the symlink found during the scan of "src/./". + See the [SYMBOLIC LINKS](#SYMBOLIC-LINKS) section for multi-option info. + 0. `--keep-dirlinks`, `-K` This option causes the receiving side to treat a symlink to a directory as @@ -1140,8 +1185,9 @@ your home directory (remove the '=' for that). "bar". One note of caution: if you use `--keep-dirlinks`, you must trust all the - symlinks in the copy! If it is possible for an untrusted user to create - their own symlink to any directory, the user could then (on a subsequent + symlinks in the copy or enable the `--munge-symlinks` option on the + receiving side! If it is possible for an untrusted user to create their + own symlink to any real directory, the user could then (on a subsequent copy) replace the symlink with a real directory and affect the content of whatever directory the symlink references. For backup copies, you are better off using something like a bind mount instead of a symlink to modify @@ -1149,6 +1195,8 @@ your home directory (remove the '=' for that). See also `--copy-dirlinks` for an analogous option for the sending side. + See the [SYMBOLIC LINKS](#SYMBOLIC-LINKS) section for multi-option info. + 0. `--hard-links`, `-H` This tells rsync to look for hard-linked files in the source and link @@ -3085,7 +3133,7 @@ your home directory (remove the '=' for that). absolute) and (2) there are no mount points in the hierarchy (since the delayed updates will fail if they can't be renamed into place). - See also the "atomic-rsync" perl script in the "support" subdir for an + See also the "atomic-rsync" python script in the "support" subdir for an update algorithm that is even more atomic (it uses `--link-dest` and a parallel hierarchy of files). @@ -4071,8 +4119,9 @@ link in the source directory. By default, symbolic links are not transferred at all. A message "skipping non-regular" file is emitted for any symlinks that exist. -If `--links` is specified, then symlinks are recreated with the same target on -the destination. Note that `--archive` implies `--links`. +If `--links` is specified, then symlinks are added to the transfer (instead of +being noisily ignored), and the default handling is to recreate them with the +same target on the destination. Note that `--archive` implies `--links`. If `--copy-links` is specified, then symlinks are "collapsed" by copying their referent, rather than the symlink. @@ -4082,25 +4131,35 @@ where this might be used is a web site mirror that wishes to ensure that the rsync module that is copied does not include symbolic links to `/etc/passwd` in the public section of the site. Using `--copy-unsafe-links` will cause any links to be copied as the file they point to on the destination. Using -`--safe-links` will cause unsafe links to be omitted altogether. (Note that you -must specify `--links` for `--safe-links` to have any effect.) +`--safe-links` will cause unsafe links to be omitted by the receiver. (Note +that you must specify or imply `--links` for `--safe-links` to have any +effect.) -Symbolic links are considered unsafe if they are absolute symlinks -(start with `/`), empty, or if they contain enough ".." -components to ascend from the directory being copied. +Symbolic links are considered unsafe if they are absolute symlinks (start with +`/`), empty, or if they contain enough ".." components to ascend from the top +of the transfer. Here's a summary of how the symlink options are interpreted. The list is in order of precedence, so if your combination of options isn't mentioned, use the first line that is a complete subset of your options: -0. `--copy-links` Turn all symlinks into normal files (leaving no symlinks for - any other options to affect). +0. `--copy-links` Turn all symlinks into normal files and directories (leaving + no symlinks in the transfer for any other options to affect). +0. `--copy-dirlinks` Turn just symlinks to directories into real directories, + leaving all other symlinks to be handled as described below. 0. `--links --copy-unsafe-links` Turn all unsafe symlinks into files and - duplicate all safe symlinks. + create all safe symlinks. 0. `--copy-unsafe-links` Turn all unsafe symlinks into files, noisily skip all safe symlinks. -0. `--links --safe-links` Duplicate safe symlinks and skip unsafe ones. -0. `--links` Duplicate all symlinks. +0. `--links --safe-links` The receiver skips creating unsafe symlinks found in + the transfer and creates the safe ones. +0. `--links` Create all symlinks. + +For the effect of `--munge-links`, see the discussion in that option's section. + +Note that the `--keep-dirlinks` option does not effect symlinks in the transfer +but instead affects how rsync treats a symlink to a directory that already +exists on the receiving side. See that option's section for a warning. ## DIAGNOSTICS From 8c4ceb3b86749523573fec0df38a3b315575735c Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Wed, 12 Jan 2022 19:50:58 -0800 Subject: [PATCH 041/350] Avoid a -8 in the progress output's remaining time If the double "remain" value is so large that it overflows an int, make the estimated seconds output as :00 instead of :-8. Similar for the estimated remaining minutes. Support larger hours values. --- NEWS.md | 5 +++++ progress.c | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/NEWS.md b/NEWS.md index 8bab61cfb..4eebaa44c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -88,6 +88,11 @@ check to see if the allowed time is over, which should make rsync exit more consistently. + - Tweak the snprintf() in progress.c that turns the remaining time into a + HHHH:MM:SS value to avoid putting a -8 into the SS or MM spots when the + remaining seconds is so large that it overflows the integer arithmetic + trying to perform a modulus. + ### ENHANCEMENTS: - Use openssl's `-verify_hostname` option in the rsync-ssl script. diff --git a/progress.c b/progress.c index 8da528622..6e39ce995 100644 --- a/progress.c +++ b/progress.c @@ -118,10 +118,10 @@ static void rprint_progress(OFF_T ofs, OFF_T size, struct timeval *now, int is_l if (remain < 0) strlcpy(rembuf, " ??:??:??", sizeof rembuf); else { - snprintf(rembuf, sizeof rembuf, "%4d:%02d:%02d", - (int) (remain / 3600.0), - (int) (remain / 60.0) % 60, - (int) remain % 60); + snprintf(rembuf, sizeof rembuf, "%4lu:%02u:%02u", + (unsigned long) (remain / 3600.0), + (unsigned int) (remain / 60.0) % 60, + (unsigned int) remain % 60); } output_needs_newline = 0; From 6130c4fa3c92803d508f947e555fcd4e325f6178 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Thu, 13 Jan 2022 08:11:50 -0800 Subject: [PATCH 042/350] Display ??:??:?? when a time estimate gets too big. --- NEWS.md | 6 ++---- progress.c | 6 +++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/NEWS.md b/NEWS.md index 4eebaa44c..959e1da7b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -88,10 +88,8 @@ check to see if the allowed time is over, which should make rsync exit more consistently. - - Tweak the snprintf() in progress.c that turns the remaining time into a - HHHH:MM:SS value to avoid putting a -8 into the SS or MM spots when the - remaining seconds is so large that it overflows the integer arithmetic - trying to perform a modulus. + - Tweak --progress to display "??:??:??" when the time-remaining value is + so large as to be meaningless. ### ENHANCEMENTS: diff --git a/progress.c b/progress.c index 6e39ce995..6af96b98b 100644 --- a/progress.c +++ b/progress.c @@ -115,11 +115,11 @@ static void rprint_progress(OFF_T ofs, OFF_T size, struct timeval *now, int is_l units = "kB/s"; } - if (remain < 0) + if (remain < 0 || remain > 9999.0 * 3600.0) strlcpy(rembuf, " ??:??:??", sizeof rembuf); else { - snprintf(rembuf, sizeof rembuf, "%4lu:%02u:%02u", - (unsigned long) (remain / 3600.0), + snprintf(rembuf, sizeof rembuf, "%4u:%02u:%02u", + (unsigned int) (remain / 3600.0), (unsigned int) (remain / 60.0) % 60, (unsigned int) remain % 60); } From c1e8809a8f9d6dec97dcefd82d548599cbad81bc Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Thu, 13 Jan 2022 23:31:43 -0800 Subject: [PATCH 043/350] Tweak a caveat. --- rsync.1.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rsync.1.md b/rsync.1.md index 9b9b6996b..74615e56d 100644 --- a/rsync.1.md +++ b/rsync.1.md @@ -1080,9 +1080,9 @@ your home directory (remove the '=' for that). slash) to "/dest/subdir" that would not allow symlinks to any files outside of "subdir". - This option has no additional effect if `--copy-links` was also specified. - The safe symlinks are only copied if `--links` was also specified or - implied by `--archive` (`-a`). + Note that safe symlinks are only copied if `--links` was also specified or + implied. The `--copy-unsafe-links` option has no extra effect when combined + with `--copy-links`. See the [SYMBOLIC LINKS](#SYMBOLIC-LINKS) section for multi-option info. From f08505e92ba385e3af1be9138916269a58c057e7 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Thu, 13 Jan 2022 23:39:40 -0800 Subject: [PATCH 044/350] Add more link targets to html man pages. --- md-convert | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/md-convert b/md-convert index d1266f45b..3b66a0594 100755 --- a/md-convert +++ b/md-convert @@ -332,10 +332,18 @@ class TransformHtml(HTMLParser): st.at_first_tag_in_dd = False - def add_anchor(self, txt): + def add_target(self, txt): st = self.state - txt = txt.lower().replace(' ', '-') - st.html_out.append('') + txt = re.sub(r'[%s](.+?)[=%s].*' % (BOLD_FONT[0], NORM_FONT[0]), r'\1', txt.strip()) + txt = re.sub(r'[%s]' % NBR_DASH[0], '-', txt) + txt = re.sub(r'[\1-\7]+', '', txt) + txt = re.sub(r'[^-A-Za-z0-9._]', '_', txt) + if txt.startswith('-'): + txt = 'opt' + txt + else: + txt = re.sub(r'^([^A-Za-z])', r't\1', txt) + if txt: + st.html_out.append('') def handle_endtag(self, tag): @@ -350,14 +358,16 @@ class TransformHtml(HTMLParser): add_to_txt = None if tag == 'h1' or tag == 'h2': st.man_out.append(st.p_macro + '.SH "' + manify(txt) + '"\n') - self.add_anchor(txt) + self.add_target(txt) elif tag == 'h3': st.man_out.append(st.p_macro + '.SS "' + manify(txt) + '"\n') - self.add_anchor(txt) + self.add_target(txt) elif tag == 'p': if st.dt_from == 'p': tag = 'dt' st.man_out.append('.IP "' + manify(txt) + '"\n') + if txt.startswith(BOLD_FONT[0]): + self.add_target(txt) st.dt_from = None elif txt != '': st.man_out.append(manify(txt) + "\n") From 8898aecb21f3f97eb02c1d7b225d8117570c6833 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Fri, 14 Jan 2022 13:55:22 -0800 Subject: [PATCH 045/350] Make it easier to get section links. --- md-convert | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/md-convert b/md-convert index 3b66a0594..05952e34f 100755 --- a/md-convert +++ b/md-convert @@ -36,7 +36,8 @@ CONSUMES_TXT = set('h1 h2 h3 p li pre'.split()) HTML_START = """\ -%s +%TITLE% +