Improve the per-branch build dir support

The release script & the patch management script now require the use of
an auto-build-save dir that makes it much easier to keep the generated
files from melding together, and remembers the configure setup for each
patch branch.

Author: WayneD

.gitignore

@@ -48,6 +48,7 @@ aclocal.m4
 /testsuite/devices-fake.test
 /testsuite/xattrs-hlink.test
 /patches
-/SaVeDiR
+/patches.gen
 /build
+/auto-build-save
 .deps

NEWS.md

@@ -53,6 +53,11 @@ Protocol: 31 (unchanged)
    checksum routines.  Just make sure that the new rsync package depends on
    xxhash >= 0.8.0.
 
+### DEVELOPER RELATED:
+
+ - Moved the version number out of configure.ac into its own version.h file so
+   that we don't need to reconfigure just because the version number changes.
+
 ------------------------------------------------------------------------------
 <a name="3.2.1"></a>
 

configure

@@ -4,14 +4,16 @@
 # then transfer control to the configure.sh script to do the real work.
 
 dir=`dirname $0`
+if test x"$dir" = x; then
+    dir=.
+fi
 
-# Support automatic switching to build/$BRANCH subdirs.  It's also good
-# to put packaging/make somewhere early in your $PATH if you use this!
-if test "$dir" = '.' -a ! -f Makefile -a -d build/master -a -d .git; then
-    builddir=build/`git rev-parse --abbrev-ref HEAD | tr / %`
-    test -d "$builddir" || mkdir "$builddir"
-    cd "$builddir" || exit 1
-    dir=../..
+if test "$dir" = '.'; then
+    branch=`packaging/prep-auto-dir` || exit 1
+    if test x"$branch" != x; then
+	cd build || exit 1
+	dir=..
+    fi
 fi
 
 if test ! -f configure.sh; then

packaging/auto-Makefile

@@ -0,0 +1,8 @@
+TARGETS := all install install-ssl-daemon install-all install-strip conf gen gensend reconfigure restatus \
+	proto man clean cleantests distclean test check check29 check30 installcheck splint doxygen doxygen-upload
+
+.PHONY: $(TARGETS)
+
+$(TARGETS):
+	@if test x`packaging/prep-auto-dir` = x; then echo "auto-build-save is not setup"; exit 1; fi
+	make -C build $@

packaging/make

@@ -14,7 +14,8 @@ from pkglib import *
 
 MAKE_GEN_CMDS = [
         './prepare-source'.split(),
-        'make restatus gen'.split(),
+        'cd build && if test -f config.status ; then ./config.status ; else ../configure ; fi',
+        'make -C build gen'.split(),
         ]
 TMP_DIR = "patches.gen"
 
@@ -32,6 +33,9 @@ def main():
 
     master_commit = latest_git_hash(args.base_branch)
 
+    if cmd_txt_chk(['packaging/prep-auto-dir']) == '':
+        die('You must setup an auto-build-save dir to use this script.')
+
     if args.gen:
         if os.path.lexists(TMP_DIR):
             die(f'"{TMP_DIR}" must not exist in the current directory.')
@@ -41,7 +45,7 @@ def main():
             cmd_chk(cmd)
         cmd_chk(['rsync', '-a', *gen_files, f'{TMP_DIR}/master/'])
 
-    last_touch = time.time()
+    last_touch = int(time.time())
 
     # Start by finding all patches so that we can load all possible parents.
     patches = sorted(list(get_patch_branches(args.base_branch)))
@@ -90,9 +94,10 @@ def main():
     if args.gen:
         shutil.rmtree(TMP_DIR)
 
-    while last_touch >= time.time():
+    while last_touch >= int(time.time()):
         time.sleep(1)
     cmd_chk(['git', 'checkout', starting_branch])
+    cmd_chk(['packaging/prep-auto-dir'], discard='output')
 
 
 def update_patch(patch):
@@ -112,7 +117,7 @@ def update_patch(patch):
 
     print(f"======== {patch} ========")
 
-    while args.gen and last_touch >= time.time():
+    while args.gen and last_touch >= int(time.time()):
         time.sleep(1)
 
     branch = f"patch/{args.base_branch}/{patch}"
@@ -123,6 +128,7 @@ def update_patch(patch):
     s = cmd_run(['git', 'merge', based_on])
     ok = s.returncode == 0
     if not ok or args.shell:
+        cmd_chk(['packaging/prep-auto-dir'], discard='output')
         m = re.search(r'([^/]+)$', parent)
         parent_dir = m[1]
         if not ok:
@@ -139,6 +145,7 @@ def update_patch(patch):
             if is_clean:
                 break
             print(status_txt, end='')
+        cmd_run('rm -f build/*.o build/*/*.o')
 
     with open(f"{args.patches_dir}/{patch}.diff", 'w', encoding='utf-8') as fh:
         fh.write(description[patch])
@@ -151,7 +158,7 @@ def update_patch(patch):
             cmd_chk(['rsync', '-a', *gen_files, f"{TMP_DIR}/{patch}/"])
         else:
             gen_files = [ ]
-        last_touch = time.time()
+        last_touch = int(time.time())
 
         proc = cmd_pipe(['git', 'diff', based_on])
         skipping = False

packaging/patch-update

@@ -186,7 +186,7 @@ def get_gen_files(want_dir_plus_list=False):
 
     gen_files = [ ]
 
-    builddir = os.path.join('build', cmd_txt('git rev-parse --abbrev-ref HEAD').replace('/', '%'))
+    auto_dir = os.path.join('auto-build-save', cmd_txt('git rev-parse --abbrev-ref HEAD').strip().replace('/', '%'))
 
     with open('Makefile.in', 'r', encoding='utf-8') as fh:
         for line in fh:
@@ -202,18 +202,18 @@ def get_gen_files(want_dir_plus_list=False):
                 break
 
     if want_dir_plus_list:
-        return (builddir, gen_files)
+        return (auto_dir, gen_files)
 
-    return [ os.path.join(builddir, fn) for fn in gen_files ]
+    return [ os.path.join(auto_dir, fn) for fn in gen_files ]
 
 
-def get_configure_version():
-    with open('configure.ac', 'r', encoding='utf-8') as fh:
-        for line in fh:
-            m = re.match(r'^AC_INIT\(\[rsync\],\s*\[(\d.+?)\]', line)
-            if m:
-                return m[1]
-    die("Unable to find AC_INIT with version in configure.ac")
+def get_rsync_version():
+    with open('version.h', 'r', encoding='utf-8') as fh:
+        txt = fh.read()
+    m = re.match(r'^#define\s+RSYNC_VERSION\s+"(\d.+?)"', txt)
+    if m:
+        return m[1]
+    die("Unable to find RSYNC_VERSION define in version.h")
 
 
 def get_NEWS_version_info():

packaging/pkglib.py

@@ -0,0 +1,39 @@
+#!/bin/sh -e
+
+# This script will setup the build dir based on the current git branch and the
+# directory auto-build-save/$BRANCH.  We don't use a symlink for the build dir
+# because we want to maximize the ccache reuse, so all builds must happen in
+# the same real dir.  When a dir is moved out of auto-build-save/$BRANCH to the
+# build dir, it is replaced with a symlink so that it can still be found under
+# that dir.  The build dir also gets a .branch -> $BRANCH symlink so that we
+# can figure out the current build dir's branch.
+
+# To get started, just clone the rsync git repo and create the auto-build-save
+# dir.  If you have an existing git checkout and it is not in a pristine state,
+# run "make distclean" before creating the auto-build-save dir.
+
+auto_top='auto-build-save'
+if test -d $auto_top -a -d .git; then
+    desired_branch=`git rev-parse --abbrev-ref HEAD | tr / %`
+    auto_dir="$auto_top/$desired_branch"
+    if test -d build; then
+	cur_branch=`readlink build/.branch`
+    else
+	cur_branch='/'
+    fi
+    if test "$desired_branch" != "$cur_branch"; then
+	if test "$cur_branch" != /; then
+	    rm -f "$auto_top/$cur_branch"
+	    mv build "$auto_top/$cur_branch"
+	fi
+	test -d "$auto_dir" || mkdir "$auto_dir"
+	test -h "$auto_dir/.branch" || ln -s "$desired_branch" "$auto_dir/.branch"
+	mv "$auto_dir" build
+	ln -s ../build "$auto_dir"
+    fi
+    if test ! -h Makefile; then
+	rm -f Makefile
+	ln -s packaging/auto-Makefile Makefile
+    fi
+    echo $desired_branch
+fi

packaging/prep-auto-dir

@@ -30,8 +30,11 @@ def main():
 
     signal.signal(signal.SIGINT, signal_handler)
 
-    builddir, gen_files = get_gen_files(True)
-    gen_pathnames = [ os.path.join(builddir, fn) for fn in gen_files ]
+    if cmd_txt_chk(['packaging/prep-auto-dir']) == '':
+        die('You must setup an auto-build-save dir to use this script.');
+
+    auto_dir, gen_files = get_gen_files(True)
+    gen_pathnames = [ os.path.join(auto_dir, fn) for fn in gen_files ]
 
     dash_line = '=' * 74
 
@@ -54,13 +57,13 @@ def main():
 
     check_git_state(args.master_branch, True, 'patches')
 
-    confversion = get_configure_version()
+    curversion = get_rsync_version()
 
     # All version values are strings!
     lastversion, last_protocol_version = get_NEWS_version_info()
     protocol_version, subprotocol_version = get_protocol_versions()
 
-    version = confversion
+    version = curversion
     m = re.search(r'pre(\d+)', version)
     if m:
         version = re.sub(r'pre\d+', 'pre' + str(int(m[1]) + 1), version)
@@ -90,8 +93,8 @@ def main():
         cmd_chk(['git', 'tag', '-d', v_ver])
 
     version = re.sub(r'[-.]*pre[-.]*', 'pre', version)
-    if 'pre' in version and not confversion.endswith('dev'):
-        lastversion = confversion
+    if 'pre' in version and not curversion.endswith('dev'):
+        lastversion = curversion
 
     ans = input(f"Enter the previous version to produce a patch against: [{lastversion}] ")
     if ans != '':
@@ -177,7 +180,7 @@ About to:
         with open(fn, 'r', encoding='utf-8') as fh:
             old_txt = txt = fh.read()
         if fn == 'version.h':
-            txt = f"#define RSYNC_VERSION {version}\n"
+            txt = f'#define RSYNC_VERSION "{version}"\n'
         elif '.spec' in fn:
             for var, val in specvars.items():
                 x_re = re.compile(r'^%s .*' % re.escape(var), re.M)
@@ -238,7 +241,6 @@ About to:
         die('Aborting')
 
     cmd_chk('make gen')
-    cmd_chk(['rsync', '-a', *gen_pathnames, 'SaVeDiR/'])
 
     print(f'Creating any missing patch branches.')
     s = cmd_run(f'packaging/branch-from-patch --branch={args.master_branch} --add-missing')
@@ -254,9 +256,6 @@ About to:
         print(f'\nVisiting all "patch/{args.master_branch}/*" branches ...')
         cmd_run(f"packaging/patch-update --branch={args.master_branch} --skip-check --shell")
 
-    cmd_run("rm -f *.[o15] *.html")
-    cmd_chk('rsync -a SaVeDiR/ .'.split())
-
     if os.path.isdir('patches/.git'):
         s = cmd_run(f"cd patches && git commit -a -m 'The patches for {version}.'")
         if s.returncode:
@@ -318,11 +317,6 @@ About to:
     os.mkdir(f"{rsync_ver}/patches", 0o755)
     cmd_chk(f"packaging/patch-update --skip-check --branch={args.master_branch} --gen={rsync_ver}/patches".split())
 
-    cmd_run("rm -f *.[o15] *.html")
-    cmd_chk('rsync -a SaVeDiR/ .'.split())
-    shutil.rmtree('SaVeDiR')
-    cmd_chk('make gen'.split())
-
     print(f"Creating {pattar_file} ...")
     cmd_chk(['fakeroot', 'tar', 'chzf', pattar_file, rsync_ver + '/patches'])
     shutil.rmtree(rsync_ver)

packaging/release-rsync

@@ -16,13 +16,12 @@ if test x"$dir" = x; then
     dir=.
 fi
 
-# Support automatic switching to build/$BRANCH subdirs.  It's also good
-# to put packaging/make somewhere early in your $PATH if you use this!
-if test "$dir" = . -a ! -f Makefile -a -d build/master -a -d .git; then
-    builddir=build/`git rev-parse --abbrev-ref HEAD | tr / %`
-    test -d "$builddir" || mkdir "$builddir"
-    cd "$builddir" || exit 1
-    dir=../..
+if test "$dir" = '.'; then
+    branch=`packaging/prep-auto-dir` || exit 1
+    if test x"$branch" != x; then
+	cd build || exit 1
+	dir=..
+    fi
 fi
 
 if test "$dir" != '.'; then