Improve various things in the checksum code

- Size flist checksum data to hold the active size, not the max.
- Add a negotiated hash method to the daemon auth code.
- Use EVP for all openssl digests. This makes it easy to add more
  openssl digest methods and avoids deprecation warnings.
- Support a way to re-enable deprecated digests via openssl conf
  file and allow a default file to be configured.
- Supply a simple openssl-rsync.cnf file to enable legacy digests.

Author: WayneD

NEWS.md

@@ -6,6 +6,28 @@
 
 - ...
 
+### ENHANCEMENTS:
+
+- Added negotiated daemon-auth support that allows a stronger checksum digest
+  to be used.
+
+### PACKAGING RELATED:
+
+- The checksum code now uses openssl's EVP methods, which gets rid of various
+  deprecation warnings and makes it easy to support more digest methods.  On
+  newer systems, the MD4 digest is marked as legacy in the openssl code, which
+  makes openssl refuse to support it via EVP.  You can just ignore this and
+  allow the included MD4 code to be used for older rsync connections (when
+  talking to an rsync prior to 3.0.0) or you can configure rsync to tell
+  openssl to enable legacy algorithms (see below).
+
+- A simple openssl config file is supplied that can be optionally installed for
+  rsync to use.  If you install packaging/openssl-rsync.cnf to a public spot
+  (such as ` /etc/ssl/openssl-rsync.cnf` or similar) and then configure rsync
+  using `--with-openssl-conf=/path/name.cnf`, this will cause rsync to export
+  the configured path in the OPENSSL_CONF environment variable (when it is not
+  already set).  This will enable openssl's MD4 code for rsync to use.
+
 ------------------------------------------------------------------------------
 
 # NEWS for rsync 3.2.6 (9 Sep 2022)

authenticate.c

@@ -24,6 +24,7 @@
 
 extern int read_only;
 extern char *password_file;
+extern struct name_num_obj valid_auth_checksums;
 
 /***************************************************************************
 encode a buffer using base64 - simple and slow algorithm. null terminates
@@ -72,9 +73,9 @@ static void gen_challenge(const char *addr, char *challenge)
 	SIVAL(input, 20, tv.tv_usec);
 	SIVAL(input, 24, getpid());
 
-	sum_init(-1, 0);
+	len = sum_init(valid_auth_checksums.negotiated_nni, 0);
 	sum_update(input, sizeof input);
-	len = sum_end(digest);
+	sum_end(digest);
 
 	base64_encode(digest, len, challenge, 0);
 }
@@ -86,10 +87,10 @@ static void generate_hash(const char *in, const char *challenge, char *out)
 	char buf[MAX_DIGEST_LEN];
 	int len;
 
-	sum_init(-1, 0);
+	len = sum_init(valid_auth_checksums.negotiated_nni, 0);
 	sum_update(in, strlen(in));
 	sum_update(challenge, strlen(challenge));
-	len = sum_end(buf);
+	sum_end(buf);
 
 	base64_encode(buf, len, out, 0);
 }
@@ -238,6 +239,7 @@ char *auth_server(int f_in, int f_out, int module, const char *host,
 	if (!users || !*users)
 		return "";
 
+	negotiate_daemon_auth(f_out, 0);
 	gen_challenge(addr, challenge);
 
 	io_printf(f_out, "%s%s\n", leader, challenge);
@@ -350,6 +352,7 @@ void auth_client(int fd, const char *user, const char *challenge)
 
 	if (!user || !*user)
 		user = "nobody";
+	negotiate_daemon_auth(-1, 1);
 
 	if (!(pass = getpassf(password_file))
 	 && !(pass = getenv("RSYNC_PASSWORD"))) {