Skip to content

Commit 2f6f051

Browse files
al45tairal45tair
committed
Use rsync for remote-run.
Not only does this give us a huge speed-up, it also works around a problem where sftp doesn't always update the modification time, which causes random test failures. rdar://88179140
1 parent 97eb38e commit 2f6f051

12 files changed

+238
-172
lines changed

‎test/remote-run/custom-options.test-sh

+10-16
Original file line numberDiff line numberDiff line change
@@ -3,30 +3,24 @@ RUN: %remote-run -n --remote-dir /xyz-REMOTE -o FIRST_OPT -o SECOND_OPT --output
33
CHECK: /usr/bin/ssh -n
44
CHECK-DAG: -p 12345
55
CHECK-DAG: -o FIRST_OPT -o SECOND_OPT
6-
CHECK-SAME: some_user@some_host -- '/usr/bin/env' '/bin/rm' '-rf' '{{.+}}-REMOTE/output'
6+
CHECK-SAME: some_user@some_host -- '/usr/bin/env'
7+
CHECK-SAME: '/bin/rm' '-rf' '{{.+}}-REMOTE/output'
78

89
CHECK-NEXT: /usr/bin/ssh -n
910
CHECK-DAG: -p 12345
1011
CHECK-DAG: -o FIRST_OPT -o SECOND_OPT
11-
CHECK-SAME: some_user@some_host -- '/usr/bin/env' '/bin/mkdir' '-p' '{{.+}}-REMOTE/output/nested'
12-
13-
CHECK-NEXT: /usr/bin/sftp
14-
CHECK-DAG: -P 12345
15-
CHECK-DAG: -o FIRST_OPT -o SECOND_OPT
16-
CHECK-SAME: some_user@some_host
17-
CHECK-DAG: -put '{{.+}}/nested/output' '/xyz-REMOTE/output/nested/output'
18-
CHECK-DAG: -put '{{.+}}/nested/input' '/xyz-REMOTE/output/nested/input'
12+
CHECK-SAME: some_user@some_host -- '/usr/bin/env'
13+
CHECK-SAME: '/bin/mkdir' '-p' '{{.+}}-REMOTE/output/nested'
1914

2015
CHECK: /usr/bin/ssh -n
2116
CHECK-DAG: -p 12345
2217
CHECK-DAG: -o FIRST_OPT -o SECOND_OPT
23-
CHECK-SAME: some_user@some_host -- '/usr/bin/env' 'cp'
18+
CHECK-SAME: some_user@some_host -- '/usr/bin/env'
19+
CHECK-SAME: 'cp'
2420

25-
CHECK-NEXT: {{^}}/bin/mkdir -p {{.+}}/nested
21+
CHECK-NEXT: {{^}}/bin/mkdir -p {{.+}}
2622

27-
CHECK-NEXT: /usr/bin/sftp
28-
CHECK-DAG: -P 12345
29-
CHECK-DAG: -o FIRST_OPT -o SECOND_OPT
23+
CHECK-NEXT: /usr/bin/rsync
24+
CHECK-DAG: '-p' '12345'
25+
CHECK-DAG: '-o' 'FIRST_OPT' '-o' 'SECOND_OPT'
3026
CHECK-SAME: some_user@some_host
31-
CHECK-DAG: -get '/xyz-REMOTE/output/nested/output' '{{.+}}/nested/output'
32-
CHECK-DAG: -get '/xyz-REMOTE/output/nested/input' '{{.+}}/nested/input'

‎test/remote-run/dry-run-remote.test-sh

+11-14
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,22 @@
11
RUN: %remote-run -n --remote-dir /xyz-REMOTE --input-prefix %S/Inputs/ some_user@some_host ls %S/Inputs/upload/1.txt %S/Inputs/upload/2.txt 2>&1 >/dev/null | %FileCheck -check-prefix CHECK-INPUT %s
22

3-
CHECK-INPUT: /usr/bin/ssh -n some_user@some_host -- '/usr/bin/env' '/bin/mkdir' '-p' '{{.+}}-REMOTE/input/upload'
4-
CHECK-INPUT-NEXT: /usr/bin/sftp
3+
CHECK-INPUT: /usr/bin/ssh -n some_user@some_host -- '/usr/bin/env' '/bin/mkdir' '-p' '{{.+}}-REMOTE/input'
4+
CHECK-INPUT-NEXT: /usr/bin/rsync
55
CHECK-INPUT-SAME: some_user@some_host
6-
CHECK-INPUT-DAG: -put '{{.+}}/Inputs/upload/1.txt' '/xyz-REMOTE/input/upload/1.txt'
7-
CHECK-INPUT-DAG: -put '{{.+}}/Inputs/upload/2.txt' '/xyz-REMOTE/input/upload/2.txt'
8-
CHECK-INPUT: /usr/bin/ssh -n some_user@some_host -- '/usr/bin/env' 'ls'
6+
7+
CHECK-INPUT: /usr/bin/ssh -n some_user@some_host -- '/usr/bin/env' {{.*}}'ls'
98

109
RUN: %empty-directory(%t)
1110
RUN: %empty-directory(%t/nested)
1211
RUN: touch %t/nested/input %t/nested/BAD
1312
RUN: %remote-run -n --remote-dir /xyz-REMOTE --output-prefix %/t some_user@some_host cp %/t/nested/input %/t/nested/output 2>&1 >/dev/null | %FileCheck -check-prefix CHECK-OUTPUT %s
1413

15-
CHECK-OUTPUT: /usr/bin/ssh -n some_user@some_host -- '/usr/bin/env' '/bin/mkdir' '-p' '{{.+}}-REMOTE/output/nested'
16-
CHECK-OUTPUT-NEXT: /usr/bin/sftp
14+
CHECK-OUTPUT: /usr/bin/ssh -n some_user@some_host -- '/usr/bin/env' '/bin/mkdir' '-p' '{{.+}}-REMOTE/output'
15+
CHECK-OUTPUT-NEXT: /usr/bin/rsync
1716
CHECK-OUTPUT-SAME: some_user@some_host
18-
CHECK-OUTPUT-DAG: -put '{{.+}}/nested/output' '/xyz-REMOTE/output/nested/output'
19-
CHECK-OUTPUT-DAG: -put '{{.+}}/nested/input' '/xyz-REMOTE/output/nested/input'
20-
CHECK-OUTPUT: /usr/bin/ssh -n some_user@some_host -- '/usr/bin/env' 'cp'
21-
CHECK-OUTPUT-NEXT: {{^}}/bin/mkdir -p {{.+}}/nested
22-
CHECK-OUTPUT-NEXT: /usr/bin/sftp
17+
18+
CHECK-OUTPUT: /usr/bin/ssh -n some_user@some_host -- '/usr/bin/env' {{.*}}'cp'
19+
CHECK-OUTPUT-NEXT: {{^}}/bin/mkdir -p {{.+}}
20+
CHECK-OUTPUT-NEXT: /usr/bin/rsync
2321
CHECK-OUTPUT-SAME: some_user@some_host
24-
CHECK-OUTPUT-DAG: -get '/xyz-REMOTE/output/nested/output' '{{.+}}/nested/output'
25-
CHECK-OUTPUT-DAG: -get '/xyz-REMOTE/output/nested/input' '{{.+}}/nested/input'
22+

‎test/remote-run/dry-run.test-sh

+8-13
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,9 @@ RUN: %debug-remote-run -n --input-prefix %S/Inputs/ ls %S/Inputs/upload/1.txt %S
55
RUN: test -z "`ls %t`"
66
RUN: test ! -e %t-REMOTE
77

8-
CHECK-INPUT: /usr/bin/env /bin/mkdir -p {{.+}}-REMOTE/input/upload
9-
CHECK-INPUT-NEXT: /usr/bin/sftp
10-
CHECK-INPUT-DAG: -put '{{.+}}/Inputs/upload/1.txt' '{{.+}}-REMOTE/input/upload/1.txt'
11-
CHECK-INPUT-DAG: -put '{{.+}}/Inputs/upload/2.txt' '{{.+}}-REMOTE/input/upload/2.txt'
12-
CHECK-INPUT: /usr/bin/env ls
8+
CHECK-INPUT: /usr/bin/env /bin/mkdir -p {{.+}}-REMOTE/input
9+
CHECK-INPUT-NEXT: /usr/bin/rsync
10+
CHECK-INPUT: /usr/bin/env {{.*}}ls
1311

1412
RUN: %empty-directory(%t)
1513
RUN: %empty-directory(%t/nested)
@@ -18,11 +16,8 @@ RUN: %debug-remote-run -n --output-prefix %t cp %t/nested/input %t/nested/output
1816
RUN: test ! -e %t-REMOTE
1917

2018
CHECK-OUTPUT: /usr/bin/env /bin/mkdir -p {{.+}}-REMOTE/output/nested
21-
CHECK-OUTPUT-NEXT: /usr/bin/sftp
22-
CHECK-OUTPUT-DAG: -put '{{.+}}/nested/output' '{{.+}}-REMOTE/output/nested/output'
23-
CHECK-OUTPUT-DAG: -put '{{.+}}/nested/input' '{{.+}}-REMOTE/output/nested/input'
24-
CHECK-OUTPUT: /usr/bin/env cp
25-
CHECK-OUTPUT-NEXT: {{^}}/bin/mkdir -p {{.+}}/nested
26-
CHECK-OUTPUT-NEXT: /usr/bin/sftp
27-
CHECK-OUTPUT-DAG: -get '{{.+}}-REMOTE/output/nested/output' '{{.+}}/nested/output'
28-
CHECK-OUTPUT-DAG: -get '{{.+}}-REMOTE/output/nested/input' '{{.+}}/nested/input'
19+
CHECK-OUTPUT: /usr/bin/rsync
20+
CHECK-OUTPUT: /usr/bin/env {{.*}}cp
21+
CHECK-OUTPUT-NEXT: {{^}}/bin/mkdir -p {{.+}}
22+
CHECK-OUTPUT-NEXT: /usr/bin/rsync
23+

‎test/remote-run/env.test-sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,4 @@ RUN: env REMOTE_RUN_CHILD_FOO=foo REMOTE_RUN_CHILD_BAR=bar %debug-remote-run -v
55

66
CHECK: {{^:foo: :bar:$}}
77

8-
VERBOSE: /usr/bin/env BAR=bar FOO=foo sh -c echo ":${FOO}:" ":${BAR}:"
8+
VERBOSE: /usr/bin/env BAR=bar FOO=foo {{.*}}sh -c echo ":${FOO}:" ":${BAR}:"

‎test/remote-run/identity.test-sh

+13-18
Original file line numberDiff line numberDiff line change
@@ -8,36 +8,31 @@ CHECK-SAME: some_user@some_host -- '/usr/bin/env' '/bin/rm' '-rf' '{{.+}}-REMOTE
88
CHECK-NEXT: /usr/bin/ssh -n
99
CHECK-DAG: -p 12345
1010
CHECK-DAG: -i spiderman
11-
CHECK-SAME: some_user@some_host -- '/usr/bin/env' '/bin/mkdir' '-p' '{{.+}}-REMOTE/output/nested'
12-
13-
CHECK-NEXT: /usr/bin/sftp
14-
CHECK-DAG: -P 12345
15-
CHECK-DAG: -i spiderman
16-
CHECK-SAME: some_user@some_host
17-
CHECK-DAG: -put '{{.+}}/nested/output' '/xyz-REMOTE/output/nested/output'
18-
CHECK-DAG: -put '{{.+}}/nested/input' '/xyz-REMOTE/output/nested/input'
11+
CHECK-SAME: some_user@some_host -- '/usr/bin/env' '/bin/mkdir' '-p' '{{.+}}'
1912

2013
CHECK: /usr/bin/ssh -n
2114
CHECK-DAG: -p 12345
2215
CHECK-DAG: -i spiderman
23-
CHECK-SAME: some_user@some_host -- '/usr/bin/env' 'cp'
16+
CHECK-SAME: some_user@some_host -- '/usr/bin/env' {{.*}}'cp'
2417

25-
CHECK-NEXT: {{^}}/bin/mkdir -p {{.+}}/nested
18+
CHECK-NEXT: {{^}}/bin/mkdir -p {{.+}}
2619

27-
CHECK-NEXT: /usr/bin/sftp
28-
CHECK-DAG: -P 12345
29-
CHECK-DAG: -i spiderman
20+
CHECK-NEXT: /usr/bin/rsync
21+
CHECK-DAG: '-p' '12345'
22+
CHECK-DAG: '-i' 'spiderman'
3023
CHECK-SAME: some_user@some_host
31-
CHECK-DAG: -get '/xyz-REMOTE/output/nested/output' '{{.+}}/nested/output'
32-
CHECK-DAG: -get '/xyz-REMOTE/output/nested/input' '{{.+}}/nested/input'
3324

3425
# Make sure things work without a port.
3526
RUN: %remote-run -n --remote-dir /xyz-REMOTE -i spiderman --output-prefix %/t some_user@some_host cp %/t/nested/input %/t/nested/output 2>&1 >/dev/null | %FileCheck -check-prefix CHECK-PORTLESS %s
3627

37-
CHECK-PORTLESS: /usr/bin/sftp
28+
CHECK-PORTLESS: /usr/bin/ssh -n
29+
CHECK-PORTLESS-SAME: -i spiderman
30+
CHECK-PORTLESS-SAME: some_user@some_host -- '/usr/bin/env' '/bin/rm' '-rf' '{{.+}}'
31+
32+
CHECK-PORTLESS: /usr/bin/ssh -n
3833
CHECK-PORTLESS-SAME: -i spiderman
39-
CHECK-PORTLESS-SAME: some_user@some_host
34+
CHECK-PORTLESS-SAME: some_user@some_host -- '/usr/bin/env' '/bin/mkdir' '-p' '{{.+}}'
4035

4136
CHECK-PORTLESS: /usr/bin/ssh -n
4237
CHECK-PORTLESS-SAME: -i spiderman
43-
CHECK-PORTLESS-SAME: some_user@some_host -- '/usr/bin/env' 'cp'
38+
CHECK-PORTLESS-SAME: some_user@some_host -- '/usr/bin/env' {{.*}}'cp'

‎test/remote-run/lit.local.cfg

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,6 @@
22
config.substitutions = list(config.substitutions)
33

44
config.substitutions.insert(0, ('%debug-remote-run',
5-
r'%r %%utils/remote-run --debug-as-local %%sftp-server --remote-dir %%t-REMOTE' % (sys.executable,)))
5+
r'%r %%utils/remote-run --debug-as-local --remote-dir %%t-REMOTE' % (sys.executable,)))
66
config.substitutions.insert(0, ('%remote-run',
77
r'%r %%utils/remote-run' % (sys.executable,)))

‎test/remote-run/port.test-sh

+4-11
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,8 @@
11
RUN: %remote-run -n --remote-dir /xyz-REMOTE --output-prefix %/t some_user@some_host:12345 cp %/t/nested/input %/t/nested/output 2>&1 >/dev/null | %FileCheck %s
22

33
CHECK: /usr/bin/ssh -n -p 12345 some_user@some_host -- '/usr/bin/env' '/bin/mkdir' '-p' '{{.+}}-REMOTE/output/nested'
4-
CHECK-NEXT: /usr/bin/sftp
5-
CHECK-SAME: -P 12345
4+
CHECK: /usr/bin/ssh -n -p 12345 some_user@some_host -- '/usr/bin/env' {{.*}}'cp'
5+
CHECK-NEXT: {{^}}/bin/mkdir -p {{.+}}
6+
CHECK-NEXT: /usr/bin/rsync
7+
CHECK-SAME: '-p' '12345'
68
CHECK-SAME: some_user@some_host
7-
CHECK-DAG: -put '{{.+}}/nested/output' '/xyz-REMOTE/output/nested/output'
8-
CHECK-DAG: -put '{{.+}}/nested/input' '/xyz-REMOTE/output/nested/input'
9-
CHECK: /usr/bin/ssh -n -p 12345 some_user@some_host -- '/usr/bin/env' 'cp'
10-
CHECK-NEXT: {{^}}/bin/mkdir -p {{.+}}/nested
11-
CHECK-NEXT: /usr/bin/sftp
12-
CHECK-SAME: -P 12345
13-
CHECK-SAME: some_user@some_host
14-
CHECK-DAG: -get '/xyz-REMOTE/output/nested/output' '{{.+}}/nested/output'
15-
CHECK-DAG: -get '/xyz-REMOTE/output/nested/input' '{{.+}}/nested/input'

‎test/remote-run/run-only.test-sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,4 @@ RUN: %debug-remote-run -v echo hello 2>&1 >/dev/null | %FileCheck -check-prefix
55

66
CHECK: {{^hello$}}
77

8-
VERBOSE: /usr/bin/env echo hello
8+
VERBOSE: /usr/bin/env {{.*}}echo hello

‎test/remote-run/upload-and-download.test-sh

+7-11
Original file line numberDiff line numberDiff line change
@@ -27,26 +27,22 @@ RUN: ls %t-REMOTE/output/nested/ | %FileCheck -check-prefix CHECK-REMOTE %s
2727
RUN: %debug-remote-run -v --output-prefix %t cp %t/nested/input %t/nested/output 2>&1 >/dev/null | %FileCheck -check-prefix VERBOSE %s
2828

2929
VERBOSE: /usr/bin/env /bin/mkdir -p {{.+}}-REMOTE/output/nested
30-
VERBOSE-NEXT: /usr/bin/sftp
31-
VERBOSE-DAG: -put '{{.+}}/nested/output' '{{.+}}-REMOTE/output/nested/output'
32-
VERBOSE-DAG: -put '{{.+}}/nested/input' '{{.+}}-REMOTE/output/nested/input'
33-
VERBOSE: /usr/bin/env cp
34-
VERBOSE-NEXT: {{^}}/bin/mkdir -p {{.+}}/nested
35-
VERBOSE-NEXT: /usr/bin/sftp
36-
VERBOSE-DAG: -get '{{.+}}-REMOTE/output/nested/output' '{{.+}}/nested/output'
37-
VERBOSE-DAG: -get '{{.+}}-REMOTE/output/nested/input' '{{.+}}/nested/input'
30+
VERBOSE: /usr/bin/rsync
31+
VERBOSE: /usr/bin/env {{.*}}cp
32+
VERBOSE-NEXT: {{^}}/bin/mkdir -p {{.+}}
33+
VERBOSE-NEXT: /usr/bin/rsync
3834

3935
RUN: %empty-directory(%t)
4036
RUN: touch %t/xyz-1before
4137
RUN: %debug-remote-run --output-prefix %t/xyz cp %t/xyz-1before %t/xyz-2after
4238
RUN: ls %t | %FileCheck -check-prefix CHECK-PREFIXED %s
43-
RUN: ls %t-REMOTE/ | %FileCheck -check-prefix CHECK-PREFIXED-REMOTE %s
39+
RUN: ls %t-REMOTE/output | %FileCheck -check-prefix CHECK-PREFIXED-REMOTE %s
4440

4541
CHECK-PREFIXED: {{^xyz-1before$}}
4642
CHECK-PREFIXED: {{^xyz-2after$}}
4743

48-
CHECK-PREFIXED-REMOTE: {{^output-1before$}}
49-
CHECK-PREFIXED-REMOTE: {{^output-2after$}}
44+
CHECK-PREFIXED-REMOTE: {{^xyz-1before$}}
45+
CHECK-PREFIXED-REMOTE: {{^xyz-2after$}}
5046

5147
RUN: %empty-directory(%t)
5248
RUN: %empty-directory(%t-REMOTE)

‎test/remote-run/upload-stderr.test-sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,6 @@ REQUIRES: sftp_server
33
RUN: %empty-directory(%t)
44
RUN: %empty-directory(%t/REMOTE/input)
55
RUN: chmod a-w %t/REMOTE/input
6-
RUN: %debug-remote-run --remote-dir %t/REMOTE --input-prefix %S/Inputs/upload/ true -- %S/Inputs/upload/1.txt 2>&1 | %FileCheck %s
6+
RUN: %debug-remote-run --ignore-rsync-failure --remote-dir %t/REMOTE --input-prefix %S/Inputs/upload/ true -- %S/Inputs/upload/1.txt 2>&1 | %FileCheck %s
77

88
CHECK: Permission denied

‎test/remote-run/upload.test-sh

+5-7
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ RUN: %debug-remote-run -v --input-prefix %S/Inputs/ ls %S/Inputs/upload/1.txt %S
1010

1111
RUN: %empty-directory(%t-REMOTE)
1212
RUN: %debug-remote-run --input-prefix %S/Inputs/upload/1 ls %S/Inputs/upload/1.txt | %FileCheck -check-prefix CHECK-REMOTE-SINGLE-FILE %s
13-
RUN: test -f %t-REMOTE/input.txt
13+
RUN: test -f %t-REMOTE/input/1.txt
1414

1515
RUN: %empty-directory(%t-REMOTE)
1616
RUN: %debug-remote-run --input-prefix %S/Inputs/upload/ --remote-input-prefix custom-input ls %S/Inputs/upload/1.txt %S/Inputs/upload/2.txt | %FileCheck -check-prefix CHECK-REMOTE-CUSTOM %s
@@ -22,7 +22,7 @@ CHECK-REMOTE-NEXT: {{-REMOTE/input/2.txt$}}
2222
CHECK-REMOTE-NESTED: {{-REMOTE/input/upload/1.txt$}}
2323
CHECK-REMOTE-NESTED-NEXT: {{-REMOTE/input/upload/2.txt$}}
2424

25-
CHECK-REMOTE-SINGLE-FILE: {{-REMOTE/input.txt$}}
25+
CHECK-REMOTE-SINGLE-FILE: {{-REMOTE/input/1.txt$}}
2626

2727
CHECK-REMOTE-CUSTOM: {{-REMOTE/custom-input/1.txt$}}
2828
CHECK-REMOTE-CUSTOM-NEXT: {{-REMOTE/custom-input/2.txt$}}
@@ -32,8 +32,6 @@ CHECK: {{^1.txt$}}
3232
CHECK-NEXT: {{^2.txt$}}
3333
CHECK-NOT: BAD
3434

35-
VERBOSE-NESTED: /usr/bin/env /bin/mkdir -p {{.+}}-REMOTE/input/upload
36-
VERBOSE-NESTED-NEXT: /usr/bin/sftp
37-
VERBOSE-NESTED-DAG: -put '{{.+}}/Inputs/upload/1.txt' '{{.+}}-REMOTE/input/upload/1.txt'
38-
VERBOSE-NESTED-DAG: -put '{{.+}}/Inputs/upload/2.txt' '{{.+}}-REMOTE/input/upload/2.txt'
39-
VERBOSE-NESTED: /usr/bin/env ls
35+
VERBOSE-NESTED: /usr/bin/env /bin/mkdir -p {{.+}}-REMOTE/input
36+
VERBOSE-NESTED-NEXT: /usr/bin/rsync
37+
VERBOSE-NESTED: /usr/bin/env {{.*}}ls

0 commit comments

Comments
 (0)