From fc114582ff4ffb81a51812d4ca2ca03d6c6f5707 Mon Sep 17 00:00:00 2001 From: Maxime Brunet Date: Tue, 2 Jul 2024 08:14:20 +0000 Subject: [PATCH 01/81] fix(remount): relocate libraries along with their symlinks (#255) (cherry picked from commit 46a78fb325b17354692fb88310d8943cb683e0e4) --- envbuilder.go | 2 +- internal/ebutil/libs.go | 86 +++++ internal/ebutil/libs_amd64.go | 7 + internal/ebutil/libs_arm64.go | 7 + internal/ebutil/mock_mounter_test.go | 44 +++ internal/ebutil/remount.go | 67 +++- internal/ebutil/remount_internal_test.go | 461 ++++++++++++++++++++++- 7 files changed, 655 insertions(+), 19 deletions(-) create mode 100644 internal/ebutil/libs.go create mode 100644 internal/ebutil/libs_amd64.go create mode 100644 internal/ebutil/libs_arm64.go diff --git a/envbuilder.go b/envbuilder.go index 5538ea9c..10311df7 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -438,7 +438,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) tempRemountDest := filepath.Join("/", MagicDir, "mnt") // ignorePrefixes is a superset of ignorePaths that we pass to kaniko's // IgnoreList. - ignorePrefixes := append([]string{"/proc", "/sys"}, ignorePaths...) + ignorePrefixes := append([]string{"/dev", "/proc", "/sys"}, ignorePaths...) restoreMounts, err := ebutil.TempRemount(options.Logger, tempRemountDest, ignorePrefixes...) defer func() { // restoreMounts should never be nil if err := restoreMounts(); err != nil { diff --git a/internal/ebutil/libs.go b/internal/ebutil/libs.go new file mode 100644 index 00000000..58206c0c --- /dev/null +++ b/internal/ebutil/libs.go @@ -0,0 +1,86 @@ +package ebutil + +import ( + "errors" + "fmt" + "os" + "path/filepath" +) + +// Container runtimes like NVIDIA mount individual libraries into the container +// (e.g. `.so.`) and create symlinks for them +// (e.g. `.so.1`). This code helps with finding the right library +// directory for the target Linux distribution as well as locating the symlinks. +// +// Please see [#143 (comment)] for further details. +// +// [#143 (comment)]: https://github.com/coder/envbuilder/issues/143#issuecomment-2192405828 + +// Based on https://github.com/NVIDIA/libnvidia-container/blob/v1.15.0/src/common.h#L29 +const usrLibDir = "/usr/lib64" + +const debianVersionFile = "/etc/debian_version" + +// libraryDirectoryPath returns the library directory. It returns a multiarch +// directory if the distribution is Debian or a derivative. +// +// Based on https://github.com/NVIDIA/libnvidia-container/blob/v1.15.0/src/nvc_container.c#L152-L165 +func libraryDirectoryPath(m mounter) (string, error) { + // Debian and its derivatives use a multiarch directory scheme. + if _, err := m.Stat(debianVersionFile); err != nil && !errors.Is(err, os.ErrNotExist) { + return "", fmt.Errorf("check if debian: %w", err) + } else if err == nil { + return usrLibMultiarchDir, nil + } + + return usrLibDir, nil +} + +// libraryDirectorySymlinks returns a mapping of each library (basename) with a +// list of their symlinks (basename). Libraries with no symlinks do not appear +// in the mapping. +func libraryDirectorySymlinks(m mounter, libDir string) (map[string][]string, error) { + des, err := m.ReadDir(libDir) + if err != nil { + return nil, fmt.Errorf("read directory %s: %w", libDir, err) + } + + libsSymlinks := make(map[string][]string) + for _, de := range des { + if de.IsDir() { + continue + } + + if de.Type()&os.ModeSymlink != os.ModeSymlink { + // Not a symlink. Skip. + continue + } + + symlink := filepath.Join(libDir, de.Name()) + path, err := m.EvalSymlinks(symlink) + if err != nil { + return nil, fmt.Errorf("eval symlink %s: %w", symlink, err) + } + + path = filepath.Base(path) + if _, ok := libsSymlinks[path]; !ok { + libsSymlinks[path] = make([]string, 0, 1) + } + + libsSymlinks[path] = append(libsSymlinks[path], de.Name()) + } + + return libsSymlinks, nil +} + +// moveLibSymlinks moves a list of symlinks from source to destination directory. +func moveLibSymlinks(m mounter, symlinks []string, srcDir, destDir string) error { + for _, l := range symlinks { + oldpath := filepath.Join(srcDir, l) + newpath := filepath.Join(destDir, l) + if err := m.Rename(oldpath, newpath); err != nil { + return fmt.Errorf("move symlink %s => %s: %w", oldpath, newpath, err) + } + } + return nil +} diff --git a/internal/ebutil/libs_amd64.go b/internal/ebutil/libs_amd64.go new file mode 100644 index 00000000..b3f8230b --- /dev/null +++ b/internal/ebutil/libs_amd64.go @@ -0,0 +1,7 @@ +//go:build amd64 + +package ebutil + +// Based on https://github.com/NVIDIA/libnvidia-container/blob/v1.15.0/src/common.h#L36 + +const usrLibMultiarchDir = "/usr/lib/x86_64-linux-gnu" diff --git a/internal/ebutil/libs_arm64.go b/internal/ebutil/libs_arm64.go new file mode 100644 index 00000000..c76fb834 --- /dev/null +++ b/internal/ebutil/libs_arm64.go @@ -0,0 +1,7 @@ +//go:build arm64 + +package ebutil + +// Based on https://github.com/NVIDIA/libnvidia-container/blob/v1.15.0/src/common.h#L52 + +const usrLibMultiarchDir = "/usr/lib/aarch64-linux-gnu" diff --git a/internal/ebutil/mock_mounter_test.go b/internal/ebutil/mock_mounter_test.go index 7445376a..4e664f4c 100644 --- a/internal/ebutil/mock_mounter_test.go +++ b/internal/ebutil/mock_mounter_test.go @@ -42,6 +42,21 @@ func (m *Mockmounter) EXPECT() *MockmounterMockRecorder { return m.recorder } +// EvalSymlinks mocks base method. +func (m *Mockmounter) EvalSymlinks(arg0 string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EvalSymlinks", arg0) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// EvalSymlinks indicates an expected call of EvalSymlinks. +func (mr *MockmounterMockRecorder) EvalSymlinks(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EvalSymlinks", reflect.TypeOf((*Mockmounter)(nil).EvalSymlinks), arg0) +} + // GetMounts mocks base method. func (m *Mockmounter) GetMounts() ([]*procfs.MountInfo, error) { m.ctrl.T.Helper() @@ -100,6 +115,35 @@ func (mr *MockmounterMockRecorder) OpenFile(arg0, arg1, arg2 any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OpenFile", reflect.TypeOf((*Mockmounter)(nil).OpenFile), arg0, arg1, arg2) } +// ReadDir mocks base method. +func (m *Mockmounter) ReadDir(arg0 string) ([]os.DirEntry, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReadDir", arg0) + ret0, _ := ret[0].([]os.DirEntry) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ReadDir indicates an expected call of ReadDir. +func (mr *MockmounterMockRecorder) ReadDir(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadDir", reflect.TypeOf((*Mockmounter)(nil).ReadDir), arg0) +} + +// Rename mocks base method. +func (m *Mockmounter) Rename(arg0, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Rename", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Rename indicates an expected call of Rename. +func (mr *MockmounterMockRecorder) Rename(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Rename", reflect.TypeOf((*Mockmounter)(nil).Rename), arg0, arg1) +} + // Stat mocks base method. func (m *Mockmounter) Stat(arg0 string) (os.FileInfo, error) { m.ctrl.T.Helper() diff --git a/internal/ebutil/remount.go b/internal/ebutil/remount.go index f4a2b416..0ae7b135 100644 --- a/internal/ebutil/remount.go +++ b/internal/ebutil/remount.go @@ -1,6 +1,7 @@ package ebutil import ( + "errors" "fmt" "os" "path/filepath" @@ -44,6 +45,16 @@ func tempRemount(m mounter, logf func(notcodersdk.LogLevel, string, ...any), bas return func() error { return nil }, fmt.Errorf("get mounts: %w", err) } + libDir, err := libraryDirectoryPath(m) + if err != nil { + return func() error { return nil }, fmt.Errorf("get lib directory: %w", err) + } + + libsSymlinks, err := libraryDirectorySymlinks(m, libDir) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return func() error { return nil }, fmt.Errorf("read lib symlinks: %w", err) + } + // temp move of all ro mounts mounts := map[string]string{} var restoreOnce sync.Once @@ -51,8 +62,19 @@ func tempRemount(m mounter, logf func(notcodersdk.LogLevel, string, ...any), bas // closer to attempt to restore original mount points restore = func() error { restoreOnce.Do(func() { + if len(mounts) == 0 { + return + } + + newLibDir, err := libraryDirectoryPath(m) + if err != nil { + merr = multierror.Append(merr, fmt.Errorf("get new lib directory: %w", err)) + return + } + for orig, moved := range mounts { - if err := remount(m, moved, orig); err != nil { + logf(notcodersdk.LogLevelTrace, "restore mount %s", orig) + if err := remount(m, moved, orig, newLibDir, libsSymlinks); err != nil { merr = multierror.Append(merr, fmt.Errorf("restore mount: %w", err)) } } @@ -77,7 +99,8 @@ outer: src := mountInfo.MountPoint dest := filepath.Join(base, src) - if err := remount(m, src, dest); err != nil { + logf(notcodersdk.LogLevelTrace, "temp remount %s", src) + if err := remount(m, src, dest, libDir, libsSymlinks); err != nil { return restore, fmt.Errorf("temp remount: %w", err) } @@ -87,30 +110,48 @@ outer: return restore, nil } -func remount(m mounter, src, dest string) error { +func remount(m mounter, src, dest, libDir string, libsSymlinks map[string][]string) error { stat, err := m.Stat(src) if err != nil { return fmt.Errorf("stat %s: %w", src, err) } + var destDir string if stat.IsDir() { destDir = dest } else { destDir = filepath.Dir(dest) + if destDir == usrLibDir || destDir == usrLibMultiarchDir { + // Restore mount to libDir + destDir = libDir + dest = filepath.Join(destDir, stat.Name()) + } } + if err := m.MkdirAll(destDir, 0o750); err != nil { return fmt.Errorf("ensure path: %w", err) } + if !stat.IsDir() { f, err := m.OpenFile(dest, os.O_CREATE, 0o640) if err != nil { return fmt.Errorf("ensure file path: %w", err) } - defer f.Close() + // This ensure the file is created, it will not be used. It can be closed immediately. + f.Close() + + if symlinks, ok := libsSymlinks[stat.Name()]; ok { + srcDir := filepath.Dir(src) + if err := moveLibSymlinks(m, symlinks, srcDir, destDir); err != nil { + return err + } + } } + if err := m.Mount(src, dest, "bind", syscall.MS_BIND, ""); err != nil { return fmt.Errorf("bind mount %s => %s: %w", src, dest, err) } + if err := m.Unmount(src, 0); err != nil { return fmt.Errorf("unmount orig src %s: %w", src, err) } @@ -131,6 +172,12 @@ type mounter interface { Mount(string, string, string, uintptr, string) error // Unmount wraps syscall.Unmount Unmount(string, int) error + // ReadDir wraps os.ReadDir + ReadDir(string) ([]os.DirEntry, error) + // EvalSymlinks wraps filepath.EvalSymlinks + EvalSymlinks(string) (string, error) + // Rename wraps os.Rename + Rename(string, string) error } // realMounter implements mounter and actually does the thing. @@ -161,3 +208,15 @@ func (m *realMounter) OpenFile(name string, flag int, perm os.FileMode) (*os.Fil func (m *realMounter) Stat(path string) (os.FileInfo, error) { return os.Stat(path) } + +func (m *realMounter) ReadDir(name string) ([]os.DirEntry, error) { + return os.ReadDir(name) +} + +func (m *realMounter) EvalSymlinks(path string) (string, error) { + return filepath.EvalSymlinks(path) +} + +func (m *realMounter) Rename(oldpath, newpath string) error { + return os.Rename(oldpath, newpath) +} diff --git a/internal/ebutil/remount_internal_test.go b/internal/ebutil/remount_internal_test.go index 41036177..ceb0bff3 100644 --- a/internal/ebutil/remount_internal_test.go +++ b/internal/ebutil/remount_internal_test.go @@ -2,6 +2,7 @@ package ebutil import ( "os" + "runtime" "strings" "syscall" "testing" @@ -15,6 +16,11 @@ import ( "github.com/prometheus/procfs" ) +var expectedLibMultiarchDir = map[string]string{ + "amd64": "/usr/lib/x86_64-linux-gnu", + "arm64": "/usr/lib/aarch64-linux-gnu", +} + func Test_tempRemount(t *testing.T) { t.Parallel() @@ -26,11 +32,14 @@ func Test_tempRemount(t *testing.T) { mounts := fakeMounts("/home", "/var/lib/modules:ro", "/proc", "/sys") mm.EXPECT().GetMounts().Return(mounts, nil) - mm.EXPECT().Stat("/var/lib/modules").Return(&fakeFileInfo{isDir: true}, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().ReadDir("/usr/lib64").Return(nil, os.ErrNotExist) + mm.EXPECT().Stat("/var/lib/modules").Return(&fakeFileInfo{name: "modules", isDir: true}, nil) mm.EXPECT().MkdirAll("/.test/var/lib/modules", os.FileMode(0o750)).Times(1).Return(nil) mm.EXPECT().Mount("/var/lib/modules", "/.test/var/lib/modules", "bind", uintptr(syscall.MS_BIND), "").Times(1).Return(nil) mm.EXPECT().Unmount("/var/lib/modules", 0).Times(1).Return(nil) - mm.EXPECT().Stat("/.test/var/lib/modules").Return(&fakeFileInfo{isDir: true}, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().Stat("/.test/var/lib/modules").Return(&fakeFileInfo{name: "modules", isDir: true}, nil) mm.EXPECT().MkdirAll("/var/lib/modules", os.FileMode(0o750)).Times(1).Return(nil) mm.EXPECT().Mount("/.test/var/lib/modules", "/var/lib/modules", "bind", uintptr(syscall.MS_BIND), "").Times(1).Return(nil) mm.EXPECT().Unmount("/.test/var/lib/modules", 0).Times(1).Return(nil) @@ -51,12 +60,15 @@ func Test_tempRemount(t *testing.T) { mounts := fakeMounts("/home", "/usr/bin/utility:ro", "/proc", "/sys") mm.EXPECT().GetMounts().Return(mounts, nil) - mm.EXPECT().Stat("/usr/bin/utility").Return(&fakeFileInfo{isDir: false}, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().ReadDir("/usr/lib64").Return(nil, os.ErrNotExist) + mm.EXPECT().Stat("/usr/bin/utility").Return(&fakeFileInfo{name: "modules", isDir: false}, nil) mm.EXPECT().MkdirAll("/.test/usr/bin", os.FileMode(0o750)).Times(1).Return(nil) mm.EXPECT().OpenFile("/.test/usr/bin/utility", os.O_CREATE, os.FileMode(0o640)).Times(1).Return(new(os.File), nil) mm.EXPECT().Mount("/usr/bin/utility", "/.test/usr/bin/utility", "bind", uintptr(syscall.MS_BIND), "").Times(1).Return(nil) mm.EXPECT().Unmount("/usr/bin/utility", 0).Times(1).Return(nil) - mm.EXPECT().Stat("/.test/usr/bin/utility").Return(&fakeFileInfo{isDir: false}, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().Stat("/.test/usr/bin/utility").Return(&fakeFileInfo{name: "modules", isDir: false}, nil) mm.EXPECT().MkdirAll("/usr/bin", os.FileMode(0o750)).Times(1).Return(nil) mm.EXPECT().OpenFile("/usr/bin/utility", os.O_CREATE, os.FileMode(0o640)).Times(1).Return(new(os.File), nil) mm.EXPECT().Mount("/.test/usr/bin/utility", "/usr/bin/utility", "bind", uintptr(syscall.MS_BIND), "").Times(1).Return(nil) @@ -70,6 +82,202 @@ func Test_tempRemount(t *testing.T) { _ = remount() }) + t.Run("OKLib", func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + mm := NewMockmounter(ctrl) + mounts := fakeMounts("/home", "/usr/lib64/lib.so.1:ro", "/proc", "/sys") + + mm.EXPECT().GetMounts().Return(mounts, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().ReadDir("/usr/lib64").Return([]os.DirEntry{ + &fakeDirEntry{ + name: "lib.so", + mode: os.ModeSymlink, + }, + &fakeDirEntry{ + name: "lib.so.1", + }, + &fakeDirEntry{ + name: "lib-other.so", + mode: os.ModeSymlink, + }, + &fakeDirEntry{ + name: "lib-other.so.1", + }, + &fakeDirEntry{ + name: "something.d", + isDir: true, + mode: os.ModeDir, + }, + }, nil) + mm.EXPECT().EvalSymlinks("/usr/lib64/lib.so").Return("/usr/lib64/lib.so.1", nil) + mm.EXPECT().EvalSymlinks("/usr/lib64/lib-other.so").Return("/usr/lib64/lib-other.so.1", nil) + mm.EXPECT().Stat("/usr/lib64/lib.so.1").Return(&fakeFileInfo{name: "lib.so.1", isDir: false}, nil) + mm.EXPECT().MkdirAll("/.test/usr/lib64", os.FileMode(0o750)).Times(1).Return(nil) + mm.EXPECT().OpenFile("/.test/usr/lib64/lib.so.1", os.O_CREATE, os.FileMode(0o640)).Times(1).Return(new(os.File), nil) + mm.EXPECT().Rename("/usr/lib64/lib.so", "/.test/usr/lib64/lib.so").Return(nil) + mm.EXPECT().Mount("/usr/lib64/lib.so.1", "/.test/usr/lib64/lib.so.1", "bind", uintptr(syscall.MS_BIND), "").Times(1).Return(nil) + mm.EXPECT().Unmount("/usr/lib64/lib.so.1", 0).Times(1).Return(nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().Stat("/.test/usr/lib64/lib.so.1").Return(&fakeFileInfo{name: "lib.so.1", isDir: false}, nil) + mm.EXPECT().MkdirAll("/usr/lib64", os.FileMode(0o750)).Times(1).Return(nil) + mm.EXPECT().OpenFile("/usr/lib64/lib.so.1", os.O_CREATE, os.FileMode(0o640)).Times(1).Return(new(os.File), nil) + mm.EXPECT().Rename("/.test/usr/lib64/lib.so", "/usr/lib64/lib.so").Return(nil) + mm.EXPECT().Mount("/.test/usr/lib64/lib.so.1", "/usr/lib64/lib.so.1", "bind", uintptr(syscall.MS_BIND), "").Times(1).Return(nil) + mm.EXPECT().Unmount("/.test/usr/lib64/lib.so.1", 0).Times(1).Return(nil) + + remount, err := tempRemount(mm, fakeLog(t), "/.test") + require.NoError(t, err) + err = remount() + require.NoError(t, err) + // sync.Once should handle multiple remount calls + _ = remount() + }) + + t.Run("OKLibDebian", func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + mm := NewMockmounter(ctrl) + mounts := fakeMounts("/home", "/usr/lib64/lib.so.1:ro", "/proc", "/sys") + + mm.EXPECT().GetMounts().Return(mounts, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().ReadDir("/usr/lib64").Return([]os.DirEntry{ + &fakeDirEntry{ + name: "lib.so", + mode: os.ModeSymlink, + }, + &fakeDirEntry{ + name: "lib.so.1", + }, + &fakeDirEntry{ + name: "lib-other.so", + mode: os.ModeSymlink, + }, + &fakeDirEntry{ + name: "lib-other.so.1", + }, + &fakeDirEntry{ + name: "something.d", + isDir: true, + mode: os.ModeDir, + }, + }, nil) + mm.EXPECT().EvalSymlinks("/usr/lib64/lib.so").Return("lib.so.1", nil) + mm.EXPECT().EvalSymlinks("/usr/lib64/lib-other.so").Return("lib-other.so.1", nil) + mm.EXPECT().Stat("/usr/lib64/lib.so.1").Return(&fakeFileInfo{name: "lib.so.1", isDir: false}, nil) + mm.EXPECT().MkdirAll("/.test/usr/lib64", os.FileMode(0o750)).Times(1).Return(nil) + mm.EXPECT().OpenFile("/.test/usr/lib64/lib.so.1", os.O_CREATE, os.FileMode(0o640)).Times(1).Return(new(os.File), nil) + mm.EXPECT().Rename("/usr/lib64/lib.so", "/.test/usr/lib64/lib.so").Return(nil) + mm.EXPECT().Mount("/usr/lib64/lib.so.1", "/.test/usr/lib64/lib.so.1", "bind", uintptr(syscall.MS_BIND), "").Times(1).Return(nil) + mm.EXPECT().Unmount("/usr/lib64/lib.so.1", 0).Times(1).Return(nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, nil) + mm.EXPECT().Stat("/.test/usr/lib64/lib.so.1").Return(&fakeFileInfo{name: "lib.so.1", isDir: false}, nil) + mm.EXPECT().MkdirAll(expectedLibMultiarchDir[runtime.GOARCH], os.FileMode(0o750)).Times(1).Return(nil) + mm.EXPECT().OpenFile(expectedLibMultiarchDir[runtime.GOARCH]+"/lib.so.1", os.O_CREATE, os.FileMode(0o640)).Times(1).Return(new(os.File), nil) + mm.EXPECT().Rename("/.test/usr/lib64/lib.so", expectedLibMultiarchDir[runtime.GOARCH]+"/lib.so").Return(nil) + mm.EXPECT().Mount("/.test/usr/lib64/lib.so.1", expectedLibMultiarchDir[runtime.GOARCH]+"/lib.so.1", "bind", uintptr(syscall.MS_BIND), "").Times(1).Return(nil) + mm.EXPECT().Unmount("/.test/usr/lib64/lib.so.1", 0).Times(1).Return(nil) + + remount, err := tempRemount(mm, fakeLog(t), "/.test") + require.NoError(t, err) + err = remount() + require.NoError(t, err) + // sync.Once should handle multiple remount calls + _ = remount() + }) + + t.Run("OKLibFromDebianToNotDebian", func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + mm := NewMockmounter(ctrl) + mounts := fakeMounts("/home", expectedLibMultiarchDir[runtime.GOARCH]+"/lib.so.1:ro", "/proc", "/sys") + + mm.EXPECT().GetMounts().Return(mounts, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, nil) + mm.EXPECT().ReadDir(expectedLibMultiarchDir[runtime.GOARCH]).Return([]os.DirEntry{ + &fakeDirEntry{ + name: "lib.so", + mode: os.ModeSymlink, + }, + &fakeDirEntry{ + name: "lib.so.1", + }, + &fakeDirEntry{ + name: "lib-other.so", + mode: os.ModeSymlink, + }, + &fakeDirEntry{ + name: "lib-other.so.1", + }, + &fakeDirEntry{ + name: "something.d", + isDir: true, + mode: os.ModeDir, + }, + }, nil) + mm.EXPECT().EvalSymlinks(expectedLibMultiarchDir[runtime.GOARCH]+"/lib.so").Return(expectedLibMultiarchDir[runtime.GOARCH]+"/lib.so.1", nil) + mm.EXPECT().EvalSymlinks(expectedLibMultiarchDir[runtime.GOARCH]+"/lib-other.so").Return(expectedLibMultiarchDir[runtime.GOARCH]+"/usr/lib64/lib-other.so.1", nil) + mm.EXPECT().Stat(expectedLibMultiarchDir[runtime.GOARCH]+"/lib.so.1").Return(&fakeFileInfo{name: "lib.so.1", isDir: false}, nil) + mm.EXPECT().MkdirAll("/.test"+expectedLibMultiarchDir[runtime.GOARCH], os.FileMode(0o750)).Times(1).Return(nil) + mm.EXPECT().OpenFile("/.test"+expectedLibMultiarchDir[runtime.GOARCH]+"/lib.so.1", os.O_CREATE, os.FileMode(0o640)).Times(1).Return(new(os.File), nil) + mm.EXPECT().Rename(expectedLibMultiarchDir[runtime.GOARCH]+"/lib.so", "/.test"+expectedLibMultiarchDir[runtime.GOARCH]+"/lib.so").Return(nil) + mm.EXPECT().Mount(expectedLibMultiarchDir[runtime.GOARCH]+"/lib.so.1", "/.test"+expectedLibMultiarchDir[runtime.GOARCH]+"/lib.so.1", "bind", uintptr(syscall.MS_BIND), "").Times(1).Return(nil) + mm.EXPECT().Unmount(expectedLibMultiarchDir[runtime.GOARCH]+"/lib.so.1", 0).Times(1).Return(nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().Stat("/.test"+expectedLibMultiarchDir[runtime.GOARCH]+"/lib.so.1").Return(&fakeFileInfo{name: "lib.so.1", isDir: false}, nil) + mm.EXPECT().MkdirAll("/usr/lib64", os.FileMode(0o750)).Times(1).Return(nil) + mm.EXPECT().OpenFile("/usr/lib64/lib.so.1", os.O_CREATE, os.FileMode(0o640)).Times(1).Return(new(os.File), nil) + mm.EXPECT().Rename("/.test"+expectedLibMultiarchDir[runtime.GOARCH]+"/lib.so", "/usr/lib64/lib.so").Return(nil) + mm.EXPECT().Mount("/.test"+expectedLibMultiarchDir[runtime.GOARCH]+"/lib.so.1", "/usr/lib64/lib.so.1", "bind", uintptr(syscall.MS_BIND), "").Times(1).Return(nil) + mm.EXPECT().Unmount("/.test"+expectedLibMultiarchDir[runtime.GOARCH]+"/lib.so.1", 0).Times(1).Return(nil) + + remount, err := tempRemount(mm, fakeLog(t), "/.test") + require.NoError(t, err) + err = remount() + require.NoError(t, err) + // sync.Once should handle multiple remount calls + _ = remount() + }) + + t.Run("OKLibNoSymlink", func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + mm := NewMockmounter(ctrl) + mounts := fakeMounts("/home", "/usr/lib64/lib.so.1:ro", "/proc", "/sys") + + mm.EXPECT().GetMounts().Return(mounts, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().ReadDir("/usr/lib64").Return([]os.DirEntry{ + &fakeDirEntry{ + name: "lib.so.1", + }, + }, nil) + mm.EXPECT().Stat("/usr/lib64/lib.so.1").Return(&fakeFileInfo{name: "lib.so.1", isDir: false}, nil) + mm.EXPECT().MkdirAll("/.test/usr/lib64", os.FileMode(0o750)).Times(1).Return(nil) + mm.EXPECT().OpenFile("/.test/usr/lib64/lib.so.1", os.O_CREATE, os.FileMode(0o640)).Times(1).Return(new(os.File), nil) + mm.EXPECT().Mount("/usr/lib64/lib.so.1", "/.test/usr/lib64/lib.so.1", "bind", uintptr(syscall.MS_BIND), "").Times(1).Return(nil) + mm.EXPECT().Unmount("/usr/lib64/lib.so.1", 0).Times(1).Return(nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().Stat("/.test/usr/lib64/lib.so.1").Return(&fakeFileInfo{name: "lib.so.1", isDir: false}, nil) + mm.EXPECT().MkdirAll("/usr/lib64", os.FileMode(0o750)).Times(1).Return(nil) + mm.EXPECT().OpenFile("/usr/lib64/lib.so.1", os.O_CREATE, os.FileMode(0o640)).Times(1).Return(new(os.File), nil) + mm.EXPECT().Mount("/.test/usr/lib64/lib.so.1", "/usr/lib64/lib.so.1", "bind", uintptr(syscall.MS_BIND), "").Times(1).Return(nil) + mm.EXPECT().Unmount("/.test/usr/lib64/lib.so.1", 0).Times(1).Return(nil) + + remount, err := tempRemount(mm, fakeLog(t), "/.test") + require.NoError(t, err) + err = remount() + require.NoError(t, err) + // sync.Once should handle multiple remount calls + _ = remount() + }) + t.Run("IgnorePrefixes", func(t *testing.T) { t.Parallel() @@ -78,6 +286,8 @@ func Test_tempRemount(t *testing.T) { mounts := fakeMounts("/home", "/var/lib/modules:ro", "/proc", "/sys") mm.EXPECT().GetMounts().Return(mounts, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().ReadDir("/usr/lib64").Return(nil, os.ErrNotExist) remount, err := tempRemount(mm, fakeLog(t), "/.test", "/var/lib") require.NoError(t, err) @@ -97,6 +307,39 @@ func Test_tempRemount(t *testing.T) { require.NoError(t, err) }) + t.Run("ErrStatDebianVersion", func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + mm := NewMockmounter(ctrl) + mounts := fakeMounts("/home", "/var/lib/modules:ro", "/proc", "/sys") + + mm.EXPECT().GetMounts().Return(mounts, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, assert.AnError) + + remount, err := tempRemount(mm, fakeLog(t), "/.test") + require.ErrorContains(t, err, assert.AnError.Error()) + err = remount() + require.NoError(t, err) + }) + + t.Run("ErrReadLibDir", func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + mm := NewMockmounter(ctrl) + mounts := fakeMounts("/home", "/var/lib/modules:ro", "/proc", "/sys") + + mm.EXPECT().GetMounts().Return(mounts, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().ReadDir("/usr/lib64").Return(nil, assert.AnError) + + remount, err := tempRemount(mm, fakeLog(t), "/.test") + require.ErrorContains(t, err, assert.AnError.Error()) + err = remount() + require.NoError(t, err) + }) + t.Run("ErrMkdirAll", func(t *testing.T) { t.Parallel() @@ -105,7 +348,9 @@ func Test_tempRemount(t *testing.T) { mounts := fakeMounts("/home", "/var/lib/modules:ro", "/proc", "/sys") mm.EXPECT().GetMounts().Return(mounts, nil) - mm.EXPECT().Stat("/var/lib/modules").Return(&fakeFileInfo{isDir: true}, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().ReadDir("/usr/lib64").Return(nil, os.ErrNotExist) + mm.EXPECT().Stat("/var/lib/modules").Return(&fakeFileInfo{name: "modules", isDir: true}, nil) mm.EXPECT().MkdirAll("/.test/var/lib/modules", os.FileMode(0o750)).Times(1).Return(assert.AnError) remount, err := tempRemount(mm, fakeLog(t), "/.test") @@ -114,6 +359,69 @@ func Test_tempRemount(t *testing.T) { require.NoError(t, err) }) + t.Run("ErrOpenFile", func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + mm := NewMockmounter(ctrl) + mounts := fakeMounts("/home", "/usr/bin/utility:ro", "/proc", "/sys") + + mm.EXPECT().GetMounts().Return(mounts, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().ReadDir("/usr/lib64").Return(nil, os.ErrNotExist) + mm.EXPECT().Stat("/usr/bin/utility").Return(&fakeFileInfo{name: "modules", isDir: false}, nil) + mm.EXPECT().MkdirAll("/.test/usr/bin", os.FileMode(0o750)).Times(1).Return(nil) + mm.EXPECT().OpenFile("/.test/usr/bin/utility", os.O_CREATE, os.FileMode(0o640)).Times(1).Return(nil, assert.AnError) + + remount, err := tempRemount(mm, fakeLog(t), "/.test") + require.ErrorContains(t, err, assert.AnError.Error()) + err = remount() + require.NoError(t, err) + }) + + t.Run("ErrMoveSymlink", func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + mm := NewMockmounter(ctrl) + mounts := fakeMounts("/home", "/usr/lib64/lib.so.1:ro", "/proc", "/sys") + + mm.EXPECT().GetMounts().Return(mounts, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().ReadDir("/usr/lib64").Return([]os.DirEntry{ + &fakeDirEntry{ + name: "lib.so", + mode: os.ModeSymlink, + }, + &fakeDirEntry{ + name: "lib.so.1", + }, + &fakeDirEntry{ + name: "lib-other.so", + mode: os.ModeSymlink, + }, + &fakeDirEntry{ + name: "lib-other.so.1", + }, + &fakeDirEntry{ + name: "something.d", + isDir: true, + mode: os.ModeDir, + }, + }, nil) + mm.EXPECT().EvalSymlinks("/usr/lib64/lib.so").Return("lib.so.1", nil) + mm.EXPECT().EvalSymlinks("/usr/lib64/lib-other.so").Return("lib-other.so.1", nil) + mm.EXPECT().Stat("/usr/lib64/lib.so.1").Return(&fakeFileInfo{name: "lib.so.1", isDir: false}, nil) + mm.EXPECT().MkdirAll("/.test/usr/lib64", os.FileMode(0o750)).Times(1).Return(nil) + mm.EXPECT().OpenFile("/.test/usr/lib64/lib.so.1", os.O_CREATE, os.FileMode(0o640)).Times(1).Return(new(os.File), nil) + mm.EXPECT().Rename("/usr/lib64/lib.so", "/.test/usr/lib64/lib.so").Return(assert.AnError) + + remount, err := tempRemount(mm, fakeLog(t), "/.test") + require.ErrorContains(t, err, assert.AnError.Error()) + err = remount() + require.NoError(t, err) + }) + t.Run("ErrMountBind", func(t *testing.T) { t.Parallel() @@ -122,7 +430,9 @@ func Test_tempRemount(t *testing.T) { mounts := fakeMounts("/home", "/var/lib/modules:ro", "/proc", "/sys") mm.EXPECT().GetMounts().Return(mounts, nil) - mm.EXPECT().Stat("/var/lib/modules").Return(&fakeFileInfo{isDir: true}, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().ReadDir("/usr/lib64").Return(nil, os.ErrNotExist) + mm.EXPECT().Stat("/var/lib/modules").Return(&fakeFileInfo{name: "modules", isDir: true}, nil) mm.EXPECT().MkdirAll("/.test/var/lib/modules", os.FileMode(0o750)).Times(1).Return(nil) mm.EXPECT().Mount("/var/lib/modules", "/.test/var/lib/modules", "bind", uintptr(syscall.MS_BIND), "").Times(1).Return(assert.AnError) @@ -140,7 +450,9 @@ func Test_tempRemount(t *testing.T) { mounts := fakeMounts("/home", "/var/lib/modules:ro", "/proc", "/sys") mm.EXPECT().GetMounts().Return(mounts, nil) - mm.EXPECT().Stat("/var/lib/modules").Return(&fakeFileInfo{isDir: true}, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().ReadDir("/usr/lib64").Return(nil, os.ErrNotExist) + mm.EXPECT().Stat("/var/lib/modules").Return(&fakeFileInfo{name: "modules", isDir: true}, nil) mm.EXPECT().MkdirAll("/.test/var/lib/modules", os.FileMode(0o750)).Times(1).Return(nil) mm.EXPECT().Mount("/var/lib/modules", "/.test/var/lib/modules", "bind", uintptr(syscall.MS_BIND), "").Times(1).Return(nil) mm.EXPECT().Unmount("/var/lib/modules", 0).Times(1).Return(assert.AnError) @@ -151,6 +463,28 @@ func Test_tempRemount(t *testing.T) { require.NoError(t, err) }) + t.Run("ErrRemountStatDebianVersion", func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + mm := NewMockmounter(ctrl) + mounts := fakeMounts("/home", "/var/lib/modules:ro", "/proc", "/sys") + + mm.EXPECT().GetMounts().Return(mounts, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().ReadDir("/usr/lib64").Return(nil, os.ErrNotExist) + mm.EXPECT().Stat("/var/lib/modules").Return(&fakeFileInfo{name: "modules", isDir: true}, nil) + mm.EXPECT().MkdirAll("/.test/var/lib/modules", os.FileMode(0o750)).Times(1).Return(nil) + mm.EXPECT().Mount("/var/lib/modules", "/.test/var/lib/modules", "bind", uintptr(syscall.MS_BIND), "").Times(1).Return(nil) + mm.EXPECT().Unmount("/var/lib/modules", 0).Times(1).Return(nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, assert.AnError) + + remount, err := tempRemount(mm, fakeLog(t), "/.test") + require.NoError(t, err) + err = remount() + require.ErrorContains(t, err, assert.AnError.Error()) + }) + t.Run("ErrRemountMkdirAll", func(t *testing.T) { t.Parallel() @@ -159,11 +493,14 @@ func Test_tempRemount(t *testing.T) { mounts := fakeMounts("/home", "/var/lib/modules:ro", "/proc", "/sys") mm.EXPECT().GetMounts().Return(mounts, nil) - mm.EXPECT().Stat("/var/lib/modules").Return(&fakeFileInfo{isDir: true}, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().ReadDir("/usr/lib64").Return(nil, os.ErrNotExist) + mm.EXPECT().Stat("/var/lib/modules").Return(&fakeFileInfo{name: "modules", isDir: true}, nil) mm.EXPECT().MkdirAll("/.test/var/lib/modules", os.FileMode(0o750)).Times(1).Return(nil) mm.EXPECT().Mount("/var/lib/modules", "/.test/var/lib/modules", "bind", uintptr(syscall.MS_BIND), "").Times(1).Return(nil) mm.EXPECT().Unmount("/var/lib/modules", 0).Times(1).Return(nil) - mm.EXPECT().Stat("/.test/var/lib/modules").Return(&fakeFileInfo{isDir: true}, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().Stat("/.test/var/lib/modules").Return(&fakeFileInfo{name: "modules", isDir: true}, nil) mm.EXPECT().MkdirAll("/var/lib/modules", os.FileMode(0o750)).Times(1).Return(assert.AnError) remount, err := tempRemount(mm, fakeLog(t), "/.test") @@ -172,6 +509,82 @@ func Test_tempRemount(t *testing.T) { require.ErrorContains(t, err, assert.AnError.Error()) }) + t.Run("ErrRemountOpenFile", func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + mm := NewMockmounter(ctrl) + mounts := fakeMounts("/home", "/usr/bin/utility:ro", "/proc", "/sys") + + mm.EXPECT().GetMounts().Return(mounts, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().ReadDir("/usr/lib64").Return(nil, os.ErrNotExist) + mm.EXPECT().Stat("/usr/bin/utility").Return(&fakeFileInfo{name: "modules", isDir: false}, nil) + mm.EXPECT().MkdirAll("/.test/usr/bin", os.FileMode(0o750)).Times(1).Return(nil) + mm.EXPECT().OpenFile("/.test/usr/bin/utility", os.O_CREATE, os.FileMode(0o640)).Times(1).Return(new(os.File), nil) + mm.EXPECT().Mount("/usr/bin/utility", "/.test/usr/bin/utility", "bind", uintptr(syscall.MS_BIND), "").Times(1).Return(nil) + mm.EXPECT().Unmount("/usr/bin/utility", 0).Times(1).Return(nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().Stat("/.test/usr/bin/utility").Return(&fakeFileInfo{name: "modules", isDir: false}, nil) + mm.EXPECT().MkdirAll("/usr/bin", os.FileMode(0o750)).Times(1).Return(nil) + mm.EXPECT().OpenFile("/usr/bin/utility", os.O_CREATE, os.FileMode(0o640)).Times(1).Return(nil, assert.AnError) + + remount, err := tempRemount(mm, fakeLog(t), "/.test") + require.NoError(t, err) + err = remount() + require.ErrorContains(t, err, assert.AnError.Error()) + }) + + t.Run("ErrRemountMoveSymlink", func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + mm := NewMockmounter(ctrl) + mounts := fakeMounts("/home", "/usr/lib64/lib.so.1:ro", "/proc", "/sys") + + mm.EXPECT().GetMounts().Return(mounts, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().ReadDir("/usr/lib64").Return([]os.DirEntry{ + &fakeDirEntry{ + name: "lib.so", + mode: os.ModeSymlink, + }, + &fakeDirEntry{ + name: "lib.so.1", + }, + &fakeDirEntry{ + name: "lib-other.so", + mode: os.ModeSymlink, + }, + &fakeDirEntry{ + name: "lib-other.so.1", + }, + &fakeDirEntry{ + name: "something.d", + isDir: true, + mode: os.ModeDir, + }, + }, nil) + mm.EXPECT().EvalSymlinks("/usr/lib64/lib.so").Return("/usr/lib64/lib.so.1", nil) + mm.EXPECT().EvalSymlinks("/usr/lib64/lib-other.so").Return("/usr/lib64/lib-other.so.1", nil) + mm.EXPECT().Stat("/usr/lib64/lib.so.1").Return(&fakeFileInfo{name: "lib.so.1", isDir: false}, nil) + mm.EXPECT().MkdirAll("/.test/usr/lib64", os.FileMode(0o750)).Times(1).Return(nil) + mm.EXPECT().OpenFile("/.test/usr/lib64/lib.so.1", os.O_CREATE, os.FileMode(0o640)).Times(1).Return(new(os.File), nil) + mm.EXPECT().Rename("/usr/lib64/lib.so", "/.test/usr/lib64/lib.so").Return(nil) + mm.EXPECT().Mount("/usr/lib64/lib.so.1", "/.test/usr/lib64/lib.so.1", "bind", uintptr(syscall.MS_BIND), "").Times(1).Return(nil) + mm.EXPECT().Unmount("/usr/lib64/lib.so.1", 0).Times(1).Return(nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().Stat("/.test/usr/lib64/lib.so.1").Return(&fakeFileInfo{name: "lib.so.1", isDir: false}, nil) + mm.EXPECT().MkdirAll("/usr/lib64", os.FileMode(0o750)).Times(1).Return(nil) + mm.EXPECT().OpenFile("/usr/lib64/lib.so.1", os.O_CREATE, os.FileMode(0o640)).Times(1).Return(new(os.File), nil) + mm.EXPECT().Rename("/.test/usr/lib64/lib.so", "/usr/lib64/lib.so").Return(assert.AnError) + + remount, err := tempRemount(mm, fakeLog(t), "/.test") + require.NoError(t, err) + err = remount() + require.ErrorContains(t, err, assert.AnError.Error()) + }) + t.Run("ErrRemountMountBind", func(t *testing.T) { t.Parallel() @@ -180,11 +593,14 @@ func Test_tempRemount(t *testing.T) { mounts := fakeMounts("/home", "/var/lib/modules:ro", "/proc", "/sys") mm.EXPECT().GetMounts().Return(mounts, nil) - mm.EXPECT().Stat("/var/lib/modules").Return(&fakeFileInfo{isDir: true}, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().ReadDir("/usr/lib64").Return(nil, os.ErrNotExist) + mm.EXPECT().Stat("/var/lib/modules").Return(&fakeFileInfo{name: "modules", isDir: true}, nil) mm.EXPECT().MkdirAll("/.test/var/lib/modules", os.FileMode(0o750)).Times(1).Return(nil) mm.EXPECT().Mount("/var/lib/modules", "/.test/var/lib/modules", "bind", uintptr(syscall.MS_BIND), "").Times(1).Return(nil) mm.EXPECT().Unmount("/var/lib/modules", 0).Times(1).Return(nil) - mm.EXPECT().Stat("/.test/var/lib/modules").Return(&fakeFileInfo{isDir: true}, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().Stat("/.test/var/lib/modules").Return(&fakeFileInfo{name: "modules", isDir: true}, nil) mm.EXPECT().MkdirAll("/var/lib/modules", os.FileMode(0o750)).Times(1).Return(nil) mm.EXPECT().Mount("/.test/var/lib/modules", "/var/lib/modules", "bind", uintptr(syscall.MS_BIND), "").Times(1).Return(assert.AnError) @@ -202,11 +618,14 @@ func Test_tempRemount(t *testing.T) { mounts := fakeMounts("/home", "/var/lib/modules:ro", "/proc", "/sys") mm.EXPECT().GetMounts().Return(mounts, nil) - mm.EXPECT().Stat("/var/lib/modules").Return(&fakeFileInfo{isDir: true}, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().ReadDir("/usr/lib64").Return(nil, os.ErrNotExist) + mm.EXPECT().Stat("/var/lib/modules").Return(&fakeFileInfo{name: "modules", isDir: true}, nil) mm.EXPECT().MkdirAll("/.test/var/lib/modules", os.FileMode(0o750)).Times(1).Return(nil) mm.EXPECT().Mount("/var/lib/modules", "/.test/var/lib/modules", "bind", uintptr(syscall.MS_BIND), "").Times(1).Return(nil) mm.EXPECT().Unmount("/var/lib/modules", 0).Times(1).Return(nil) - mm.EXPECT().Stat("/.test/var/lib/modules").Return(&fakeFileInfo{isDir: true}, nil) + mm.EXPECT().Stat("/etc/debian_version").Return(nil, os.ErrNotExist) + mm.EXPECT().Stat("/.test/var/lib/modules").Return(&fakeFileInfo{name: "modules", isDir: true}, nil) mm.EXPECT().MkdirAll("/var/lib/modules", os.FileMode(0o750)).Times(1).Return(nil) mm.EXPECT().Mount("/.test/var/lib/modules", "/var/lib/modules", "bind", uintptr(syscall.MS_BIND), "").Times(1).Return(nil) mm.EXPECT().Unmount("/.test/var/lib/modules", 0).Times(1).Return(assert.AnError) @@ -241,10 +660,11 @@ func fakeLog(t *testing.T) func(notcodersdk.LogLevel, string, ...any) { } type fakeFileInfo struct { + name string isDir bool } -func (fi *fakeFileInfo) Name() string { return "" } +func (fi *fakeFileInfo) Name() string { return fi.name } func (fi *fakeFileInfo) Size() int64 { return 0 } func (fi *fakeFileInfo) Mode() os.FileMode { return 0 } func (fi *fakeFileInfo) ModTime() time.Time { return time.Time{} } @@ -252,3 +672,16 @@ func (fi *fakeFileInfo) IsDir() bool { return fi.isDir } func (fi *fakeFileInfo) Sys() any { return nil } var _ os.FileInfo = &fakeFileInfo{} + +type fakeDirEntry struct { + name string + isDir bool + mode os.FileMode +} + +func (de *fakeDirEntry) Name() string { return de.name } +func (de *fakeDirEntry) IsDir() bool { return de.isDir } +func (de *fakeDirEntry) Type() os.FileMode { return de.mode } +func (de *fakeDirEntry) Info() (os.FileInfo, error) { return nil, nil } + +var _ os.DirEntry = &fakeDirEntry{} From dda52530c288ee67bf00d82d021fa96ea3f616cb Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 2 Jul 2024 10:20:59 +0100 Subject: [PATCH 02/81] fix: add usrLibMultiarchDir for 32-bit ARM, whatever that means (#258) (cherry picked from commit aad3b53c25840fb51b3d51e886b7b9f85f38904d) --- internal/ebutil/libs_arm.go | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 internal/ebutil/libs_arm.go diff --git a/internal/ebutil/libs_arm.go b/internal/ebutil/libs_arm.go new file mode 100644 index 00000000..7c015b42 --- /dev/null +++ b/internal/ebutil/libs_arm.go @@ -0,0 +1,6 @@ +//go:build arm + +package ebutil + +// Based on a 32-bit Raspbian system. +const usrLibMultiarchDir = "/usr/lib/arm-linux-gnueabihf" From 00c6d775b61c57d8748da278890e9a5570147e41 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 2 Jul 2024 19:18:17 +0100 Subject: [PATCH 03/81] ci: add linters, run linters, make linters happy (#261) (cherry picked from commit 017ff9278939e064126bad4ff9884f3818af8c82) --- .github/workflows/ci.yaml | 3 +++ Makefile | 15 +++++++++++++++ cmd/envbuilder/main.go | 15 ++++++++++----- envbuilder_internal_test.go | 15 ++++++++++----- init.sh | 4 ++-- integration/integration_test.go | 6 ++++-- scripts/build.sh | 12 ++++++------ scripts/develop.sh | 2 +- scripts/diagram.sh | 2 +- scripts/version.sh | 2 +- testutil/registrytest/registrytest.go | 10 ---------- 11 files changed, 53 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6dc12b3d..dbd4e181 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -39,6 +39,9 @@ jobs: with: go-version: "~1.22" + - name: Lint + run: make -j lint + - name: Test run: make test docs: diff --git a/Makefile b/Makefile index 42fd1db4..01a4f216 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,25 @@ PWD=$(shell pwd) GO_SRC_FILES := $(shell find . -type f -name '*.go' -not -name '*_test.go') GO_TEST_FILES := $(shell find . -type f -not -name '*.go' -name '*_test.go') GOLDEN_FILES := $(shell find . -type f -name '*.golden') +SHELL_SRC_FILES := $(shell find . -type f -name '*.sh') +GOLANGCI_LINT_VERSION := v1.59.1 fmt: $(shell find . -type f -name '*.go') go run mvdan.cc/gofumpt@v0.6.0 -l -w . +.PHONY: lint +lint: lint/go lint/shellcheck + +.PHONY: lint/go +lint/go: $(GO_SRC_FILES) + go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) + golangci-lint run + +.PHONY: lint/shellcheck +lint/shellcheck: $(SHELL_SRC_FILES) + echo "--- shellcheck" + shellcheck --external-sources $(SHELL_SRC_FILES) + develop: ./scripts/develop.sh diff --git a/cmd/envbuilder/main.go b/cmd/envbuilder/main.go index aa3b3ec4..aea83d25 100644 --- a/cmd/envbuilder/main.go +++ b/cmd/envbuilder/main.go @@ -57,8 +57,11 @@ func envbuilderCmd() serpent.Command { }, } var flushAndClose func(ctx context.Context) error + // nolint: staticcheck // FIXME: https://github.com/coder/envbuilder/issues/260 sendLogs, flushAndClose = notcodersdk.LogsSender(notcodersdk.ExternalLogSourceID, client.PatchLogs, slog.Logger{}) - defer flushAndClose(inv.Context()) + defer func() { + _ = flushAndClose(inv.Context()) + }() // This adds the envbuilder subsystem. // If telemetry is enabled in a Coder deployment, @@ -66,19 +69,21 @@ func envbuilderCmd() serpent.Command { // envbuilder usage. if !slices.Contains(options.CoderAgentSubsystem, string(notcodersdk.AgentSubsystemEnvbuilder)) { options.CoderAgentSubsystem = append(options.CoderAgentSubsystem, string(notcodersdk.AgentSubsystemEnvbuilder)) - os.Setenv("CODER_AGENT_SUBSYSTEM", strings.Join(options.CoderAgentSubsystem, ",")) + _ = os.Setenv("CODER_AGENT_SUBSYSTEM", strings.Join(options.CoderAgentSubsystem, ",")) } } options.Logger = func(level notcodersdk.LogLevel, format string, args ...interface{}) { output := fmt.Sprintf(format, args...) - fmt.Fprintln(inv.Stderr, output) + _, _ = fmt.Fprintln(inv.Stderr, output) if sendLogs != nil { - sendLogs(inv.Context(), notcodersdk.Log{ + if err := sendLogs(inv.Context(), notcodersdk.Log{ CreatedAt: time.Now(), Output: output, Level: level, - }) + }); err != nil { + _, _ = fmt.Fprintf(inv.Stderr, "failed to send logs: %s\n", err.Error()) + } } } diff --git a/envbuilder_internal_test.go b/envbuilder_internal_test.go index 6ca5fc12..65edb9cd 100644 --- a/envbuilder_internal_test.go +++ b/envbuilder_internal_test.go @@ -52,7 +52,8 @@ func TestFindDevcontainerJSON(t *testing.T) { fs := memfs.New() err := fs.MkdirAll("/workspace/.devcontainer", 0o600) require.NoError(t, err) - fs.Create("/workspace/.devcontainer/devcontainer.json") + _, err = fs.Create("/workspace/.devcontainer/devcontainer.json") + require.NoError(t, err) // when devcontainerPath, devcontainerDir, err := findDevcontainerJSON(Options{ @@ -73,7 +74,8 @@ func TestFindDevcontainerJSON(t *testing.T) { fs := memfs.New() err := fs.MkdirAll("/workspace/experimental-devcontainer", 0o600) require.NoError(t, err) - fs.Create("/workspace/experimental-devcontainer/devcontainer.json") + _, err = fs.Create("/workspace/experimental-devcontainer/devcontainer.json") + require.NoError(t, err) // when devcontainerPath, devcontainerDir, err := findDevcontainerJSON(Options{ @@ -95,7 +97,8 @@ func TestFindDevcontainerJSON(t *testing.T) { fs := memfs.New() err := fs.MkdirAll("/workspace/.devcontainer", 0o600) require.NoError(t, err) - fs.Create("/workspace/.devcontainer/experimental.json") + _, err = fs.Create("/workspace/.devcontainer/experimental.json") + require.NoError(t, err) // when devcontainerPath, devcontainerDir, err := findDevcontainerJSON(Options{ @@ -117,7 +120,8 @@ func TestFindDevcontainerJSON(t *testing.T) { fs := memfs.New() err := fs.MkdirAll("/workspace", 0o600) require.NoError(t, err) - fs.Create("/workspace/devcontainer.json") + _, err = fs.Create("/workspace/devcontainer.json") + require.NoError(t, err) // when devcontainerPath, devcontainerDir, err := findDevcontainerJSON(Options{ @@ -138,7 +142,8 @@ func TestFindDevcontainerJSON(t *testing.T) { fs := memfs.New() err := fs.MkdirAll("/workspace/.devcontainer/sample", 0o600) require.NoError(t, err) - fs.Create("/workspace/.devcontainer/sample/devcontainer.json") + _, err = fs.Create("/workspace/.devcontainer/sample/devcontainer.json") + require.NoError(t, err) // when devcontainerPath, devcontainerDir, err := findDevcontainerJSON(Options{ diff --git a/init.sh b/init.sh index 350a664a..a2990e0d 100644 --- a/init.sh +++ b/init.sh @@ -3,5 +3,5 @@ echo hey there sleep 1 -echo INIT_COMMAND=/bin/sh >> $ENVBUILDER_ENV -echo INIT_ARGS="-c /bin/bash" >> $ENVBUILDER_ENV \ No newline at end of file +echo INIT_COMMAND=/bin/sh >> "${ENVBUILDER_ENV}" +echo INIT_ARGS="-c /bin/bash" >> "${ENVBUILDER_ENV}" \ No newline at end of file diff --git a/integration/integration_test.go b/integration/integration_test.go index 1364e966..ae7047c0 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -1612,9 +1612,11 @@ func cleanOldEnvbuilders() { panic(err) } for _, ctr := range ctrs { - cli.ContainerRemove(ctx, ctr.ID, container.RemoveOptions{ + if err := cli.ContainerRemove(ctx, ctr.ID, container.RemoveOptions{ Force: true, - }) + }); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "failed to remove old test container: %s\n", err.Error()) + } } } diff --git a/scripts/build.sh b/scripts/build.sh index 2fac5e04..e186dc02 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -cd $(dirname "${BASH_SOURCE[0]}") +cd "$(dirname "${BASH_SOURCE[0]}")" set -euo pipefail archs=() @@ -48,13 +48,13 @@ docker buildx inspect --bootstrap &> /dev/null for arch in "${archs[@]}"; do echo "Building for $arch..." - GOARCH=$arch CGO_ENABLED=0 go build -o ./envbuilder-$arch ../cmd/envbuilder & + GOARCH=$arch CGO_ENABLED=0 go build -o "./envbuilder-${arch}" ../cmd/envbuilder & done wait args=() for arch in "${archs[@]}"; do - args+=( --platform linux/$arch ) + args+=( --platform "linux/${arch}" ) done if [ "$push" = true ]; then args+=( --push ) @@ -62,10 +62,10 @@ else args+=( --load ) fi -docker buildx build --builder $BUILDER_NAME "${args[@]}" -t $base:$tag -t $base:latest -f Dockerfile . +docker buildx build --builder $BUILDER_NAME "${args[@]}" -t "${base}:${tag}" -t "${base}:latest" -f Dockerfile . # Check if archs contains the current. If so, then output a message! -if [[ -z "${CI:-}" ]] && [[ " ${archs[@]} " =~ " ${current} " ]]; then - docker tag $base:$tag envbuilder:latest +if [[ -z "${CI:-}" ]] && [[ " ${archs[*]} " =~ ${current} ]]; then + docker tag "${base}:${tag}" envbuilder:latest echo "Tagged $current as envbuilder:latest!" fi diff --git a/scripts/develop.sh b/scripts/develop.sh index 8336eca7..3147244b 100755 --- a/scripts/develop.sh +++ b/scripts/develop.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -cd $(dirname "${BASH_SOURCE[0]}") +cd "$(dirname "${BASH_SOURCE[0]}")" set -euxo pipefail ./build.sh diff --git a/scripts/diagram.sh b/scripts/diagram.sh index e0c5e6b4..b6fe5da2 100755 --- a/scripts/diagram.sh +++ b/scripts/diagram.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -cd $(dirname "${BASH_SOURCE[0]}") +cd "$(dirname "${BASH_SOURCE[0]}")" set -euxo pipefail d2 ./diagram.d2 --pad=32 -t 1 ./diagram-light.svg diff --git a/scripts/version.sh b/scripts/version.sh index bf78d02c..31968d27 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -cd $(dirname "${BASH_SOURCE[0]}") +cd "$(dirname "${BASH_SOURCE[0]}")" last_tag="$(git describe --tags --abbrev=0)" version="$last_tag" diff --git a/testutil/registrytest/registrytest.go b/testutil/registrytest/registrytest.go index 0bc3d312..033fd75b 100644 --- a/testutil/registrytest/registrytest.go +++ b/testutil/registrytest/registrytest.go @@ -44,16 +44,6 @@ func New(t *testing.T) string { return srv.URL } -type logrusFormatter struct { - callback func(entry *logrus.Entry) - empty []byte -} - -func (f *logrusFormatter) Format(entry *logrus.Entry) ([]byte, error) { - f.callback(entry) - return f.empty, nil -} - // WriteContainer uploads a container to the registry server. // It returns the reference to the uploaded container. func WriteContainer(t *testing.T, serverURL, containerRef, mediaType string, files map[string]any) string { From f0835fe1dd356456a1c70f1f444f7e48ef4bd4aa Mon Sep 17 00:00:00 2001 From: Maxime Brunet Date: Wed, 3 Jul 2024 15:51:08 +0000 Subject: [PATCH 04/81] fix(remount): correct usrLibMultiarchDir value for 32-bit ARM (#259) Co-authored-by: Cian Johnston (cherry picked from commit 0c49e02bdad706a5ab7fa2fa605eb7d7fd4098be) --- internal/ebutil/libs_arm.go | 5 +++-- internal/ebutil/remount_internal_test.go | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/ebutil/libs_arm.go b/internal/ebutil/libs_arm.go index 7c015b42..f73e3c44 100644 --- a/internal/ebutil/libs_arm.go +++ b/internal/ebutil/libs_arm.go @@ -2,5 +2,6 @@ package ebutil -// Based on a 32-bit Raspbian system. -const usrLibMultiarchDir = "/usr/lib/arm-linux-gnueabihf" +// This constant is for 64-bit systems. 32-bit ARM is not supported. +// If ever it becomes supported, it should be handled with a `usrLib32MultiarchDir` constant. +const usrLibMultiarchDir = "/var/empty" diff --git a/internal/ebutil/remount_internal_test.go b/internal/ebutil/remount_internal_test.go index ceb0bff3..fe44728e 100644 --- a/internal/ebutil/remount_internal_test.go +++ b/internal/ebutil/remount_internal_test.go @@ -18,6 +18,7 @@ import ( var expectedLibMultiarchDir = map[string]string{ "amd64": "/usr/lib/x86_64-linux-gnu", + "arm": "/var/empty", "arm64": "/usr/lib/aarch64-linux-gnu", } From bb37c71772878137b63ace6a51e4b393c594f3da Mon Sep 17 00:00:00 2001 From: Maxime Brunet Date: Wed, 3 Jul 2024 19:38:44 +0000 Subject: [PATCH 05/81] docs(readme): correct dockerconfig mount path (#263) (cherry picked from commit 6cb9954156774dcec4a71d8a13706037510fb3ea) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aa5c5e64..8525ca3d 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ resource "kubernetes_deployment" "example" { # Define the volumeMount with the pull credentials volume_mount { name = "docker-config-volume" - mount_path = "/envbuilder/config.json" + mount_path = "/.envbuilder/config.json" sub_path = ".dockerconfigjson" } } From 2538741c14879b71c4f9497bbd0daccd013edcc0 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 5 Jul 2024 11:33:05 +0100 Subject: [PATCH 06/81] feat: send logs using agent api v2 if available (#264) Closes #260 - Removes manually vendored codersdk, we're back on the real deal now - Moves logging setup to package internal/eblog - Modifies logging to use new agent api v2 methods, falling back to PatchLogs - Adds ENVBUILDER_VERBOSE Tested manually against Coder version 2.8.0, 2.9.0, and 2.13.0. (cherry picked from commit 5be06117e1bdf2f1d578601bef7c70ce6ba01d4e) --- .github/workflows/ci.yaml | 3 + Makefile | 2 +- README.md | 1 + cmd/envbuilder/main.go | 66 +--- cmd/envbuilder/main_test.go | 84 ---- envbuilder.go | 84 ++-- git.go | 26 +- git_test.go | 6 +- go.mod | 133 ++++++- go.sum | 468 ++++++++++++++++++++--- internal/ebutil/remount.go | 14 +- internal/ebutil/remount_internal_test.go | 6 +- internal/log/coder.go | 156 ++++++++ internal/log/coder_internal_test.go | 184 +++++++++ internal/log/log.go | 47 +++ internal/log/log_test.go | 29 ++ internal/notcodersdk/agentclient.go | 430 --------------------- internal/notcodersdk/doc.go | 13 - internal/notcodersdk/logs.go | 169 -------- options.go | 14 +- scripts/develop.sh | 6 +- testdata/options.golden | 3 + 22 files changed, 1053 insertions(+), 891 deletions(-) delete mode 100644 cmd/envbuilder/main_test.go create mode 100644 internal/log/coder.go create mode 100644 internal/log/coder_internal_test.go create mode 100644 internal/log/log.go create mode 100644 internal/log/log_test.go delete mode 100644 internal/notcodersdk/agentclient.go delete mode 100644 internal/notcodersdk/doc.go delete mode 100644 internal/notcodersdk/logs.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index dbd4e181..4e0d51cc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -39,6 +39,9 @@ jobs: with: go-version: "~1.22" + - name: Download Go modules + run: go mod download + - name: Lint run: make -j lint diff --git a/Makefile b/Makefile index 01a4f216..28827efc 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ lint: lint/go lint/shellcheck .PHONY: lint/go lint/go: $(GO_SRC_FILES) go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) - golangci-lint run + golangci-lint run --timeout=10m .PHONY: lint/shellcheck lint/shellcheck: $(SHELL_SRC_FILES) diff --git a/README.md b/README.md index 8525ca3d..34237c58 100644 --- a/README.md +++ b/README.md @@ -388,4 +388,5 @@ On MacOS or Windows systems, we recommend either using a VM or the provided `.de | `--coder-agent-subsystem` | `CODER_AGENT_SUBSYSTEM` | | Coder agent subsystems to report when forwarding logs. The envbuilder subsystem is always included. | | `--push-image` | `ENVBUILDER_PUSH_IMAGE` | | Push the built image to a remote registry. This option forces a reproducible build. | | `--get-cached-image` | `ENVBUILDER_GET_CACHED_IMAGE` | | Print the digest of the cached image, if available. Exits with an error if not found. | +| `--verbose` | `ENVBUILDER_VERBOSE` | | Enable verbose logging. | diff --git a/cmd/envbuilder/main.go b/cmd/envbuilder/main.go index aea83d25..77405536 100644 --- a/cmd/envbuilder/main.go +++ b/cmd/envbuilder/main.go @@ -1,20 +1,16 @@ package main import ( - "context" - "crypto/tls" "errors" "fmt" - "net/http" "net/url" "os" "slices" "strings" - "time" - "cdr.dev/slog" + "github.com/coder/coder/v2/codersdk" "github.com/coder/envbuilder" - "github.com/coder/envbuilder/internal/notcodersdk" + "github.com/coder/envbuilder/internal/log" "github.com/coder/serpent" // *Never* remove this. Certificates are not bundled as part @@ -38,58 +34,36 @@ func envbuilderCmd() serpent.Command { Use: "envbuilder", Options: options.CLI(), Handler: func(inv *serpent.Invocation) error { - var sendLogs func(ctx context.Context, log ...notcodersdk.Log) error - if options.CoderAgentToken != "" { - if options.CoderAgentURL == "" { + options.Logger = log.New(os.Stderr, options.Verbose) + if options.CoderAgentURL != "" { + if options.CoderAgentToken == "" { return errors.New("CODER_AGENT_URL must be set if CODER_AGENT_TOKEN is set") } u, err := url.Parse(options.CoderAgentURL) if err != nil { return fmt.Errorf("unable to parse CODER_AGENT_URL as URL: %w", err) } - client := notcodersdk.New(u) - client.SetSessionToken(options.CoderAgentToken) - client.HTTPClient = &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: options.Insecure, - }, - }, - } - var flushAndClose func(ctx context.Context) error - // nolint: staticcheck // FIXME: https://github.com/coder/envbuilder/issues/260 - sendLogs, flushAndClose = notcodersdk.LogsSender(notcodersdk.ExternalLogSourceID, client.PatchLogs, slog.Logger{}) - defer func() { - _ = flushAndClose(inv.Context()) - }() - - // This adds the envbuilder subsystem. - // If telemetry is enabled in a Coder deployment, - // this will be reported and help us understand - // envbuilder usage. - if !slices.Contains(options.CoderAgentSubsystem, string(notcodersdk.AgentSubsystemEnvbuilder)) { - options.CoderAgentSubsystem = append(options.CoderAgentSubsystem, string(notcodersdk.AgentSubsystemEnvbuilder)) - _ = os.Setenv("CODER_AGENT_SUBSYSTEM", strings.Join(options.CoderAgentSubsystem, ",")) - } - } - - options.Logger = func(level notcodersdk.LogLevel, format string, args ...interface{}) { - output := fmt.Sprintf(format, args...) - _, _ = fmt.Fprintln(inv.Stderr, output) - if sendLogs != nil { - if err := sendLogs(inv.Context(), notcodersdk.Log{ - CreatedAt: time.Now(), - Output: output, - Level: level, - }); err != nil { - _, _ = fmt.Fprintf(inv.Stderr, "failed to send logs: %s\n", err.Error()) + coderLog, closeLogs, err := log.Coder(inv.Context(), u, options.CoderAgentToken) + if err == nil { + options.Logger = log.Wrap(options.Logger, coderLog) + defer closeLogs() + // This adds the envbuilder subsystem. + // If telemetry is enabled in a Coder deployment, + // this will be reported and help us understand + // envbuilder usage. + if !slices.Contains(options.CoderAgentSubsystem, string(codersdk.AgentSubsystemEnvbuilder)) { + options.CoderAgentSubsystem = append(options.CoderAgentSubsystem, string(codersdk.AgentSubsystemEnvbuilder)) + _ = os.Setenv("CODER_AGENT_SUBSYSTEM", strings.Join(options.CoderAgentSubsystem, ",")) } + } else { + // Failure to log to Coder should cause a fatal error. + options.Logger(log.LevelError, "unable to send logs to Coder: %s", err.Error()) } } err := envbuilder.Run(inv.Context(), options) if err != nil { - options.Logger(notcodersdk.LogLevelError, "error: %s", err) + options.Logger(log.LevelError, "error: %s", err) } return err }, diff --git a/cmd/envbuilder/main_test.go b/cmd/envbuilder/main_test.go deleted file mode 100644 index ed1e0377..00000000 --- a/cmd/envbuilder/main_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "path/filepath" - "testing" - "time" - - "cdr.dev/slog/sloggers/slogtest" - "github.com/coder/envbuilder/internal/notcodersdk" - "github.com/coder/serpent" - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func Test_sendLogs(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - // Random token for testing log fowarding - agentToken := uuid.NewString() - - // Server to read logs posted by envbuilder. Matched to backlog limit. - logCh := make(chan notcodersdk.Log, 100) - logs := make([]notcodersdk.Log, 0) - go func() { - for { - select { - case <-ctx.Done(): - return - case log, ok := <-logCh: - if !ok { - return - } - logs = append(logs, log) - } - } - }() - logSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !assert.Equal(t, http.MethodPatch, r.Method) { - w.WriteHeader(http.StatusMethodNotAllowed) - return - } - assert.Equal(t, agentToken, r.Header.Get(notcodersdk.SessionTokenHeader)) - var res notcodersdk.PatchLogs - if !assert.NoError(t, json.NewDecoder(r.Body).Decode(&res)) { - w.WriteHeader(http.StatusInternalServerError) - return - } - if !assert.Equal(t, notcodersdk.ExternalLogSourceID, res.LogSourceID) { - w.WriteHeader(http.StatusInternalServerError) - return - } - for _, log := range res.Logs { - logCh <- log - } - w.WriteHeader(http.StatusOK) - })) - - // Make an empty working directory - tmpDir := t.TempDir() - t.Setenv("ENVBUILDER_DEVCONTAINER_DIR", tmpDir) - t.Setenv("ENVBUILDER_DOCKERFILE_DIR", filepath.Join(tmpDir, "Dockerfile")) - t.Setenv("ENVBUILDER_WORKSPACE_FOLDER", tmpDir) - t.Setenv("CODER_AGENT_TOKEN", agentToken) - t.Setenv("CODER_AGENT_URL", logSrv.URL) - - testLogger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) - cmd := envbuilderCmd() - inv := &serpent.Invocation{ - Command: &cmd, - Args: []string{}, - Logger: testLogger, - Environ: serpent.Environ{}, - } - - err := inv.WithOS().Run() - require.ErrorContains(t, err, "no such file or directory") - require.NotEmpty(t, logs) - require.Contains(t, logs[len(logs)-1].Output, "no such file or directory") -} diff --git a/envbuilder.go b/envbuilder.go index 10311df7..10773dd5 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -34,7 +34,7 @@ import ( giturls "github.com/chainguard-dev/git-urls" "github.com/coder/envbuilder/devcontainer" "github.com/coder/envbuilder/internal/ebutil" - "github.com/coder/envbuilder/internal/notcodersdk" + "github.com/coder/envbuilder/internal/log" "github.com/containerd/containerd/platforms" "github.com/distribution/distribution/v3/configuration" "github.com/distribution/distribution/v3/registry/handlers" @@ -125,14 +125,14 @@ func Run(ctx context.Context, options Options) error { now := time.Now() stageNumber++ stageNum := stageNumber - options.Logger(notcodersdk.LogLevelInfo, "#%d: %s", stageNum, fmt.Sprintf(format, args...)) + options.Logger(log.LevelInfo, "#%d: %s", stageNum, fmt.Sprintf(format, args...)) return func(format string, args ...any) { - options.Logger(notcodersdk.LogLevelInfo, "#%d: %s [%s]", stageNum, fmt.Sprintf(format, args...), time.Since(now)) + options.Logger(log.LevelInfo, "#%d: %s [%s]", stageNum, fmt.Sprintf(format, args...), time.Since(now)) } } - options.Logger(notcodersdk.LogLevelInfo, "%s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder")) + options.Logger(log.LevelInfo, "%s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder")) var caBundle []byte if options.SSLCertBase64 != "" { @@ -194,7 +194,7 @@ func Run(ctx context.Context, options Options) error { if line == "" { continue } - options.Logger(notcodersdk.LogLevelInfo, "#1: %s", strings.TrimSpace(line)) + options.Logger(log.LevelInfo, "#1: %s", strings.TrimSpace(line)) } } }() @@ -225,8 +225,8 @@ func Run(ctx context.Context, options Options) error { endStage("📦 The repository already exists!") } } else { - options.Logger(notcodersdk.LogLevelError, "Failed to clone repository: %s", fallbackErr.Error()) - options.Logger(notcodersdk.LogLevelError, "Falling back to the default image...") + options.Logger(log.LevelError, "Failed to clone repository: %s", fallbackErr.Error()) + options.Logger(log.LevelError, "Falling back to the default image...") } } @@ -270,8 +270,8 @@ func Run(ctx context.Context, options Options) error { var err error devcontainerPath, devcontainerDir, err = findDevcontainerJSON(options) if err != nil { - options.Logger(notcodersdk.LogLevelError, "Failed to locate devcontainer.json: %s", err.Error()) - options.Logger(notcodersdk.LogLevelError, "Falling back to the default image...") + options.Logger(log.LevelError, "Failed to locate devcontainer.json: %s", err.Error()) + options.Logger(log.LevelError, "Falling back to the default image...") } else { // We know a devcontainer exists. // Let's parse it and use it! @@ -292,7 +292,7 @@ func Run(ctx context.Context, options Options) error { if err != nil { return fmt.Errorf("no Dockerfile or image found: %w", err) } - options.Logger(notcodersdk.LogLevelInfo, "No Dockerfile or image specified; falling back to the default image...") + options.Logger(log.LevelInfo, "No Dockerfile or image specified; falling back to the default image...") fallbackDockerfile = defaultParams.DockerfilePath } buildParams, err = devContainer.Compile(options.Filesystem, devcontainerDir, MagicDir, fallbackDockerfile, options.WorkspaceFolder, false, os.LookupEnv) @@ -301,8 +301,8 @@ func Run(ctx context.Context, options Options) error { } scripts = devContainer.LifecycleScripts } else { - options.Logger(notcodersdk.LogLevelError, "Failed to parse devcontainer.json: %s", err.Error()) - options.Logger(notcodersdk.LogLevelError, "Falling back to the default image...") + options.Logger(log.LevelError, "Failed to parse devcontainer.json: %s", err.Error()) + options.Logger(log.LevelError, "Falling back to the default image...") } } } else { @@ -313,8 +313,8 @@ func Run(ctx context.Context, options Options) error { // not defined, show a warning dockerfileDir := filepath.Dir(dockerfilePath) if dockerfileDir != filepath.Clean(options.WorkspaceFolder) && options.BuildContextPath == "" { - options.Logger(notcodersdk.LogLevelWarn, "given dockerfile %q is below %q and no custom build context has been defined", dockerfilePath, options.WorkspaceFolder) - options.Logger(notcodersdk.LogLevelWarn, "\t-> set BUILD_CONTEXT_PATH to %q to fix", dockerfileDir) + options.Logger(log.LevelWarn, "given dockerfile %q is below %q and no custom build context has been defined", dockerfilePath, options.WorkspaceFolder) + options.Logger(log.LevelWarn, "\t-> set BUILD_CONTEXT_PATH to %q to fix", dockerfileDir) } dockerfile, err := options.Filesystem.Open(dockerfilePath) @@ -343,7 +343,7 @@ func Run(ctx context.Context, options Options) error { HijackLogrus(func(entry *logrus.Entry) { for _, line := range strings.Split(entry.Message, "\r") { - options.Logger(notcodersdk.LogLevelInfo, "#%d: %s", stageNumber, color.HiBlackString(line)) + options.Logger(log.LevelInfo, "#%d: %s", stageNumber, color.HiBlackString(line)) } }) @@ -376,7 +376,7 @@ func Run(ctx context.Context, options Options) error { go func() { err := srv.Serve(listener) if err != nil && !errors.Is(err, http.ErrServerClosed) { - options.Logger(notcodersdk.LogLevelError, "Failed to serve registry: %s", err.Error()) + options.Logger(log.LevelError, "Failed to serve registry: %s", err.Error()) } }() closeAfterBuild = func() { @@ -384,7 +384,7 @@ func Run(ctx context.Context, options Options) error { _ = listener.Close() } if options.CacheRepo != "" { - options.Logger(notcodersdk.LogLevelWarn, "Overriding cache repo with local registry...") + options.Logger(log.LevelWarn, "Overriding cache repo with local registry...") } options.CacheRepo = fmt.Sprintf("localhost:%d/local/cache", tcpAddr.Port) } @@ -442,7 +442,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) restoreMounts, err := ebutil.TempRemount(options.Logger, tempRemountDest, ignorePrefixes...) defer func() { // restoreMounts should never be nil if err := restoreMounts(); err != nil { - options.Logger(notcodersdk.LogLevelError, "restore mounts: %s", err.Error()) + options.Logger(log.LevelError, "restore mounts: %s", err.Error()) } }() if err != nil { @@ -488,13 +488,13 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) go func() { scanner := bufio.NewScanner(stdoutReader) for scanner.Scan() { - options.Logger(notcodersdk.LogLevelInfo, "%s", scanner.Text()) + options.Logger(log.LevelInfo, "%s", scanner.Text()) } }() go func() { scanner := bufio.NewScanner(stderrReader) for scanner.Scan() { - options.Logger(notcodersdk.LogLevelInfo, "%s", scanner.Text()) + options.Logger(log.LevelInfo, "%s", scanner.Text()) } }() cacheTTL := time.Hour * 24 * 7 @@ -607,13 +607,13 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) fallback = true fallbackErr = err case strings.Contains(err.Error(), "unexpected status code 401 Unauthorized"): - options.Logger(notcodersdk.LogLevelError, "Unable to pull the provided image. Ensure your registry credentials are correct!") + options.Logger(log.LevelError, "Unable to pull the provided image. Ensure your registry credentials are correct!") } if !fallback || options.ExitOnBuildFailure { return err } - options.Logger(notcodersdk.LogLevelError, "Failed to build: %s", err) - options.Logger(notcodersdk.LogLevelError, "Falling back to the default image...") + options.Logger(log.LevelError, "Failed to build: %s", err) + options.Logger(log.LevelError, "Falling back to the default image...") buildParams, err = defaultBuildParams() if err != nil { return err @@ -660,10 +660,10 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) if err != nil { return fmt.Errorf("unmarshal metadata: %w", err) } - options.Logger(notcodersdk.LogLevelInfo, "#3: 👀 Found devcontainer.json label metadata in image...") + options.Logger(log.LevelInfo, "#3: 👀 Found devcontainer.json label metadata in image...") for _, container := range devContainer { if container.RemoteUser != "" { - options.Logger(notcodersdk.LogLevelInfo, "#3: 🧑 Updating the user to %q!", container.RemoteUser) + options.Logger(log.LevelInfo, "#3: 🧑 Updating the user to %q!", container.RemoteUser) configFile.Config.User = container.RemoteUser } @@ -770,7 +770,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) username = buildParams.User } if username == "" { - options.Logger(notcodersdk.LogLevelWarn, "#3: no user specified, using root") + options.Logger(log.LevelWarn, "#3: no user specified, using root") } userInfo, err := getUser(username) @@ -793,7 +793,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) } return os.Chown(path, userInfo.uid, userInfo.gid) }); chownErr != nil { - options.Logger(notcodersdk.LogLevelError, "chown %q: %s", userInfo.user.HomeDir, chownErr.Error()) + options.Logger(log.LevelError, "chown %q: %s", userInfo.user.HomeDir, chownErr.Error()) endStage("⚠️ Failed to the ownership of the workspace, you may need to fix this manually!") } else { endStage("👤 Updated the ownership of the workspace!") @@ -810,7 +810,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) } return os.Chown(path, userInfo.uid, userInfo.gid) }); chownErr != nil { - options.Logger(notcodersdk.LogLevelError, "chown %q: %s", userInfo.user.HomeDir, chownErr.Error()) + options.Logger(log.LevelError, "chown %q: %s", userInfo.user.HomeDir, chownErr.Error()) endStage("⚠️ Failed to update ownership of %s, you may need to fix this manually!", userInfo.user.HomeDir) } else { endStage("🏡 Updated ownership of %s!", userInfo.user.HomeDir) @@ -846,7 +846,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) // We execute the initialize script as the root user! os.Setenv("HOME", "/root") - options.Logger(notcodersdk.LogLevelInfo, "=== Running the setup command %q as the root user...", options.SetupScript) + options.Logger(log.LevelInfo, "=== Running the setup command %q as the root user...", options.SetupScript) envKey := "ENVBUILDER_ENV" envFile := filepath.Join("/", MagicDir, "environ") @@ -873,7 +873,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) go func() { scanner := bufio.NewScanner(&buf) for scanner.Scan() { - options.Logger(notcodersdk.LogLevelInfo, "%s", scanner.Text()) + options.Logger(log.LevelInfo, "%s", scanner.Text()) } }() @@ -939,7 +939,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) return fmt.Errorf("set uid: %w", err) } - options.Logger(notcodersdk.LogLevelInfo, "=== Running the init command %s %+v as the %q user...", options.InitCommand, initArgs, userInfo.user.Username) + options.Logger(log.LevelInfo, "=== Running the init command %s %+v as the %q user...", options.InitCommand, initArgs, userInfo.user.Username) err = syscall.Exec(options.InitCommand, append([]string{options.InitCommand}, initArgs...), os.Environ()) if err != nil { @@ -1017,7 +1017,7 @@ func findUser(nameOrID string) (*user.User, error) { func execOneLifecycleScript( ctx context.Context, - logf func(level notcodersdk.LogLevel, format string, args ...any), + logf func(level log.Level, format string, args ...any), s devcontainer.LifecycleScript, scriptName string, userInfo userInfo, @@ -1025,9 +1025,9 @@ func execOneLifecycleScript( if s.IsEmpty() { return nil } - logf(notcodersdk.LogLevelInfo, "=== Running %s as the %q user...", scriptName, userInfo.user.Username) + logf(log.LevelInfo, "=== Running %s as the %q user...", scriptName, userInfo.user.Username) if err := s.Execute(ctx, userInfo.uid, userInfo.gid); err != nil { - logf(notcodersdk.LogLevelError, "Failed to run %s: %v", scriptName, err) + logf(log.LevelError, "Failed to run %s: %v", scriptName, err) return err } return nil @@ -1173,13 +1173,13 @@ func findDevcontainerJSON(options Options) (string, string, error) { for _, fileInfo := range fileInfos { if !fileInfo.IsDir() { - options.Logger(notcodersdk.LogLevelDebug, `%s is a file`, fileInfo.Name()) + options.Logger(log.LevelDebug, `%s is a file`, fileInfo.Name()) continue } location := filepath.Join(devcontainerDir, fileInfo.Name(), "devcontainer.json") if _, err := options.Filesystem.Stat(location); err != nil { - options.Logger(notcodersdk.LogLevelDebug, `stat %s failed: %s`, location, err.Error()) + options.Logger(log.LevelDebug, `stat %s failed: %s`, location, err.Error()) continue } @@ -1191,20 +1191,20 @@ func findDevcontainerJSON(options Options) (string, string, error) { // maybeDeleteFilesystem wraps util.DeleteFilesystem with a guard to hopefully stop // folks from unwittingly deleting their entire root directory. -func maybeDeleteFilesystem(log LoggerFunc, force bool) error { +func maybeDeleteFilesystem(logger log.Func, force bool) error { kanikoDir, ok := os.LookupEnv("KANIKO_DIR") if !ok || strings.TrimSpace(kanikoDir) != MagicDir { if force { bailoutSecs := 10 - log(notcodersdk.LogLevelWarn, "WARNING! BYPASSING SAFETY CHECK! THIS WILL DELETE YOUR ROOT FILESYSTEM!") - log(notcodersdk.LogLevelWarn, "You have %d seconds to bail out!", bailoutSecs) + logger(log.LevelWarn, "WARNING! BYPASSING SAFETY CHECK! THIS WILL DELETE YOUR ROOT FILESYSTEM!") + logger(log.LevelWarn, "You have %d seconds to bail out!", bailoutSecs) for i := bailoutSecs; i > 0; i-- { - log(notcodersdk.LogLevelWarn, "%d...", i) + logger(log.LevelWarn, "%d...", i) <-time.After(time.Second) } } else { - log(notcodersdk.LogLevelError, "KANIKO_DIR is not set to %s. Bailing!\n", MagicDir) - log(notcodersdk.LogLevelError, "To bypass this check, set FORCE_SAFE=true.") + logger(log.LevelError, "KANIKO_DIR is not set to %s. Bailing!\n", MagicDir) + logger(log.LevelError, "To bypass this check, set FORCE_SAFE=true.") return errors.New("safety check failed") } } diff --git a/git.go b/git.go index 09984fb4..f28cab8d 100644 --- a/git.go +++ b/git.go @@ -10,7 +10,7 @@ import ( "strings" giturls "github.com/chainguard-dev/git-urls" - "github.com/coder/envbuilder/internal/notcodersdk" + "github.com/coder/envbuilder/internal/log" "github.com/go-git/go-billy/v5" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" @@ -145,14 +145,14 @@ func ReadPrivateKey(path string) (gossh.Signer, error) { // LogHostKeyCallback is a HostKeyCallback that just logs host keys // and does nothing else. -func LogHostKeyCallback(log LoggerFunc) gossh.HostKeyCallback { +func LogHostKeyCallback(logger log.Func) gossh.HostKeyCallback { return func(hostname string, remote net.Addr, key gossh.PublicKey) error { var sb strings.Builder _ = knownhosts.WriteKnownHost(&sb, hostname, remote, key) // skeema/knownhosts uses a fake public key to determine the host key // algorithms. Ignore this one. if s := sb.String(); !strings.Contains(s, "fake-public-key ZmFrZSBwdWJsaWMga2V5") { - log(notcodersdk.LogLevelInfo, "#1: 🔑 Got host key: %s", strings.TrimSpace(s)) + logger(log.LevelInfo, "#1: 🔑 Got host key: %s", strings.TrimSpace(s)) } return nil } @@ -179,19 +179,19 @@ func LogHostKeyCallback(log LoggerFunc) gossh.HostKeyCallback { // performed as usual. func SetupRepoAuth(options *Options) transport.AuthMethod { if options.GitURL == "" { - options.Logger(notcodersdk.LogLevelInfo, "#1: ❔ No Git URL supplied!") + options.Logger(log.LevelInfo, "#1: ❔ No Git URL supplied!") return nil } if strings.HasPrefix(options.GitURL, "http://") || strings.HasPrefix(options.GitURL, "https://") { // Special case: no auth if options.GitUsername == "" && options.GitPassword == "" { - options.Logger(notcodersdk.LogLevelInfo, "#1: 👤 Using no authentication!") + options.Logger(log.LevelInfo, "#1: 👤 Using no authentication!") return nil } // Basic Auth // NOTE: we previously inserted the credentials into the repo URL. // This was removed in https://github.com/coder/envbuilder/pull/141 - options.Logger(notcodersdk.LogLevelInfo, "#1: 🔒 Using HTTP basic authentication!") + options.Logger(log.LevelInfo, "#1: 🔒 Using HTTP basic authentication!") return &githttp.BasicAuth{ Username: options.GitUsername, Password: options.GitPassword, @@ -205,29 +205,29 @@ func SetupRepoAuth(options *Options) transport.AuthMethod { } // Assume SSH auth for all other formats. - options.Logger(notcodersdk.LogLevelInfo, "#1: 🔑 Using SSH authentication!") + options.Logger(log.LevelInfo, "#1: 🔑 Using SSH authentication!") var signer ssh.Signer if options.GitSSHPrivateKeyPath != "" { s, err := ReadPrivateKey(options.GitSSHPrivateKeyPath) if err != nil { - options.Logger(notcodersdk.LogLevelError, "#1: ❌ Failed to read private key from %s: %s", options.GitSSHPrivateKeyPath, err.Error()) + options.Logger(log.LevelError, "#1: ❌ Failed to read private key from %s: %s", options.GitSSHPrivateKeyPath, err.Error()) } else { - options.Logger(notcodersdk.LogLevelInfo, "#1: 🔑 Using %s key!", s.PublicKey().Type()) + options.Logger(log.LevelInfo, "#1: 🔑 Using %s key!", s.PublicKey().Type()) signer = s } } // If no SSH key set, fall back to agent auth. if signer == nil { - options.Logger(notcodersdk.LogLevelError, "#1: 🔑 No SSH key found, falling back to agent!") + options.Logger(log.LevelError, "#1: 🔑 No SSH key found, falling back to agent!") auth, err := gitssh.NewSSHAgentAuth(options.GitUsername) if err != nil { - options.Logger(notcodersdk.LogLevelError, "#1: ❌ Failed to connect to SSH agent: %s", err.Error()) + options.Logger(log.LevelError, "#1: ❌ Failed to connect to SSH agent: %s", err.Error()) return nil // nothing else we can do } if os.Getenv("SSH_KNOWN_HOSTS") == "" { - options.Logger(notcodersdk.LogLevelWarn, "#1: 🔓 SSH_KNOWN_HOSTS not set, accepting all host keys!") + options.Logger(log.LevelWarn, "#1: 🔓 SSH_KNOWN_HOSTS not set, accepting all host keys!") auth.HostKeyCallback = LogHostKeyCallback(options.Logger) } return auth @@ -246,7 +246,7 @@ func SetupRepoAuth(options *Options) transport.AuthMethod { // Duplicated code due to Go's type system. if os.Getenv("SSH_KNOWN_HOSTS") == "" { - options.Logger(notcodersdk.LogLevelWarn, "#1: 🔓 SSH_KNOWN_HOSTS not set, accepting all host keys!") + options.Logger(log.LevelWarn, "#1: 🔓 SSH_KNOWN_HOSTS not set, accepting all host keys!") auth.HostKeyCallback = LogHostKeyCallback(options.Logger) } return auth diff --git a/git_test.go b/git_test.go index 35a1289c..38efee1a 100644 --- a/git_test.go +++ b/git_test.go @@ -13,7 +13,7 @@ import ( "testing" "github.com/coder/envbuilder" - "github.com/coder/envbuilder/internal/notcodersdk" + "github.com/coder/envbuilder/internal/log" "github.com/coder/envbuilder/testutil/gittest" "github.com/coder/envbuilder/testutil/mwtest" "github.com/go-git/go-billy/v5" @@ -404,8 +404,8 @@ func randKeygen(t *testing.T) gossh.Signer { return signer } -func testLog(t *testing.T) envbuilder.LoggerFunc { - return func(_ notcodersdk.LogLevel, format string, args ...interface{}) { +func testLog(t *testing.T) log.Func { + return func(_ log.Level, format string, args ...interface{}) { t.Logf(format, args...) } } diff --git a/go.mod b/go.mod index c831fdfc..5a56db96 100644 --- a/go.mod +++ b/go.mod @@ -1,18 +1,20 @@ module github.com/coder/envbuilder -go 1.22 - -toolchain go1.22.3 +go 1.22.4 // There are a few options we need added to Kaniko! // See: https://github.com/GoogleContainerTools/kaniko/compare/main...coder:kaniko:main replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240624091120-7208a49f5b15 +// Required to import codersdk due to gvisor dependency. +replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20240702054557-aa558fbe5374 + require ( cdr.dev/slog v1.6.2-0.20240126064726-20367d4aede6 github.com/GoogleContainerTools/kaniko v1.9.2 github.com/breml/rootcerts v0.2.10 github.com/chainguard-dev/git-urls v1.0.2 + github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352 github.com/coder/retry v1.5.1 github.com/coder/serpent v0.7.0 github.com/containerd/containerd v1.7.15 @@ -37,6 +39,7 @@ require ( github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a go.uber.org/mock v0.4.0 golang.org/x/crypto v0.24.0 + golang.org/x/mod v0.18.0 golang.org/x/sync v0.7.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 ) @@ -44,6 +47,7 @@ require ( require ( cloud.google.com/go/compute/metadata v0.3.0 // indirect dario.cat/mergo v1.0.0 // indirect + filippo.io/edwards25519 v1.1.0 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect @@ -56,12 +60,23 @@ require ( github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/DataDog/appsec-internal-go v1.5.0 // indirect + github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0 // indirect + github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1 // indirect + github.com/DataDog/datadog-go/v5 v5.3.0 // indirect + github.com/DataDog/go-libddwaf/v2 v2.4.2 // indirect + github.com/DataDog/go-tuf v1.0.2-0.5.2 // indirect + github.com/DataDog/gostackparse v0.7.0 // indirect + github.com/DataDog/sketches-go v1.4.2 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/Microsoft/hcsshim v0.11.4 // indirect - github.com/ProtonMail/go-crypto v1.0.0 // indirect + github.com/ProtonMail/go-crypto v1.1.0-alpha.2 // indirect github.com/agext/levenshtein v1.2.3 // indirect + github.com/akutz/memconn v0.1.0 // indirect + github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 // indirect github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect - github.com/aws/aws-sdk-go-v2 v1.26.1 // indirect + github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.30.0 // indirect github.com/aws/aws-sdk-go-v2/config v1.27.11 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.11 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect @@ -72,6 +87,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.23.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssm v1.49.3 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect @@ -79,12 +95,15 @@ require ( github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20240419161514-af205d85bb44 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/charmbracelet/lipgloss v0.8.0 // indirect github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 // indirect github.com/cilium/ebpf v0.12.3 // indirect github.com/cloudflare/circl v1.3.7 // indirect github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 // indirect + github.com/coder/quartz v0.1.0 // indirect + github.com/coder/terraform-provider-coder v0.23.0 // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/containerd/cgroups/v3 v3.0.2 // indirect github.com/containerd/continuity v0.4.3 // indirect @@ -93,9 +112,12 @@ require ( github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect github.com/containerd/ttrpc v1.2.3 // indirect github.com/containerd/typeurl/v2 v2.1.1 // indirect + github.com/coreos/go-iptables v0.6.0 // indirect + github.com/coreos/go-oidc/v3 v3.10.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/distribution/reference v0.6.0 // indirect @@ -105,39 +127,74 @@ require ( github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/ePirat/docker-credential-gitlabci v1.0.0 // indirect + github.com/ebitengine/purego v0.6.0-alpha.5 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/frankban/quicktest v1.14.6 // indirect + github.com/fxamacker/cbor/v2 v2.4.0 // indirect + github.com/go-chi/chi/v5 v5.0.10 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-jose/go-jose/v4 v4.0.1 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/google/btree v1.1.2 // indirect github.com/google/go-cmp v0.6.0 // indirect + github.com/google/nftables v0.1.1-0.20230115205135-9aa6fdf5a28c // indirect + github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect github.com/gorilla/handlers v1.5.1 // indirect github.com/gorilla/mux v1.8.1 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320 // indirect + github.com/hashicorp/go-hclog v1.5.0 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-memdb v1.3.2 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/golang-lru/arc/v2 v2.0.5 // indirect github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect + github.com/hashicorp/hcl/v2 v2.21.0 // indirect + github.com/hashicorp/logutils v1.0.0 // indirect + github.com/hashicorp/terraform-plugin-go v0.12.0 // indirect + github.com/hashicorp/terraform-plugin-log v0.7.0 // indirect + github.com/hashicorp/terraform-plugin-sdk/v2 v2.20.0 // indirect + github.com/hashicorp/yamux v0.1.1 // indirect + github.com/hdevalence/ed25519consensus v0.1.0 // indirect + github.com/illarion/gonotify v1.0.1 // indirect + github.com/insomniacslk/dhcp v0.0.0-20231206064809-8c70d406f6d2 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/josharian/native v1.1.1-0.20230202152459-5c7d0dd6ab86 // indirect + github.com/jsimonetti/rtnetlink v1.3.5 // indirect github.com/karrick/godirwalk v1.16.1 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/compress v1.17.4 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/kortschak/wol v0.0.0-20200729010619-da482cc4850a // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/mdlayher/genetlink v1.3.2 // indirect + github.com/mdlayher/netlink v1.7.2 // indirect + github.com/mdlayher/sdnotify v1.0.0 // indirect + github.com/mdlayher/socket v0.5.0 // indirect + github.com/miekg/dns v1.1.55 // indirect github.com/minio/highwayhash v1.0.2 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/go-ps v1.0.0 // indirect + github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect + github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/patternmatcher v0.6.0 // indirect @@ -156,45 +213,85 @@ require ( github.com/opencontainers/image-spec v1.1.0 // indirect github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/opencontainers/selinux v1.11.0 // indirect + github.com/outcaste-io/ristretto v0.2.3 // indirect + github.com/philhofer/fwd v1.1.2 // indirect + github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pion/transport/v2 v2.0.0 // indirect github.com/pion/udp v0.1.4 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.18.0 // indirect - github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.46.0 // indirect + github.com/prometheus/client_golang v1.19.1 // indirect + github.com/prometheus/client_model v0.6.0 // indirect + github.com/prometheus/common v0.48.0 // indirect github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5 // indirect github.com/redis/go-redis/extra/redisotel/v9 v9.0.5 // indirect github.com/redis/go-redis/v9 v9.1.0 // indirect + github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052 // indirect github.com/rivo/uniseg v0.4.4 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/secure-systems-lab/go-securesystemslib v0.7.0 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/tailscale/certstore v0.1.1-0.20220316223106-78d6e1c49d8d // indirect + github.com/tailscale/golang-x-crypto v0.0.0-20230713185742-f0b76a10a08e // indirect + github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05 // indirect + github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85 // indirect + github.com/tailscale/wireguard-go v0.0.0-20231121184858-cc193a0b3272 // indirect + github.com/tcnksm/go-httpstat v0.2.0 // indirect + github.com/tinylib/msgp v1.1.8 // indirect + github.com/u-root/uio v0.0.0-20240209044354-b3d14b93376a // indirect + github.com/valyala/fasthttp v1.55.0 // indirect github.com/vbatts/tar-split v0.11.5 // indirect + github.com/vishvananda/netlink v1.2.1-beta.2 // indirect + github.com/vishvananda/netns v0.0.4 // indirect + github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect + github.com/vmihailenco/msgpack/v4 v4.3.12 // indirect + github.com/vmihailenco/tagparser v0.1.2 // indirect + github.com/x448/float16 v0.8.4 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect + github.com/zclconf/go-cty v1.14.4 // indirect + github.com/zeebo/errs v1.3.0 // indirect go.etcd.io/etcd/raft/v3 v3.5.6 // indirect + go.nhat.io/otelsql v0.13.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 // indirect go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/sdk v1.24.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect - go.uber.org/goleak v1.3.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go4.org/mem v0.0.0-20220726221520-4f986261bf13 // indirect + go4.org/netipx v0.0.0-20230728180743-ad4cb58a6516 // indirect golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect - golang.org/x/mod v0.17.0 // indirect - golang.org/x/net v0.25.0 // indirect - golang.org/x/oauth2 v0.19.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/oauth2 v0.20.0 // indirect golang.org/x/sys v0.21.0 // indirect golang.org/x/term v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect - google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect - google.golang.org/grpc v1.63.2 // indirect - google.golang.org/protobuf v1.33.0 // indirect + golang.org/x/tools v0.22.0 // indirect + golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect + golang.zx2c4.com/wireguard/windows v0.5.3 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect + google.golang.org/grpc v1.64.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect + gopkg.in/DataDog/dd-trace-go.v1 v1.64.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + gvisor.dev/gvisor v0.0.0-20240509041132-65b30f7869dc // indirect + inet.af/peercred v0.0.0-20210906144145-0893ea02156a // indirect + nhooyr.io/websocket v1.8.7 // indirect + storj.io/drpc v0.0.33 // indirect + tailscale.com v1.46.1 // indirect ) diff --git a/go.sum b/go.sum index ee16941c..8a431485 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,19 @@ cdr.dev/slog v1.6.2-0.20240126064726-20367d4aede6 h1:KHblWIE/KHOwQ6lEbMZt6YpcGve2FEZ1sDtrW1Am5UI= cdr.dev/slog v1.6.2-0.20240126064726-20367d4aede6/go.mod h1:NaoTA7KwopCrnaSb0JXTC0PTp/O/Y83Lndnq0OEV3ZQ= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= +cloud.google.com/go v0.112.2 h1:ZaGT6LiG7dBzi6zNOvVZwacaXlmf3lRqnC4DQzqyRQw= cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/logging v1.9.0 h1:iEIOXFO9EmSiTjDmfpbRjOxECO7R8C7b8IXUGOj7xZw= cloud.google.com/go/logging v1.9.0/go.mod h1:1Io0vnZv4onoUnsVUQY3HZ3Igb1nBchky0A0y7BBBhE= -cloud.google.com/go/longrunning v0.5.5 h1:GOE6pZFdSrTb4KAiKnXsJBtlE6mEyaW44oKyMILWnOg= -cloud.google.com/go/longrunning v0.5.5/go.mod h1:WV2LAxD8/rg5Z1cNW6FJ/ZpX4E4VnDnoTk0yawPBB7s= +cloud.google.com/go/longrunning v0.5.6 h1:xAe8+0YaWoCKr9t1+aWe+OeQgN/iJK1fEgZSXmjuEaE= +cloud.google.com/go/longrunning v0.5.6/go.mod h1:vUaDrWYOMKRuhiv6JBnn49YxCPz2Ayn9GqyjaBT8/mA= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +filippo.io/mkcert v1.4.4 h1:8eVbbwfVlaqUM7OwuftKc2nuYOoTDQWqsoXmzoXZdbc= +filippo.io/mkcert v1.4.4/go.mod h1:VyvOchVuAye3BoUsPUOOofKygVwLV2KQMVFJNRq+1dA= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 h1:59MxjQVfjXsBpLy+dbd2/ELV5ofnUkUZBvWSC85sheA= @@ -42,23 +46,53 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= +github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= +github.com/DataDog/appsec-internal-go v1.5.0 h1:8kS5zSx5T49uZ8dZTdT19QVAvC/B8ByyZdhQKYQWHno= +github.com/DataDog/appsec-internal-go v1.5.0/go.mod h1:pEp8gjfNLtEOmz+iZqC8bXhu0h4k7NUsW/qiQb34k1U= +github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0 h1:bUMSNsw1iofWiju9yc1f+kBd33E3hMJtq9GuU602Iy8= +github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0/go.mod h1:HzySONXnAgSmIQfL6gOv9hWprKJkx8CicuXuUbmgWfo= +github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1 h1:5nE6N3JSs2IG3xzMthNFhXfOaXlrsdgqmJ73lndFf8c= +github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1/go.mod h1:Vc+snp0Bey4MrrJyiV2tVxxJb6BmLomPvN1RgAvjGaQ= +github.com/DataDog/datadog-go/v5 v5.3.0 h1:2q2qjFOb3RwAZNU+ez27ZVDwErJv5/VpbBPprz7Z+s8= +github.com/DataDog/datadog-go/v5 v5.3.0/go.mod h1:XRDJk1pTc00gm+ZDiBKsjh7oOOtJfYfglVCmFb8C2+Q= +github.com/DataDog/go-libddwaf/v2 v2.4.2 h1:ilquGKUmN9/Ty0sIxiEyznVRxP3hKfmH15Y1SMq5gjA= +github.com/DataDog/go-libddwaf/v2 v2.4.2/go.mod h1:gsCdoijYQfj8ce/T2bEDNPZFIYnmHluAgVDpuQOWMZE= +github.com/DataDog/go-tuf v1.0.2-0.5.2 h1:EeZr937eKAWPxJ26IykAdWA4A0jQXJgkhUjqEI/w7+I= +github.com/DataDog/go-tuf v1.0.2-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= +github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4= +github.com/DataDog/gostackparse v0.7.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM= +github.com/DataDog/sketches-go v1.4.2 h1:gppNudE9d19cQ98RYABOetxIhpTCl4m7CnbRZjvVA/o= +github.com/DataDog/sketches-go v1.4.2/go.mod h1:xJIXldczJyyjnbDop7ZZcLxJdV3+7Kra7H1KMgpgkLk= +github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= -github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= -github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/ProtonMail/go-crypto v1.1.0-alpha.2 h1:bkyFVUP+ROOARdgCiJzNQo2V2kiB97LyUpzH9P6Hrlg= +github.com/ProtonMail/go-crypto v1.1.0-alpha.2/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/akutz/memconn v0.1.0 h1:NawI0TORU4hcOMsMr11g7vwlCdkYeLKXBcxWu2W/P8A= +github.com/akutz/memconn v0.1.0/go.mod h1:Jo8rI7m0NieZyLI5e2CDlRdRqRRB4S7Xp77ukDjH+Fw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA= +github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/apparentlymart/go-dump v0.0.0-20190214190832-042adf3cf4a0 h1:MzVXffFUye+ZcSR6opIgz9Co7WcDx6ZcY+RjfFHoA0I= +github.com/apparentlymart/go-dump v0.0.0-20190214190832-042adf3cf4a0/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM= +github.com/apparentlymart/go-textseg/v12 v12.0.0/go.mod h1:S/4uRK2UtaQttw1GenVJEynmyUenKwP++x/+DdGV/Ec= +github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= +github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA= -github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= +github.com/aws/aws-sdk-go-v2 v1.30.0 h1:6qAwtzlfcTtcL8NHtbDQAqgM5s6NDipQTkPxyH/6kAA= +github.com/aws/aws-sdk-go-v2 v1.30.0/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA= github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE= github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs= @@ -79,6 +113,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1x github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk= +github.com/aws/aws-sdk-go-v2/service/ssm v1.49.3 h1:iT1/grX+znbCNKzF3nd54/5Zq6CYNnR5ZEHWnuWqULM= +github.com/aws/aws-sdk-go-v2/service/ssm v1.49.3/go.mod h1:loBAHYxz7JyucJvq4xuW9vunu8iCzjNYfSrQg2QEczA= github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w= github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE= @@ -95,6 +131,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bool64/shared v0.1.5 h1:fp3eUhBsrSjNCQPcSdQqZxxh9bBwrYiZ+zOKFkM0/2E= +github.com/bool64/shared v0.1.5/go.mod h1:081yz68YC9jeFB3+Bbmno2RFWvGKv1lPKkMP6MHJlPs= github.com/breml/rootcerts v0.2.10 h1:UGVZ193UTSUASpGtg6pbDwzOd7XQP+at0Ssg1/2E4h8= github.com/breml/rootcerts v0.2.10/go.mod h1:24FDtzYMpqIeYC7QzaE8VPRQaFZU5TIUDlyk8qwjD88= github.com/bsm/ginkgo/v2 v2.7.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w= @@ -102,38 +140,51 @@ github.com/bsm/ginkgo/v2 v2.9.5 h1:rtVBYPs3+TC5iLUVOis1B9tjLTup7Cj5IfzosKtvTJ0= github.com/bsm/ginkgo/v2 v2.9.5/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= github.com/bsm/gomega v1.26.0/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= -github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/bytedance/sonic v1.10.0 h1:qtNZduETEIWJVIyDl01BeNxur2rW9OwTQ/yBqFRkKEk= +github.com/bytedance/sonic v1.10.0/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chainguard-dev/git-urls v1.0.2 h1:pSpT7ifrpc5X55n4aTTm7FFUE+ZQHKiqpiwNkJrVcKQ= github.com/chainguard-dev/git-urls v1.0.2/go.mod h1:rbGgj10OS7UgZlbzdUQIQpT0k/D4+An04HJY7Ol+Y/o= github.com/charmbracelet/lipgloss v0.8.0 h1:IS00fk4XAHcf8uZKc3eHeMUTCxUH6NkaTrdyCQk84RU= github.com/charmbracelet/lipgloss v0.8.0/go.mod h1:p4eYUZZJ/0oXTuCQKFF8mqyKCz0ja6y+7DniDDw5KKU= +github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0= +github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA= +github.com/chenzhuoyu/iasm v0.9.0 h1:9fhXjVzq5hUy2gkhhgHl95zG2cEAhw9OSGs8toWWAwo= +github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 h1:krfRl01rzPzxSxyLyrChD+U+MzsBXbm0OwYYB67uF+4= github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589/go.mod h1:OuDyvmLnMCwa2ep4Jkm6nyA0ocJuZlGyk2gGseVzERM= github.com/cilium/ebpf v0.12.3 h1:8ht6F9MquybnY97at+VDZb3eQQr8ev79RueWeVaEcG4= github.com/cilium/ebpf v0.12.3/go.mod h1:TctK1ivibvI3znr66ljgi4hqOT8EYQjz1KWBfb1UVgM= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352 h1:L/EjCuZxs5tOcqqCaASj/nu65TRYEFcTt8qRQfHZXX0= +github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352/go.mod h1:P1KoQSgnKEAG6Mnd3YlGzAophty+yKA9VV48LpfNRvo= github.com/coder/kaniko v0.0.0-20240624091120-7208a49f5b15 h1:Rne2frxrqtLEQ/v4f/wS550Yp/WXLCRFzDuxg8b9woM= github.com/coder/kaniko v0.0.0-20240624091120-7208a49f5b15/go.mod h1:YMK7BlxerzLlMwihGxNWUaFoN9LXCij4P+w/8/fNlcM= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 h1:3A0ES21Ke+FxEM8CXx9n47SZOKOpgSE1bbJzlE4qPVs= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0/go.mod h1:5UuS2Ts+nTToAMeOjNlnHFkPahrtDkmpydBen/3wgZc= +github.com/coder/quartz v0.1.0 h1:cLL+0g5l7xTf6ordRnUMMiZtRE8Sq5LxpghS63vEXrQ= +github.com/coder/quartz v0.1.0/go.mod h1:vsiCc+AHViMKH2CQpGIpFgdHIEQsxwm8yCscqKmzbRA= github.com/coder/retry v1.5.1 h1:iWu8YnD8YqHs3XwqrqsjoBTAVqT9ml6z9ViJ2wlMiqc= github.com/coder/retry v1.5.1/go.mod h1:blHMk9vs6LkoRT9ZHyuZo360cufXEhrxqvEzeMtRGoY= github.com/coder/serpent v0.7.0 h1:zGpD2GlF3lKIVkMjNGKbkip88qzd5r/TRcc30X/SrT0= github.com/coder/serpent v0.7.0/go.mod h1:REkJ5ZFHQUWFTPLExhXYZ1CaHFjxvGNRlLXLdsI08YA= +github.com/coder/tailscale v1.1.1-0.20240702054557-aa558fbe5374 h1:a5Eg7D5e2oAc0tN56ee4yxtiTo76ztpRlk6geljaZp8= +github.com/coder/tailscale v1.1.1-0.20240702054557-aa558fbe5374/go.mod h1:rp6BIJxCp127/hvvDWNkHC9MxAlKvQfoOtBr8s5sCqo= +github.com/coder/terraform-provider-coder v0.23.0 h1:DuNLWxhnGlXyG0g+OCAZRI6xd8+bJjIEnE4F3hYgA4E= +github.com/coder/terraform-provider-coder v0.23.0/go.mod h1:wMun9UZ9HT2CzF6qPPBup1odzBpVUc0/xSFoXgdI3tk= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/cgroups/v3 v3.0.2 h1:f5WFqIVSgo5IZmtTT3qVBo6TzI1ON6sycSBKkymb9L0= @@ -152,18 +203,25 @@ github.com/containerd/ttrpc v1.2.3 h1:4jlhbXIGvijRtNC8F/5CpuJZ7yKOBFGFOOXg1bkISz github.com/containerd/ttrpc v1.2.3/go.mod h1:ieWsXucbb8Mj9PH0rXCw1i8IunRbbAiDkpXkbfflWBM= github.com/containerd/typeurl/v2 v2.1.1 h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9FqcttATPO/4= github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0= +github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk= +github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= +github.com/coreos/go-oidc/v3 v3.10.0 h1:tDnXHnLyiTVyT/2zLDGj09pFPkhND8Gl8lnTRhoEaJU= +github.com/coreos/go-oidc/v3 v3.10.0/go.mod h1:5j11xcw0D3+SGxn6Z/WFADsgcWVMyNAlSQupk0KK3ac= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= -github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0= +github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= @@ -188,8 +246,13 @@ github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQ github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/ePirat/docker-credential-gitlabci v1.0.0 h1:YRkUSvkON6rT88vtscClAmPEYWhtltGEAuRVYtz1/+Y= github.com/ePirat/docker-credential-gitlabci v1.0.0/go.mod h1:Ptmh+D0lzBQtgb6+QHjXl9HqOn3T1P8fKUHldiSQQGA= +github.com/ebitengine/purego v0.6.0-alpha.5 h1:EYID3JOAdmQ4SNZYJHu9V6IqOeRQDBYxqKAg9PyoHFY= +github.com/ebitengine/purego v0.6.0-alpha.5/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -198,6 +261,7 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= @@ -205,9 +269,22 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fxamacker/cbor/v2 v2.4.0 h1:ri0ArlOR+5XunOP8CRUowT0pSJOwhW098ZCUyskZD88= +github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= +github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= +github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= +github.com/github/fakeca v0.1.0 h1:Km/MVOFvclqxPM9dZBC4+QE564nU4gz4iZ0D9pMw28I= +github.com/github/fakeca v0.1.0/go.mod h1:+bormgoGMMuamOscx7N91aOuUST7wdaJ2rNjeohylyo= github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE= github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8= +github.com/go-chi/chi/v5 v5.0.10 h1:rLz5avzKpjqxrYwXNfmjkrYYXOyLJd37pz53UFHC6vk= +github.com/go-chi/chi/v5 v5.0.10/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= @@ -216,6 +293,8 @@ github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMj github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys= github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY= +github.com/go-jose/go-jose/v4 v4.0.1 h1:QVEPDE3OluqXBQZDcnNvQrInro2h0e4eqNbnZSWqS6U= +github.com/go-jose/go-jose/v4 v4.0.1/go.mod h1:WVf9LFMHh/QVrmqrOfqun0C45tMe3RoiKJMPvgWwLfY= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= @@ -224,7 +303,32 @@ github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= +github.com/go-playground/validator/v10 v10.22.0 h1:k6HsTZ0sTnROkhS//R0O+55JgM8C4Bx7ia+JlgcnOao= +github.com/go-playground/validator/v10 v10.22.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= +github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/gobwas/ws v1.2.1 h1:F2aeBZrm2NDsc7vbovKrWSogd4wvfAxg0FQ89/iqOTk= +github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -237,15 +341,22 @@ github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/mock v1.7.0-rc.1 h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/U= +github.com/golang/mock v1.7.0-rc.1/go.mod h1:s42URUywIqd+OcERslBJvOjepvNymP31m3q8d/GkuRs= +github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -257,6 +368,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -270,6 +383,13 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-containerregistry v0.19.1 h1:yMQ62Al6/V0Z7CqIrrS1iYoA5/oQCm88DeNujc7C1KY= github.com/google/go-containerregistry v0.19.1/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/nftables v0.1.1-0.20230115205135-9aa6fdf5a28c h1:06RMfw+TMMHtRuUOroMeatRCCgSMWXCJQeABvHU69YQ= +github.com/google/nftables v0.1.1-0.20230115205135-9aa6fdf5a28c/go.mod h1:BVIYo3cdnT4qSylnYqcd5YtmXhr51cJPGtnLBe/uLBU= +github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b h1:h9U78+dx9a4BKdQkBBos92HalKpaGKHrp+3Uo6yTodo= +github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -277,11 +397,22 @@ github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 h1:RtRsiaGvWxcwd8y3BiRZxsylPT8hLWZ5SPcfI+3IDNk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0/go.mod h1:TzP6duP4Py2pHLVPPQp42aoYI92+PCrVotyR5e8Vqlk= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-checkpoint v0.5.0 h1:MFYpPZCnQqQTE18jFwSII6eUQrD/oxMFp3mlgcqk5mU= +github.com/hashicorp/go-checkpoint v0.5.0/go.mod h1:7nfLNL10NsxqO4iWuW6tWW0HjZuDrwkBuEQsVcpCOgg= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320 h1:1/D3zfFHttUKaCaGKZ/dR2roBXv0vKbSCnssIldfQdI= +github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320/go.mod h1:EiZBMaudVLy8fmjf9Npq1dq9RalhveqZG5w/yz3mHWs= +github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= +github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= @@ -289,9 +420,13 @@ github.com/hashicorp/go-memdb v1.3.2 h1:RBKHOsnSszpU6vxq80LzC2BaQjuuvoyaQbkLTf7V github.com/hashicorp/go-memdb v1.3.2/go.mod h1:Mluclgwib3R93Hk5fxEfiRhB+6Dar64wWh71LpNSe3g= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-plugin v1.4.4 h1:NVdrSdFRt3SkZtNckJ6tog7gbpRrcbOjQi/rgF7JYWQ= +github.com/hashicorp/go-plugin v1.4.4/go.mod h1:viDMjcLJuDui6pXb8U4HVfb8AamCWhHGUjr2IrTF67s= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= @@ -300,14 +435,53 @@ github.com/hashicorp/golang-lru/arc/v2 v2.0.5 h1:l2zaLDubNhW4XO3LnliVj0GXO3+/CGN github.com/hashicorp/golang-lru/arc/v2 v2.0.5/go.mod h1:ny6zBSQZi2JxIeYcv7kt2sH2PXJtirBN7RDhRpxPkxU= github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4= github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/hc-install v0.7.0 h1:Uu9edVqjKQxxuD28mR5TikkKDd/p55S8vzPC1659aBk= +github.com/hashicorp/hc-install v0.7.0/go.mod h1:ELmmzZlGnEcqoUMKUuykHaPCIR1sYLYX+KSggWSKZuA= +github.com/hashicorp/hcl/v2 v2.21.0 h1:lve4q/o/2rqwYOgUg3y3V2YPyD1/zkCLGjIV74Jit14= +github.com/hashicorp/hcl/v2 v2.21.0/go.mod h1:62ZYHrXgPoX8xBnzl8QzbWq4dyDsDtfCRgIq1rbJEvA= +github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/terraform-exec v0.17.2 h1:EU7i3Fh7vDUI9nNRdMATCEfnm9axzTnad8zszYZ73Go= +github.com/hashicorp/terraform-exec v0.17.2/go.mod h1:tuIbsL2l4MlwwIZx9HPM+LOV9vVyEfBYu2GsO1uH3/8= +github.com/hashicorp/terraform-json v0.22.1 h1:xft84GZR0QzjPVWs4lRUwvTcPnegqlyS7orfb5Ltvec= +github.com/hashicorp/terraform-json v0.22.1/go.mod h1:JbWSQCLFSXFFhg42T7l9iJwdGXBYV8fmmD6o/ML4p3A= +github.com/hashicorp/terraform-plugin-go v0.12.0 h1:6wW9mT1dSs0Xq4LR6HXj1heQ5ovr5GxXNJwkErZzpJw= +github.com/hashicorp/terraform-plugin-go v0.12.0/go.mod h1:kwhmaWHNDvT1B3QiSJdAtrB/D4RaKSY/v3r2BuoWK4M= +github.com/hashicorp/terraform-plugin-log v0.7.0 h1:SDxJUyT8TwN4l5b5/VkiTIaQgY6R+Y2BQ0sRZftGKQs= +github.com/hashicorp/terraform-plugin-log v0.7.0/go.mod h1:p4R1jWBXRTvL4odmEkFfDdhUjHf9zcs/BCoNHAc7IK4= +github.com/hashicorp/terraform-plugin-sdk/v2 v2.20.0 h1:+KxZULPsbjpAVoP0WNj/8aVW6EqpcX5JcUcQ5wl7Da4= +github.com/hashicorp/terraform-plugin-sdk/v2 v2.20.0/go.mod h1:DwGJG3KNxIPluVk6hexvDfYR/MS/eKGpiztJoT3Bbbw= +github.com/hashicorp/terraform-registry-address v0.0.0-20220623143253-7d51757b572c h1:D8aRO6+mTqHfLsK/BC3j5OAoogv1WLRWzY1AaTo3rBg= +github.com/hashicorp/terraform-registry-address v0.0.0-20220623143253-7d51757b572c/go.mod h1:Wn3Na71knbXc1G8Lh+yu/dQWWJeFQEpDeJMtWMtlmNI= +github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734 h1:HKLsbzeOsfXmKNpr3GiT18XAblV0BjCbzL8KQAMZGa0= +github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734/go.mod h1:kNDNcF7sN4DocDLBkQYz73HGKwN1ANB1blq4lIYLYvg= +github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= +github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= +github.com/hdevalence/ed25519consensus v0.1.0 h1:jtBwzzcHuTmFrQN6xQZn6CQEO/V9f7HsjsjeEZ6auqU= +github.com/hdevalence/ed25519consensus v0.1.0/go.mod h1:w3BHWjwJbFU29IRHL1Iqkw3sus+7FctEyM4RqDxYNzo= +github.com/hinshun/vt10x v0.0.0-20220301184237-5011da428d02 h1:AgcIVYPa6XJnU3phs104wLj8l5GEththEw6+F79YsIY= +github.com/hinshun/vt10x v0.0.0-20220301184237-5011da428d02/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= +github.com/iancoleman/orderedmap v0.3.0 h1:5cbR2grmZR/DiVt+VJopEhtVs9YGInGIxAoMJn+Ichc= +github.com/iancoleman/orderedmap v0.3.0/go.mod h1:XuLcCUkdL5owUCQeF2Ue9uuw1EptkJDkXXS7VoV7XGE= +github.com/illarion/gonotify v1.0.1 h1:F1d+0Fgbq/sDWjj/r66ekjDG+IDeecQKUFH4wNwsoio= +github.com/illarion/gonotify v1.0.1/go.mod h1:zt5pmDofZpU1f8aqlK0+95eQhoEAn/d4G4B/FjVW4jE= +github.com/insomniacslk/dhcp v0.0.0-20231206064809-8c70d406f6d2 h1:9K06NfxkBh25x56yVhWWlKFE8YpicaSfHwoV8SFbueA= +github.com/insomniacslk/dhcp v0.0.0-20231206064809-8c70d406f6d2/go.mod h1:3A9PQ1cunSDF/1rbTq99Ts4pVnycWg+vlPkfeD2NLFI= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/josharian/native v1.1.1-0.20230202152459-5c7d0dd6ab86 h1:elKwZS1OcdQ0WwEDBeqxKwb7WB62QX8bvZ/FJnVXIfk= +github.com/josharian/native v1.1.1-0.20230202152459-5c7d0dd6ab86/go.mod h1:aFAMtuldEgx/4q7iSGazk22+IcgvtiC+HIimFO9XlS8= +github.com/jsimonetti/rtnetlink v1.3.5 h1:hVlNQNRlLDGZz31gBPicsG7Q53rnlsz1l1Ix/9XlpVA= +github.com/jsimonetti/rtnetlink v1.3.5/go.mod h1:0LFedyiTkebnd43tE4YAkWGIq9jQphow4CcwxaT2Y00= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= @@ -318,9 +492,14 @@ github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kortschak/wol v0.0.0-20200729010619-da482cc4850a h1:+RR6SqnTkDLWyICxS1xpjCi/3dhyV+TgZwA6Ww3KncQ= +github.com/kortschak/wol v0.0.0-20200729010619-da482cc4850a/go.mod h1:YTtCCM3ryyfiu4F7t8HQ1mxvp1UBdWM2r6Xa+nGWvDk= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -329,10 +508,17 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= @@ -340,12 +526,32 @@ github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRC github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mdlayher/genetlink v1.3.2 h1:KdrNKe+CTu+IbZnm/GVUMXSqBBLqcGpRDa0xkQy56gw= +github.com/mdlayher/genetlink v1.3.2/go.mod h1:tcC3pkCrPUGIKKsCsp0B3AdaaKuHtaxoJRz3cc+528o= +github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= +github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= +github.com/mdlayher/sdnotify v1.0.0 h1:Ma9XeLVN/l0qpyx1tNeMSeTjCPH6NtuD6/N9XdTlQ3c= +github.com/mdlayher/sdnotify v1.0.0/go.mod h1:HQUmpM4XgYkhDLtd+Uad8ZFK1T9D5+pNxnXQjCeJlGE= +github.com/mdlayher/socket v0.5.0 h1:ilICZmJcQz70vrWVes1MFera4jGiWNocSkykwwoy3XI= +github.com/mdlayher/socket v0.5.0/go.mod h1:WkcBFfvyG8QENs5+hfQPl1X6Jpd2yeLIYgrGFmJiJxI= +github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= +github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= +github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= +github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= +github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= +github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c h1:cqn374mizHuIWj+OSJCajGr/phAmuMug9qIX3l9CflE= +github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/buildkit v0.13.1 h1:L8afOFhPq2RPJJSr/VyzbufwID7jquZVB7oFHbPRcPE= github.com/moby/buildkit v0.13.1/go.mod h1:aNmNQKLBFYAOFuzQjR3VA27/FijlvtBD1pjNwTSN37k= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= @@ -372,9 +578,12 @@ github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4 github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= @@ -382,6 +591,10 @@ github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKt github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nsf/jsondiff v0.0.0-20200515183724-f29ed568f4ce h1:RPclfga2SEJmgMmz2k+Mg7cowZ8yv4Trqw9UsJby758= +github.com/nsf/jsondiff v0.0.0-20200515183724-f29ed568f4ce/go.mod h1:uFMI8w+ref4v2r9jz+c9i1IfIttS/OkmLfrk1jne5hs= +github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -392,10 +605,21 @@ github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bl github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= +github.com/outcaste-io/ristretto v0.2.3 h1:AK4zt/fJ76kjlYObOeNwh4T3asEuaCmp26pOvUOL9w0= +github.com/outcaste-io/ristretto v0.2.3/go.mod h1:W8HywhmtlopSB1jeMg3JtdIhf+DYkLAr0VN/s4+MHac= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= +github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= +github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= +github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= github.com/pion/transport/v2 v2.0.0 h1:bsMYyqHCbkvHwj+eNCFBuxtlKndKfyGI2vaQmM3fIE4= github.com/pion/transport/v2 v2.0.0/go.mod h1:HS2MEBJTwD+1ZI2eSXSvHJx/HnzQqRy2/LXxt6eVMHc= @@ -414,17 +638,17 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= -github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= -github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= +github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y= -github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= @@ -437,13 +661,19 @@ github.com/redis/go-redis/extra/redisotel/v9 v9.0.5/go.mod h1:WZjPDy7VNzn77AAfnA github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/redis/go-redis/v9 v9.1.0 h1:137FnGdk+EQdCbye1FW+qOEcY5S+SpY9T0NiuqvtfMY= github.com/redis/go-redis/v9 v9.1.0/go.mod h1:urWj3He21Dj5k4TK1y59xH8Uj6ATueP8AH1cY3lZl4c= +github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052 h1:Qp27Idfgi6ACvFQat5+VJvlYToylpM/hcyLBI3WaKPA= +github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052/go.mod h1:uvX/8buq8uVeiZiFht+0lqSLBHF+uGV8BrTv8W/SIwk= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/secure-systems-lab/go-securesystemslib v0.7.0 h1:OwvJ5jQf9LnIAS83waAjPbcMsODrTQUpJ02eNLUoxBg= +github.com/secure-systems-lab/go-securesystemslib v0.7.0/go.mod h1:/2gYnlnHVQ6xeGtfIqFy7Do03K4cdCY0A/GlJLDKLHI= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -452,6 +682,8 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skeema/knownhosts v1.2.2 h1:Iug2P4fLmDw9f41PB6thxUkNUkJzB5i+1/exaj40L3A= github.com/skeema/knownhosts v1.2.2/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -460,28 +692,92 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/swaggest/assertjson v1.9.0 h1:dKu0BfJkIxv/xe//mkCrK5yZbs79jL7OVf9Ija7o2xQ= +github.com/swaggest/assertjson v1.9.0/go.mod h1:b+ZKX2VRiUjxfUIal0HDN85W0nHPAYUbYH5WkkSsFsU= +github.com/tailscale/certstore v0.1.1-0.20220316223106-78d6e1c49d8d h1:K3j02b5j2Iw1xoggN9B2DIEkhWGheqFOeDkdJdBrJI8= +github.com/tailscale/certstore v0.1.1-0.20220316223106-78d6e1c49d8d/go.mod h1:2P+hpOwd53e7JMX/L4f3VXkv1G+33ES6IWZSrkIeWNs= +github.com/tailscale/golang-x-crypto v0.0.0-20230713185742-f0b76a10a08e h1:JyeJF/HuSwvxWtsR1c0oKX1lzaSH5Wh4aX+MgiStaGQ= +github.com/tailscale/golang-x-crypto v0.0.0-20230713185742-f0b76a10a08e/go.mod h1:DjoeCULdP6vTJ/xY+nzzR9LaUHprkbZEpNidX0aqEEk= +github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05 h1:4chzWmimtJPxRs2O36yuGRW3f9SYV+bMTTvMBI0EKio= +github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8= github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a h1:SJy1Pu0eH1C29XwJucQo73FrleVK6t4kYz4NVhp34Yw= github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a/go.mod h1:DFSS3NAGHthKo1gTlmEcSBiZrRJXi28rLNd/1udP1c8= +github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85 h1:zrsUcqrG2uQSPhaUPjUQwozcRdDdSxxqhNgNZ3drZFk= +github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85/go.mod h1:NzVQi3Mleb+qzq8VmcWpSkcSYxXIg0DkI6XDzpVkhJ0= +github.com/tailscale/wireguard-go v0.0.0-20231121184858-cc193a0b3272 h1:zwsem4CaamMdC3tFoTpzrsUSMDPV0K6rhnQdF7kXekQ= +github.com/tailscale/wireguard-go v0.0.0-20231121184858-cc193a0b3272/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4= +github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0= +github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8= +github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= +github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/u-root/uio v0.0.0-20240209044354-b3d14b93376a h1:BH1SOPEvehD2kVrndDnGJiUF0TrBpNs+iyYocu6h0og= +github.com/u-root/uio v0.0.0-20240209044354-b3d14b93376a/go.mod h1:P3a5rG4X7tI17Nn3aOIAYr5HbIMukwXG0urG0WuL8OA= +github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= +github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8= +github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM= github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts= github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk= +github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= +github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= +github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= +github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= +github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= +github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= +github.com/vmihailenco/msgpack/v4 v4.3.12 h1:07s4sz9IReOgdikxLTKNbBdqDMLsjPKXwvCazn8G65U= +github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= +github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= +github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zclconf/go-cty v1.14.4 h1:uXXczd9QDGsgu0i/QFR/hzI5NYCHLf6NQw/atrbnhq8= +github.com/zclconf/go-cty v1.14.4/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= +github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo= +github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/errs v1.3.0 h1:hmiaKqgYZzcVgRL1Vkc1Mn2914BbzB0IBxs+ebeutGs= +github.com/zeebo/errs v1.3.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= go.etcd.io/etcd/client/pkg/v3 v3.5.6/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= go.etcd.io/etcd/raft/v3 v3.5.6 h1:tOmx6Ym6rn2GpZOrvTGJZciJHek6RnC3U/zNInzIN50= go.etcd.io/etcd/raft/v3 v3.5.6/go.mod h1:wL8kkRGx1Hp8FmZUuHfL3K2/OaGIDaXGr1N7i2G07J0= +go.nhat.io/otelsql v0.13.0 h1:L6obwZRxgFQqeSvo7jCemP659fu7pqsDHQNuZ3Ev1yI= +go.nhat.io/otelsql v0.13.0/go.mod h1:HyYpqd7G9BK+9cPLydV+2JN/4J5D3wlX6+jDLTk52GE= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= @@ -490,23 +786,40 @@ go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 h1:tIqheXEFWAZ7O8A7m+J0aPTmpJN3YQ7qetUAdkkkKpk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0/go.mod h1:nUeKExfxAQVbiVFn32YXpXZZHZ61Cc3s3Rn1pDBGAb0= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 h1:digkEZCJWobwBqMwC0cwCq8/wkkRy/OowZg5OArWZrM= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0/go.mod h1:/OpE/y70qVkndM0TrxT4KBoN3RsFZP0QaofcfYrj76I= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.24.0 h1:JYE2HM7pZbOt5Jhk8ndWZTUWYOVift2cHjXVMkPdmdc= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.24.0/go.mod h1:yMb/8c6hVsnma0RpsBMNo0fEiQKeclawtgaIaOp2MLY= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.24.0 h1:s0PHtIkN+3xrbDOpt2M8OTG92cWqUESvzh2MxiR5xY8= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.24.0/go.mod h1:hZlFbDbRt++MMPCCfSJfmhkGIWnX1h3XjkfxZUjLrIA= go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= -go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= -go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= +go.opentelemetry.io/otel/sdk/metric v1.24.0 h1:yyMQrPzF+k88/DbH7o4FMAs80puqd+9osbiBrJrz/w8= +go.opentelemetry.io/otel/sdk/metric v1.24.0/go.mod h1:I6Y5FjH6rvEnTTAYQz3Mmv2kl6Ek5IIrmwTLqMrrOE0= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.1-0.20240429205332-517bace7cc29 h1:w0QrHuh0hhUZ++UTQaBM2DMdrWQghZ/UsUb+Wb1+8YE= +go.uber.org/goleak v1.3.1-0.20240429205332-517bace7cc29/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go4.org/mem v0.0.0-20220726221520-4f986261bf13 h1:CbZeCBZ0aZj8EfVgnqQcYZgf0lpZ3H9rmp5nkDTAst8= +go4.org/mem v0.0.0-20220726221520-4f986261bf13/go.mod h1:reUoABIJ9ikfM5sgtSF3Wushcza7+WeD01VB9Lirh3g= +go4.org/netipx v0.0.0-20230728180743-ad4cb58a6516 h1:X66ZEoMN2SuaoI/dfZVYobB6E5zjZyyHUMWlCA7MgGE= +go4.org/netipx v0.0.0-20230728180743-ad4cb58a6516/go.mod h1:TQvodOM+hJTioNQJilmLXu08JNb8i+ccq418+KWu1/Y= +golang.org/x/arch v0.4.0 h1:A8WCeEWhLwPBKNbFi5Wv5UTCBx5zzubnXDlMOFAzFMc= +golang.org/x/arch v0.4.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -515,9 +828,7 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -528,39 +839,43 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.19.0 h1:9+E/EZBCbTLNrbN35fHv/a/d/mOBatymz1zbtQrXpIg= -golang.org/x/oauth2 v0.19.0/go.mod h1:vYi7skDa1x015PmRRYZ7+s1cWyPgrPiSYRe4rnsexc8= +golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= +golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= @@ -572,22 +887,35 @@ golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210301091718-77cc2087c03b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.1-0.20230131160137-e7d7f63158de/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= @@ -595,20 +923,22 @@ golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -620,34 +950,42 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= +golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg= +golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= +golang.zx2c4.com/wireguard/windows v0.5.3 h1:On6j2Rpn3OEMXqBq00QEDC7bWSZrPIHKIus8eIuExIE= +golang.zx2c4.com/wireguard/windows v0.5.3/go.mod h1:9TEe8TJmtwyQebdFwAkEWOPr3prrtqm+REGFifP60hI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= -google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= -google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c h1:kaI7oewGK5YnVwj+Y+EJBO/YN1ht8iTL9XkFHtVZLsc= -google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be h1:LG9vZxsWGOmUKieR8wPAUR3u3MpnYFQZROPIMaXh7/A= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda h1:wu/KJm9KJwpfHWhkkZGohVC6KRrc1oJNr4jwtQMOQXw= +google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda/go.mod h1:g2LLCvCeCSir/JJSWosk19BR4NVxGqHUC6rxIRsd7Aw= +google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8 h1:W5Xj/70xIA4x60O/IFyXivR5MGqblAb8R3w26pnD6No= +google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8/go.mod h1:vPrPUTsDCYxXWjP7clS81mZ6/803D8K4iM9Ma27VKas= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e h1:Elxv5MwEkCI9f5SkoL6afed6NTdxaGoAo39eANBwHL8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= -google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -660,10 +998,14 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/DataDog/dd-trace-go.v1 v1.64.0 h1:zXQo6iv+dKRrDBxMXjRXLSKN2lY9uM34XFI4nPyp0eA= +gopkg.in/DataDog/dd-trace-go.v1 v1.64.0/go.mod h1:qzwVu8Qr8CqzQNw2oKEXRdD+fMnjYatjYMGE0tdCVG4= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -678,7 +1020,23 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= +gvisor.dev/gvisor v0.0.0-20240509041132-65b30f7869dc h1:DXLLFYv/k/xr0rWcwVEvWme1GR36Oc4kNMspg38JeiE= +gvisor.dev/gvisor v0.0.0-20240509041132-65b30f7869dc/go.mod h1:sxc3Uvk/vHcd3tj7/DHVBoR5wvWT/MmRq2pj7HRJnwU= +honnef.co/go/gotraceui v0.2.0 h1:dmNsfQ9Vl3GwbiVD7Z8d/osC6WtGGrasyrC2suc4ZIQ= +honnef.co/go/gotraceui v0.2.0/go.mod h1:qHo4/W75cA3bX0QQoSvDjbJa4R8mAyyFjbWAj63XElc= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM= +howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= +inet.af/peercred v0.0.0-20210906144145-0893ea02156a h1:qdkS8Q5/i10xU2ArJMKYhVa1DORzBfYS/qA2UK2jheg= +inet.af/peercred v0.0.0-20210906144145-0893ea02156a/go.mod h1:FjawnflS/udxX+SvpsMgZfdqx2aykOlkISeAsADi5IU= +nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= +nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= +software.sslmate.com/src/go-pkcs12 v0.2.0 h1:nlFkj7bTysH6VkC4fGphtjXRbezREPgrHuJG20hBGPE= +software.sslmate.com/src/go-pkcs12 v0.2.0/go.mod h1:23rNcYsMabIc1otwLpTkCCPwUq6kQsTyowttG/as0kQ= +storj.io/drpc v0.0.33 h1:yCGZ26r66ZdMP0IcTYsj7WDAUIIjzXk6DJhbhvt9FHI= +storj.io/drpc v0.0.33/go.mod h1:vR804UNzhBa49NOJ6HeLjd2H3MakC1j5Gv8bsOQT6N4= diff --git a/internal/ebutil/remount.go b/internal/ebutil/remount.go index 0ae7b135..21de0a3a 100644 --- a/internal/ebutil/remount.go +++ b/internal/ebutil/remount.go @@ -9,7 +9,7 @@ import ( "sync" "syscall" - "github.com/coder/envbuilder/internal/notcodersdk" + "github.com/coder/envbuilder/internal/log" "github.com/hashicorp/go-multierror" "github.com/prometheus/procfs" ) @@ -34,12 +34,12 @@ import ( // to restore the original mount points. If an error is encountered while attempting to perform // the operation, calling the returned function will make a best-effort attempt to restore // the original state. -func TempRemount(logf func(notcodersdk.LogLevel, string, ...any), dest string, ignorePrefixes ...string) (restore func() error, err error, +func TempRemount(logf log.Func, dest string, ignorePrefixes ...string) (restore func() error, err error, ) { return tempRemount(&realMounter{}, logf, dest, ignorePrefixes...) } -func tempRemount(m mounter, logf func(notcodersdk.LogLevel, string, ...any), base string, ignorePrefixes ...string) (restore func() error, err error) { +func tempRemount(m mounter, logf log.Func, base string, ignorePrefixes ...string) (restore func() error, err error) { mountInfos, err := m.GetMounts() if err != nil { return func() error { return nil }, fmt.Errorf("get mounts: %w", err) @@ -73,7 +73,7 @@ func tempRemount(m mounter, logf func(notcodersdk.LogLevel, string, ...any), bas } for orig, moved := range mounts { - logf(notcodersdk.LogLevelTrace, "restore mount %s", orig) + logf(log.LevelDebug, "restore mount %s", orig) if err := remount(m, moved, orig, newLibDir, libsSymlinks); err != nil { merr = multierror.Append(merr, fmt.Errorf("restore mount: %w", err)) } @@ -86,20 +86,20 @@ outer: for _, mountInfo := range mountInfos { // TODO: do this for all mounts if _, ok := mountInfo.Options["ro"]; !ok { - logf(notcodersdk.LogLevelTrace, "skip rw mount %s", mountInfo.MountPoint) + logf(log.LevelDebug, "skip rw mount %s", mountInfo.MountPoint) continue } for _, prefix := range ignorePrefixes { if strings.HasPrefix(mountInfo.MountPoint, prefix) { - logf(notcodersdk.LogLevelTrace, "skip mount %s under ignored prefix %s", mountInfo.MountPoint, prefix) + logf(log.LevelDebug, "skip mount %s under ignored prefix %s", mountInfo.MountPoint, prefix) continue outer } } src := mountInfo.MountPoint dest := filepath.Join(base, src) - logf(notcodersdk.LogLevelTrace, "temp remount %s", src) + logf(log.LevelDebug, "temp remount %s", src) if err := remount(m, src, dest, libDir, libsSymlinks); err != nil { return restore, fmt.Errorf("temp remount: %w", err) } diff --git a/internal/ebutil/remount_internal_test.go b/internal/ebutil/remount_internal_test.go index fe44728e..f6b68170 100644 --- a/internal/ebutil/remount_internal_test.go +++ b/internal/ebutil/remount_internal_test.go @@ -8,7 +8,7 @@ import ( "testing" time "time" - "github.com/coder/envbuilder/internal/notcodersdk" + "github.com/coder/envbuilder/internal/log" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -653,9 +653,9 @@ func fakeMounts(mounts ...string) []*procfs.MountInfo { return m } -func fakeLog(t *testing.T) func(notcodersdk.LogLevel, string, ...any) { +func fakeLog(t *testing.T) func(log.Level, string, ...any) { t.Helper() - return func(_ notcodersdk.LogLevel, s string, a ...any) { + return func(_ log.Level, s string, a ...any) { t.Logf(s, a...) } } diff --git a/internal/log/coder.go b/internal/log/coder.go new file mode 100644 index 00000000..38e9373e --- /dev/null +++ b/internal/log/coder.go @@ -0,0 +1,156 @@ +package log + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "time" + + "cdr.dev/slog" + "cdr.dev/slog/sloggers/sloghuman" + "github.com/coder/coder/v2/agent/proto" + "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/codersdk/agentsdk" + "github.com/coder/retry" + "github.com/google/uuid" + "golang.org/x/mod/semver" +) + +var ( + rpcConnectTimeout = 10 * time.Second + logSendGracePeriod = 10 * time.Second + minAgentAPIV2 = "v2.9" +) + +// Coder establishes a connection to the Coder instance located at +// coderURL and authenticates using token. It then establishes a +// dRPC connection to the Agent API and begins sending logs. +// If the version of Coder does not support the Agent API, it will +// fall back to using the PatchLogs endpoint. +// The returned function is used to block until all logs are sent. +func Coder(ctx context.Context, coderURL *url.URL, token string) (Func, func(), error) { + // To troubleshoot issues, we need some way of logging. + metaLogger := slog.Make(sloghuman.Sink(os.Stderr)) + defer metaLogger.Sync() + client := initClient(coderURL, token) + bi, err := client.SDK.BuildInfo(ctx) + if err != nil { + return nil, nil, fmt.Errorf("get coder build version: %w", err) + } + if semver.Compare(semver.MajorMinor(bi.Version), minAgentAPIV2) < 0 { + metaLogger.Warn(ctx, "Detected Coder version incompatible with AgentAPI v2, falling back to deprecated API", slog.F("coder_version", bi.Version)) + sendLogs, flushLogs := sendLogsV1(ctx, client, metaLogger.Named("send_logs_v1")) + return sendLogs, flushLogs, nil + } + dac, err := initRPC(ctx, client, metaLogger.Named("init_rpc")) + if err != nil { + // Logged externally + return nil, nil, fmt.Errorf("init coder rpc client: %w", err) + } + ls := agentsdk.NewLogSender(metaLogger.Named("coder_log_sender")) + metaLogger.Warn(ctx, "Sending logs via AgentAPI v2", slog.F("coder_version", bi.Version)) + sendLogs, doneFunc := sendLogsV2(ctx, dac, ls, metaLogger.Named("send_logs_v2")) + return sendLogs, doneFunc, nil +} + +type coderLogSender interface { + Enqueue(uuid.UUID, ...agentsdk.Log) + SendLoop(context.Context, agentsdk.LogDest) error + Flush(uuid.UUID) + WaitUntilEmpty(context.Context) error +} + +func initClient(coderURL *url.URL, token string) *agentsdk.Client { + client := agentsdk.New(coderURL) + client.SetSessionToken(token) + return client +} + +func initRPC(ctx context.Context, client *agentsdk.Client, l slog.Logger) (proto.DRPCAgentClient20, error) { + var c proto.DRPCAgentClient20 + var err error + retryCtx, retryCancel := context.WithTimeout(context.Background(), rpcConnectTimeout) + defer retryCancel() + attempts := 0 + for r := retry.New(100*time.Millisecond, time.Second); r.Wait(retryCtx); { + attempts++ + // Maximize compatibility. + c, err = client.ConnectRPC20(ctx) + if err != nil { + l.Debug(ctx, "Failed to connect to Coder", slog.F("error", err), slog.F("attempt", attempts)) + continue + } + break + } + if c == nil { + return nil, err + } + return proto.NewDRPCAgentClient(c.DRPCConn()), nil +} + +// sendLogsV1 uses the PatchLogs endpoint to send logs. +// This is deprecated, but required for backward compatibility with older versions of Coder. +func sendLogsV1(ctx context.Context, client *agentsdk.Client, l slog.Logger) (Func, func()) { + // nolint: staticcheck // required for backwards compatibility + sendLogs, flushLogs := agentsdk.LogsSender(agentsdk.ExternalLogSourceID, client.PatchLogs, slog.Logger{}) + return func(lvl Level, msg string, args ...any) { + log := agentsdk.Log{ + CreatedAt: time.Now(), + Output: fmt.Sprintf(msg, args...), + Level: codersdk.LogLevel(lvl), + } + if err := sendLogs(ctx, log); err != nil { + l.Warn(ctx, "failed to send logs to Coder", slog.Error(err)) + } + }, func() { + if err := flushLogs(ctx); err != nil { + l.Warn(ctx, "failed to flush logs", slog.Error(err)) + } + } +} + +// sendLogsV2 uses the v2 agent API to send logs. Only compatibile with coder versions >= 2.9. +func sendLogsV2(ctx context.Context, dest agentsdk.LogDest, ls coderLogSender, l slog.Logger) (Func, func()) { + done := make(chan struct{}) + uid := uuid.New() + go func() { + defer close(done) + if err := ls.SendLoop(ctx, dest); err != nil { + if !errors.Is(err, context.Canceled) { + l.Warn(ctx, "failed to send logs to Coder", slog.Error(err)) + } + } + + // Wait for up to 10 seconds for logs to finish sending. + sendCtx, sendCancel := context.WithTimeout(context.Background(), logSendGracePeriod) + defer sendCancel() + // Try once more to send any pending logs + if err := ls.SendLoop(sendCtx, dest); err != nil { + if !errors.Is(err, context.DeadlineExceeded) { + l.Warn(ctx, "failed to send remaining logs to Coder", slog.Error(err)) + } + } + ls.Flush(uid) + if err := ls.WaitUntilEmpty(sendCtx); err != nil { + if !errors.Is(err, context.DeadlineExceeded) { + l.Warn(ctx, "log sender did not empty", slog.Error(err)) + } + } + }() + + logFunc := func(l Level, msg string, args ...any) { + ls.Enqueue(uid, agentsdk.Log{ + CreatedAt: time.Now(), + Output: fmt.Sprintf(msg, args...), + Level: codersdk.LogLevel(l), + }) + } + + doneFunc := func() { + <-done + } + + return logFunc, doneFunc +} diff --git a/internal/log/coder_internal_test.go b/internal/log/coder_internal_test.go new file mode 100644 index 00000000..22b6f249 --- /dev/null +++ b/internal/log/coder_internal_test.go @@ -0,0 +1,184 @@ +package log + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "sync" + "testing" + "time" + + "cdr.dev/slog/sloggers/slogtest" + "github.com/coder/coder/v2/agent/proto" + "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/codersdk/agentsdk" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCoder(t *testing.T) { + t.Parallel() + + t.Run("V1/OK", func(t *testing.T) { + t.Parallel() + + token := uuid.NewString() + gotLogs := make(chan struct{}) + var closeOnce sync.Once + handler := func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v2/buildinfo" { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"version": "v2.8.9"}`)) + return + } + defer closeOnce.Do(func() { close(gotLogs) }) + tokHdr := r.Header.Get(codersdk.SessionTokenHeader) + assert.Equal(t, token, tokHdr) + var req agentsdk.PatchLogs + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if assert.Len(t, req.Logs, 1) { + assert.Equal(t, "hello world", req.Logs[0].Output) + assert.Equal(t, codersdk.LogLevelInfo, req.Logs[0].Level) + } + } + srv := httptest.NewServer(http.HandlerFunc(handler)) + defer srv.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + u, err := url.Parse(srv.URL) + require.NoError(t, err) + log, closeLog, err := Coder(ctx, u, token) + require.NoError(t, err) + defer closeLog() + log(LevelInfo, "hello %s", "world") + <-gotLogs + }) + + t.Run("V1/ErrUnauthorized", func(t *testing.T) { + t.Parallel() + + token := uuid.NewString() + authFailed := make(chan struct{}) + var closeOnce sync.Once + handler := func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v2/buildinfo" { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"version": "v2.8.9"}`)) + return + } + defer closeOnce.Do(func() { close(authFailed) }) + w.WriteHeader(http.StatusUnauthorized) + } + srv := httptest.NewServer(http.HandlerFunc(handler)) + defer srv.Close() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + u, err := url.Parse(srv.URL) + require.NoError(t, err) + log, _, err := Coder(ctx, u, token) + require.NoError(t, err) + // defer closeLog() + log(LevelInfo, "hello %s", "world") + <-authFailed + }) + + t.Run("V1/ErrNotCoder", func(t *testing.T) { + t.Parallel() + + token := uuid.NewString() + handlerCalled := make(chan struct{}) + var closeOnce sync.Once + handler := func(w http.ResponseWriter, r *http.Request) { + defer closeOnce.Do(func() { close(handlerCalled) }) + _, _ = fmt.Fprintf(w, `hello world`) + } + srv := httptest.NewServer(http.HandlerFunc(handler)) + defer srv.Close() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + u, err := url.Parse(srv.URL) + require.NoError(t, err) + _, _, err = Coder(ctx, u, token) + require.ErrorContains(t, err, "get coder build version") + require.ErrorContains(t, err, "unexpected non-JSON response") + <-handlerCalled + }) + + // In this test, we just fake out the DRPC server. + t.Run("V2/OK", func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ld := &fakeLogDest{t: t} + ls := agentsdk.NewLogSender(slogtest.Make(t, nil)) + logFunc, logsDone := sendLogsV2(ctx, ld, ls, slogtest.Make(t, nil)) + defer logsDone() + + // Send some logs + for i := 0; i < 10; i++ { + logFunc(LevelInfo, "info log %d", i+1) + } + + // Cancel and wait for flush + cancel() + t.Logf("cancelled") + logsDone() + + require.Len(t, ld.logs, 10) + }) + + // In this test, we just stand up an endpoint that does not + // do dRPC. We'll try to connect, fail to websocket upgrade + // and eventually give up. + t.Run("V2/Err", func(t *testing.T) { + t.Parallel() + + token := uuid.NewString() + handlerDone := make(chan struct{}) + var closeOnce sync.Once + handler := func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v2/buildinfo" { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"version": "v2.9.0"}`)) + return + } + defer closeOnce.Do(func() { close(handlerDone) }) + w.WriteHeader(http.StatusOK) + } + srv := httptest.NewServer(http.HandlerFunc(handler)) + defer srv.Close() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + u, err := url.Parse(srv.URL) + require.NoError(t, err) + _, _, err = Coder(ctx, u, token) + require.ErrorContains(t, err, "failed to WebSocket dial") + require.ErrorIs(t, err, context.DeadlineExceeded) + <-handlerDone + }) +} + +type fakeLogDest struct { + t testing.TB + logs []*proto.Log +} + +func (d *fakeLogDest) BatchCreateLogs(ctx context.Context, request *proto.BatchCreateLogsRequest) (*proto.BatchCreateLogsResponse, error) { + d.t.Logf("got %d logs, ", len(request.Logs)) + d.logs = append(d.logs, request.Logs...) + return &proto.BatchCreateLogsResponse{}, nil +} diff --git a/internal/log/log.go b/internal/log/log.go new file mode 100644 index 00000000..da308266 --- /dev/null +++ b/internal/log/log.go @@ -0,0 +1,47 @@ +package log + +import ( + "fmt" + "io" + "strings" + + "github.com/coder/coder/v2/codersdk" +) + +type Func func(l Level, msg string, args ...any) + +type Level string + +// Below constants are the same as their codersdk equivalents. +const ( + LevelTrace = Level(codersdk.LogLevelTrace) + LevelDebug = Level(codersdk.LogLevelDebug) + LevelInfo = Level(codersdk.LogLevelInfo) + LevelWarn = Level(codersdk.LogLevelWarn) + LevelError = Level(codersdk.LogLevelError) +) + +// New logs to the provided io.Writer. +func New(w io.Writer, verbose bool) Func { + return func(l Level, msg string, args ...any) { + if !verbose { + switch l { + case LevelDebug, LevelTrace: + return + } + } + _, _ = fmt.Fprintf(w, msg, args...) + if !strings.HasSuffix(msg, "\n") { + _, _ = fmt.Fprintf(w, "\n") + } + } +} + +// Wrap wraps the provided LogFuncs into a single Func. +func Wrap(fs ...Func) Func { + return func(l Level, msg string, args ...any) { + for _, f := range fs { + f(l, msg, args...) + } + } +} diff --git a/internal/log/log_test.go b/internal/log/log_test.go new file mode 100644 index 00000000..acf6247c --- /dev/null +++ b/internal/log/log_test.go @@ -0,0 +1,29 @@ +package log_test + +import ( + "strings" + "testing" + + "github.com/coder/envbuilder/internal/log" + "github.com/stretchr/testify/require" +) + +func Test_Verbose(t *testing.T) { + t.Parallel() + + t.Run("true", func(t *testing.T) { + var sb strings.Builder + l := log.New(&sb, true) + l(log.LevelDebug, "hello") + l(log.LevelInfo, "world") + require.Equal(t, "hello\nworld\n", sb.String()) + }) + + t.Run("false", func(t *testing.T) { + var sb strings.Builder + l := log.New(&sb, false) + l(log.LevelDebug, "hello") + l(log.LevelInfo, "world") + require.Equal(t, "world\n", sb.String()) + }) +} diff --git a/internal/notcodersdk/agentclient.go b/internal/notcodersdk/agentclient.go deleted file mode 100644 index e65bc4cc..00000000 --- a/internal/notcodersdk/agentclient.go +++ /dev/null @@ -1,430 +0,0 @@ -package notcodersdk - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "mime" - "net/http" - "net/http/httputil" - "net/url" - "strings" - "sync" - "time" - - "github.com/google/uuid" - "golang.org/x/xerrors" -) - -const ( - SessionTokenHeader = "Coder-Session-Token" -) - -type AgentSubsystem string - -const ( - AgentSubsystemEnvbuilder AgentSubsystem = "envbuilder" -) - -// ExternalLogSourceID is the statically-defined ID of a log-source that -// appears as "External" in the dashboard. -// -// This is to support legacy API-consumers that do not create their own -// log-source. This should be removed in the future. -var ExternalLogSourceID = uuid.MustParse("3b579bf4-1ed8-4b99-87a8-e9a1e3410410") - -type LogLevel string - -const ( - LogLevelTrace LogLevel = "trace" - LogLevelDebug LogLevel = "debug" - LogLevelInfo LogLevel = "info" - LogLevelWarn LogLevel = "warn" - LogLevelError LogLevel = "error" -) - -type Log struct { - CreatedAt time.Time `json:"created_at"` - Output string `json:"output"` - Level LogLevel `json:"level"` -} - -type PatchLogs struct { - LogSourceID uuid.UUID `json:"log_source_id"` - Logs []Log `json:"logs"` -} - -// New returns a client that is used to interact with the -// Coder API from a workspace agent. -func New(serverURL *url.URL) *Client { - return &Client{ - URL: serverURL, - HTTPClient: &http.Client{}, - } -} - -// Client wraps `notcodersdk.Client` with specific functions -// scoped to a workspace agent. -type Client struct { - // mu protects the fields sessionToken, logger, and logBodies. These - // need to be safe for concurrent access. - mu sync.RWMutex - sessionToken string - logBodies bool - - HTTPClient *http.Client - URL *url.URL - - // SessionTokenHeader is an optional custom header to use for setting tokens. By - // default 'Coder-Session-Token' is used. - SessionTokenHeader string - - // PlainLogger may be set to log HTTP traffic in a human-readable form. - // It uses the LogBodies option. - PlainLogger io.Writer -} - -// SessionToken returns the currently set token for the client. -func (c *Client) SessionToken() string { - c.mu.RLock() - defer c.mu.RUnlock() - return c.sessionToken -} - -// SetSessionToken returns the currently set token for the client. -func (c *Client) SetSessionToken(token string) { - c.mu.Lock() - defer c.mu.Unlock() - c.sessionToken = token -} - -// PatchLogs writes log messages to the agent startup script. -// Log messages are limited to 1MB in total. -// -// Deprecated: use the DRPCAgentClient.BatchCreateLogs instead -func (c *Client) PatchLogs(ctx context.Context, req PatchLogs) error { - res, err := c.Request(ctx, http.MethodPatch, "/api/v2/workspaceagents/me/logs", req) - if err != nil { - return err - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return ReadBodyAsError(res) - } - return nil -} - -// RequestOption is a function that can be used to modify an http.Request. -type RequestOption func(*http.Request) - -// Request performs a HTTP request with the body provided. The caller is -// responsible for closing the response body. -func (c *Client) Request(ctx context.Context, method, path string, body interface{}, opts ...RequestOption) (*http.Response, error) { - serverURL, err := c.URL.Parse(path) - if err != nil { - return nil, xerrors.Errorf("parse url: %w", err) - } - - var r io.Reader - if body != nil { - switch data := body.(type) { - case io.Reader: - r = data - case []byte: - r = bytes.NewReader(data) - default: - // Assume JSON in all other cases. - buf := bytes.NewBuffer(nil) - enc := json.NewEncoder(buf) - enc.SetEscapeHTML(false) - err = enc.Encode(body) - if err != nil { - return nil, xerrors.Errorf("encode body: %w", err) - } - r = buf - } - } - - // Copy the request body so we can log it. - var reqBody []byte - c.mu.RLock() - logBodies := c.logBodies - c.mu.RUnlock() - if r != nil && logBodies { - reqBody, err = io.ReadAll(r) - if err != nil { - return nil, xerrors.Errorf("read request body: %w", err) - } - r = bytes.NewReader(reqBody) - } - - req, err := http.NewRequestWithContext(ctx, method, serverURL.String(), r) - if err != nil { - return nil, xerrors.Errorf("create request: %w", err) - } - - tokenHeader := c.SessionTokenHeader - if tokenHeader == "" { - tokenHeader = SessionTokenHeader - } - req.Header.Set(tokenHeader, c.SessionToken()) - - if r != nil { - req.Header.Set("Content-Type", "application/json") - } - for _, opt := range opts { - opt(req) - } - - resp, err := c.HTTPClient.Do(req) - - // We log after sending the request because the HTTP Transport may modify - // the request within Do, e.g. by adding headers. - if resp != nil && c.PlainLogger != nil { - out, err := httputil.DumpRequest(resp.Request, logBodies) - if err != nil { - return nil, xerrors.Errorf("dump request: %w", err) - } - out = prefixLines([]byte("http --> "), out) - _, _ = c.PlainLogger.Write(out) - } - - if err != nil { - return nil, err - } - - if c.PlainLogger != nil { - out, err := httputil.DumpResponse(resp, logBodies) - if err != nil { - return nil, xerrors.Errorf("dump response: %w", err) - } - out = prefixLines([]byte("http <-- "), out) - _, _ = c.PlainLogger.Write(out) - } - - // Copy the response body so we can log it if it's a loggable mime type. - var respBody []byte - if resp.Body != nil && logBodies { - mimeType := parseMimeType(resp.Header.Get("Content-Type")) - if _, ok := loggableMimeTypes[mimeType]; ok { - respBody, err = io.ReadAll(resp.Body) - if err != nil { - return nil, xerrors.Errorf("copy response body for logs: %w", err) - } - err = resp.Body.Close() - if err != nil { - return nil, xerrors.Errorf("close response body: %w", err) - } - resp.Body = io.NopCloser(bytes.NewReader(respBody)) - } - } - - return resp, err -} - -func parseMimeType(contentType string) string { - mimeType, _, err := mime.ParseMediaType(contentType) - if err != nil { - mimeType = strings.TrimSpace(strings.Split(contentType, ";")[0]) - } - - return mimeType -} - -// loggableMimeTypes is a list of MIME types that are safe to log -// the output of. This is useful for debugging or testing. -var loggableMimeTypes = map[string]struct{}{ - "application/json": {}, - "text/plain": {}, - // lots of webserver error pages are HTML - "text/html": {}, -} - -func prefixLines(prefix, s []byte) []byte { - ss := bytes.NewBuffer(make([]byte, 0, len(s)*2)) - for _, line := range bytes.Split(s, []byte("\n")) { - _, _ = ss.Write(prefix) - _, _ = ss.Write(line) - _ = ss.WriteByte('\n') - } - return ss.Bytes() -} - -// ReadBodyAsError reads the response as a codersdk.Response, and -// wraps it in a codersdk.Error type for easy marshaling. -// -// This will always return an error, so only call it if the response failed -// your expectations. Usually via status code checking. -// nolint:staticcheck -func ReadBodyAsError(res *http.Response) error { - if res == nil { - return xerrors.Errorf("no body returned") - } - defer res.Body.Close() - - var requestMethod, requestURL string - if res.Request != nil { - requestMethod = res.Request.Method - if res.Request.URL != nil { - requestURL = res.Request.URL.String() - } - } - - var helpMessage string - if res.StatusCode == http.StatusUnauthorized { - // 401 means the user is not logged in - // 403 would mean that the user is not authorized - helpMessage = "Try logging in using 'coder login'." - } - - resp, err := io.ReadAll(res.Body) - if err != nil { - return xerrors.Errorf("read body: %w", err) - } - - if mimeErr := ExpectJSONMime(res); mimeErr != nil { - if len(resp) > 2048 { - resp = append(resp[:2048], []byte("...")...) - } - if len(resp) == 0 { - resp = []byte("no response body") - } - return &Error{ - statusCode: res.StatusCode, - method: requestMethod, - url: requestURL, - Response: Response{ - Message: mimeErr.Error(), - Detail: string(resp), - }, - Helper: helpMessage, - } - } - - var m Response - err = json.NewDecoder(bytes.NewBuffer(resp)).Decode(&m) - if err != nil { - if errors.Is(err, io.EOF) { - return &Error{ - statusCode: res.StatusCode, - Response: Response{ - Message: "empty response body", - }, - Helper: helpMessage, - } - } - return xerrors.Errorf("decode body: %w", err) - } - if m.Message == "" { - if len(resp) > 1024 { - resp = append(resp[:1024], []byte("...")...) - } - m.Message = fmt.Sprintf("unexpected status code %d, response has no message", res.StatusCode) - m.Detail = string(resp) - } - - return &Error{ - Response: m, - statusCode: res.StatusCode, - method: requestMethod, - url: requestURL, - Helper: helpMessage, - } -} - -// Response represents a generic HTTP response. -type Response struct { - // Message is an actionable message that depicts actions the request took. - // These messages should be fully formed sentences with proper punctuation. - // Examples: - // - "A user has been created." - // - "Failed to create a user." - Message string `json:"message"` - // Detail is a debug message that provides further insight into why the - // action failed. This information can be technical and a regular golang - // err.Error() text. - // - "database: too many open connections" - // - "stat: too many open files" - Detail string `json:"detail,omitempty"` - // Validations are form field-specific friendly error messages. They will be - // shown on a form field in the UI. These can also be used to add additional - // context if there is a set of errors in the primary 'Message'. - Validations []ValidationError `json:"validations,omitempty"` -} - -// ValidationError represents a scoped error to a user input. -type ValidationError struct { - Field string `json:"field" validate:"required"` - Detail string `json:"detail" validate:"required"` -} - -func (e ValidationError) Error() string { - return fmt.Sprintf("field: %s detail: %s", e.Field, e.Detail) -} - -var _ error = (*ValidationError)(nil) - -// Error represents an unaccepted or invalid request to the API. -// @typescript-ignore Error -type Error struct { - Response - - statusCode int - method string - url string - - Helper string -} - -func (e *Error) StatusCode() int { - return e.statusCode -} - -func (e *Error) Method() string { - return e.method -} - -func (e *Error) URL() string { - return e.url -} - -func (e *Error) Friendly() string { - var sb strings.Builder - _, _ = fmt.Fprintf(&sb, "%s. %s", strings.TrimSuffix(e.Message, "."), e.Helper) - for _, err := range e.Validations { - _, _ = fmt.Fprintf(&sb, "\n- %s: %s", err.Field, err.Detail) - } - return sb.String() -} - -func (e *Error) Error() string { - var builder strings.Builder - if e.method != "" && e.url != "" { - _, _ = fmt.Fprintf(&builder, "%v %v: ", e.method, e.url) - } - _, _ = fmt.Fprintf(&builder, "unexpected status code %d: %s", e.statusCode, e.Message) - if e.Helper != "" { - _, _ = fmt.Fprintf(&builder, ": %s", e.Helper) - } - if e.Detail != "" { - _, _ = fmt.Fprintf(&builder, "\n\tError: %s", e.Detail) - } - for _, err := range e.Validations { - _, _ = fmt.Fprintf(&builder, "\n\t%s: %s", err.Field, err.Detail) - } - return builder.String() -} - -// ExpectJSONMime is a helper function that will assert the content type -// of the response is application/json. -func ExpectJSONMime(res *http.Response) error { - contentType := res.Header.Get("Content-Type") - mimeType := parseMimeType(contentType) - if mimeType != "application/json" { - return xerrors.Errorf("unexpected non-JSON response %q", contentType) - } - return nil -} diff --git a/internal/notcodersdk/doc.go b/internal/notcodersdk/doc.go deleted file mode 100644 index cfa92db6..00000000 --- a/internal/notcodersdk/doc.go +++ /dev/null @@ -1,13 +0,0 @@ -// Package notcodersdk contains manually-vendored code from -// github.com/coder/coder/v2/codersdk. -// -// This code is currently required for sending workspace build logs to -// coder. It was manually vendored to avoid dependency issues. -// -// If the direct integration is moved outside of envbuilder, -// this package can safely be removed. -// See the below issues for context: -// - https://github.com/coder/envbuilder/issues/178 -// - https://github.com/coder/coder/issues/11342 -// - https://github.com/coder/envbuilder/issues/193 -package notcodersdk diff --git a/internal/notcodersdk/logs.go b/internal/notcodersdk/logs.go deleted file mode 100644 index 6ca4aca8..00000000 --- a/internal/notcodersdk/logs.go +++ /dev/null @@ -1,169 +0,0 @@ -package notcodersdk - -import ( - "context" - "errors" - "net/http" - "time" - - "github.com/google/uuid" - "golang.org/x/xerrors" - - "cdr.dev/slog" - "github.com/coder/retry" -) - -type logsSenderOptions struct { - flushTimeout time.Duration -} - -// LogsSender will send agent startup logs to the server. Calls to -// sendLog are non-blocking and will return an error if flushAndClose -// has been called. Calling sendLog concurrently is not supported. If -// the context passed to flushAndClose is canceled, any remaining logs -// will be discarded. -// -// Deprecated: Use NewLogSender instead, based on the v2 Agent API. -func LogsSender(sourceID uuid.UUID, patchLogs func(ctx context.Context, req PatchLogs) error, logger slog.Logger, opts ...func(*logsSenderOptions)) (sendLog func(ctx context.Context, log ...Log) error, flushAndClose func(context.Context) error) { - o := logsSenderOptions{ - flushTimeout: 250 * time.Millisecond, - } - for _, opt := range opts { - opt(&o) - } - - // The main context is used to close the sender goroutine and cancel - // any outbound requests to the API. The shutdown context is used to - // signal the sender goroutine to flush logs and then exit. - ctx, cancel := context.WithCancel(context.Background()) - shutdownCtx, shutdown := context.WithCancel(ctx) - - // Synchronous sender, there can only be one outbound send at a time. - sendDone := make(chan struct{}) - send := make(chan []Log, 1) - go func() { - // Set flushTimeout and backlogLimit so that logs are uploaded - // once every 250ms or when 100 logs have been added to the - // backlog, whichever comes first. - backlogLimit := 100 - - flush := time.NewTicker(o.flushTimeout) - - var backlog []Log - defer func() { - flush.Stop() - if len(backlog) > 0 { - logger.Warn(ctx, "startup logs sender exiting early, discarding logs", slog.F("discarded_logs_count", len(backlog))) - } - logger.Debug(ctx, "startup logs sender exited") - close(sendDone) - }() - - done := false - for { - flushed := false - select { - case <-ctx.Done(): - return - case <-shutdownCtx.Done(): - done = true - - // Check queued logs before flushing. - select { - case logs := <-send: - backlog = append(backlog, logs...) - default: - } - case <-flush.C: - flushed = true - case logs := <-send: - backlog = append(backlog, logs...) - flushed = len(backlog) >= backlogLimit - } - - if (done || flushed) && len(backlog) > 0 { - flush.Stop() // Lower the chance of a double flush. - - // Retry uploading logs until successful or a specific - // error occurs. Note that we use the main context here, - // meaning these requests won't be interrupted by - // shutdown. - var err error - for r := retry.New(time.Second, 5*time.Second); r.Wait(ctx); { - err = patchLogs(ctx, PatchLogs{ - Logs: backlog, - LogSourceID: sourceID, - }) - if err == nil { - break - } - - if errors.Is(err, context.Canceled) { - break - } - // This error is expected to be codersdk.Error, but it has - // private fields so we can't fake it in tests. - var statusErr interface{ StatusCode() int } - if errors.As(err, &statusErr) { - if statusErr.StatusCode() == http.StatusRequestEntityTooLarge { - logger.Warn(ctx, "startup logs too large, discarding logs", slog.F("discarded_logs_count", len(backlog)), slog.Error(err)) - err = nil - break - } - } - logger.Error(ctx, "startup logs sender failed to upload logs, retrying later", slog.F("logs_count", len(backlog)), slog.Error(err)) - } - if err != nil { - return - } - backlog = nil - - // Anchor flush to the last log upload. - flush.Reset(o.flushTimeout) - } - if done { - return - } - } - }() - - var queue []Log - sendLog = func(callCtx context.Context, log ...Log) error { - select { - case <-shutdownCtx.Done(): - return xerrors.Errorf("closed: %w", shutdownCtx.Err()) - case <-callCtx.Done(): - return callCtx.Err() - case queue = <-send: - // Recheck to give priority to context cancellation. - select { - case <-shutdownCtx.Done(): - return xerrors.Errorf("closed: %w", shutdownCtx.Err()) - case <-callCtx.Done(): - return callCtx.Err() - default: - } - // Queue has not been captured by sender yet, re-use. - default: - } - - queue = append(queue, log...) - send <- queue // Non-blocking. - queue = nil - - return nil - } - flushAndClose = func(callCtx context.Context) error { - defer cancel() - shutdown() - select { - case <-sendDone: - return nil - case <-callCtx.Done(): - cancel() - <-sendDone - return callCtx.Err() - } - } - return sendLog, flushAndClose -} diff --git a/options.go b/options.go index 2913fdea..76eddb60 100644 --- a/options.go +++ b/options.go @@ -3,13 +3,11 @@ package envbuilder import ( "strings" - "github.com/coder/envbuilder/internal/notcodersdk" + "github.com/coder/envbuilder/internal/log" "github.com/coder/serpent" "github.com/go-git/go-billy/v5" ) -type LoggerFunc func(level notcodersdk.LogLevel, format string, args ...interface{}) - // Options contains the configuration for the envbuilder. type Options struct { // SetupScript is the script to run before the init script. It runs as the @@ -125,7 +123,9 @@ type Options struct { // execute it after successful startup. PostStartScriptPath string // Logger is the logger to use for all operations. - Logger LoggerFunc + Logger log.Func + // Verbose controls whether to send verbose logs. + Verbose bool // Filesystem is the filesystem to use for all operations. Defaults to the // host filesystem. Filesystem billy.Filesystem @@ -416,6 +416,12 @@ func (o *Options) CLI() serpent.OptionSet { Description: "Print the digest of the cached image, if available. " + "Exits with an error if not found.", }, + { + Flag: "verbose", + Env: WithEnvPrefix("VERBOSE"), + Value: serpent.BoolOf(&o.Verbose), + Description: "Enable verbose logging.", + }, } // Add options without the prefix for backward compatibility. These options diff --git a/scripts/develop.sh b/scripts/develop.sh index 3147244b..c209c8aa 100755 --- a/scripts/develop.sh +++ b/scripts/develop.sh @@ -3,9 +3,9 @@ cd "$(dirname "${BASH_SOURCE[0]}")" set -euxo pipefail -./build.sh +./build.sh || exit 1 docker run --rm -it \ - -e GIT_URL=https://github.com/denoland/deno \ - -e INIT_SCRIPT="bash" \ + -e ENVBUILDER_GIT_URL=https://github.com/denoland/deno \ + -e ENVBUILDER_INIT_SCRIPT="bash" \ envbuilder:latest diff --git a/testdata/options.golden b/testdata/options.golden index 73e68540..d59ccd21 100644 --- a/testdata/options.golden +++ b/testdata/options.golden @@ -155,6 +155,9 @@ OPTIONS: The content of an SSL cert file. This is useful for self-signed certificates. + --verbose bool, $ENVBUILDER_VERBOSE + Enable verbose logging. + --workspace-folder string, $ENVBUILDER_WORKSPACE_FOLDER The path to the workspace folder that will be built. This is optional. From 1f71283cc75ccea064c09fe836880016b94194c2 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 18 Jul 2024 17:24:18 +0300 Subject: [PATCH 07/81] chore: update kaniko fork for better cache probing (#273) (cherry picked from commit 1b358e290043807e8ed57a9a9a7716c051c7abc4) --- go.mod | 2 +- go.sum | 4 +-- integration/integration_test.go | 58 ++++++++++++++++++++++++++------- 3 files changed, 50 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 5a56db96..fb9a0496 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22.4 // There are a few options we need added to Kaniko! // See: https://github.com/GoogleContainerTools/kaniko/compare/main...coder:kaniko:main -replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240624091120-7208a49f5b15 +replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240717115058-0ba2908ca4d3 // Required to import codersdk due to gvisor dependency. replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20240702054557-aa558fbe5374 diff --git a/go.sum b/go.sum index 8a431485..3a91b326 100644 --- a/go.sum +++ b/go.sum @@ -171,8 +171,8 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352 h1:L/EjCuZxs5tOcqqCaASj/nu65TRYEFcTt8qRQfHZXX0= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352/go.mod h1:P1KoQSgnKEAG6Mnd3YlGzAophty+yKA9VV48LpfNRvo= -github.com/coder/kaniko v0.0.0-20240624091120-7208a49f5b15 h1:Rne2frxrqtLEQ/v4f/wS550Yp/WXLCRFzDuxg8b9woM= -github.com/coder/kaniko v0.0.0-20240624091120-7208a49f5b15/go.mod h1:YMK7BlxerzLlMwihGxNWUaFoN9LXCij4P+w/8/fNlcM= +github.com/coder/kaniko v0.0.0-20240717115058-0ba2908ca4d3 h1:Q7L6cjKfw3DIyhKIcgCJEmgxnUTBajmMDrHxXvxgBZs= +github.com/coder/kaniko v0.0.0-20240717115058-0ba2908ca4d3/go.mod h1:YMK7BlxerzLlMwihGxNWUaFoN9LXCij4P+w/8/fNlcM= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 h1:3A0ES21Ke+FxEM8CXx9n47SZOKOpgSE1bbJzlE4qPVs= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0/go.mod h1:5UuS2Ts+nTToAMeOjNlnHFkPahrtDkmpydBen/3wgZc= github.com/coder/quartz v0.1.0 h1:cLL+0g5l7xTf6ordRnUMMiZtRE8Sq5LxpghS63vEXrQ= diff --git a/integration/integration_test.go b/integration/integration_test.go index ae7047c0..29723573 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -1065,7 +1065,13 @@ func TestPushImage(t *testing.T) { srv := createGitServer(t, gitServerOptions{ files: map[string]string{ - ".devcontainer/Dockerfile": fmt.Sprintf("FROM %s\nRUN date --utc > /root/date.txt", testImageAlpine), + ".devcontainer/Dockerfile": fmt.Sprintf(`FROM %s +USER root +ARG WORKDIR=/ +WORKDIR $WORKDIR +ENV FOO=bar +RUN echo $FOO > /root/foo.txt +RUN date --utc > /root/date.txt`, testImageAlpine), ".devcontainer/devcontainer.json": `{ "name": "Test", "build": { @@ -1089,7 +1095,7 @@ func TestPushImage(t *testing.T) { envbuilderEnv("CACHE_REPO", testRepo), envbuilderEnv("GET_CACHED_IMAGE", "1"), }}) - require.ErrorContains(t, err, "error probing build cache: uncached command") + require.ErrorContains(t, err, "error probing build cache: uncached RUN command") // Then: it should fail to build the image and nothing should be pushed _, err = remote.Image(ref) require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") @@ -1119,7 +1125,13 @@ func TestPushImage(t *testing.T) { srv := createGitServer(t, gitServerOptions{ files: map[string]string{ - ".devcontainer/Dockerfile": fmt.Sprintf("FROM %s\nRUN date --utc > /root/date.txt", testImageAlpine), + ".devcontainer/Dockerfile": fmt.Sprintf(`FROM %s +USER root +ARG WORKDIR=/ +WORKDIR $WORKDIR +ENV FOO=bar +RUN echo $FOO > /root/foo.txt +RUN date --utc > /root/date.txt`, testImageAlpine), ".devcontainer/devcontainer.json": `{ "name": "Test", "build": { @@ -1143,7 +1155,7 @@ func TestPushImage(t *testing.T) { envbuilderEnv("CACHE_REPO", testRepo), envbuilderEnv("GET_CACHED_IMAGE", "1"), }}) - require.ErrorContains(t, err, "error probing build cache: uncached command") + require.ErrorContains(t, err, "error probing build cache: uncached RUN command") // Then: it should fail to build the image and nothing should be pushed _, err = remote.Image(ref) require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") @@ -1232,7 +1244,13 @@ func TestPushImage(t *testing.T) { srv := createGitServer(t, gitServerOptions{ files: map[string]string{ - ".devcontainer/Dockerfile": fmt.Sprintf("FROM %s\nRUN date --utc > /root/date.txt", testImageAlpine), + ".devcontainer/Dockerfile": fmt.Sprintf(`FROM %s +USER root +ARG WORKDIR=/ +WORKDIR $WORKDIR +ENV FOO=bar +RUN echo $FOO > /root/foo.txt +RUN date --utc > /root/date.txt`, testImageAlpine), ".devcontainer/devcontainer.json": `{ "name": "Test", "build": { @@ -1270,7 +1288,7 @@ func TestPushImage(t *testing.T) { envbuilderEnv("CACHE_REPO", testRepo), envbuilderEnv("GET_CACHED_IMAGE", "1"), }}) - require.ErrorContains(t, err, "error probing build cache: uncached command") + require.ErrorContains(t, err, "error probing build cache: uncached RUN command") // Then: it should fail to build the image and nothing should be pushed _, err = remote.Image(ref, remoteAuthOpt) require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") @@ -1303,7 +1321,13 @@ func TestPushImage(t *testing.T) { srv := createGitServer(t, gitServerOptions{ files: map[string]string{ - ".devcontainer/Dockerfile": fmt.Sprintf("FROM %s\nRUN date --utc > /root/date.txt", testImageAlpine), + ".devcontainer/Dockerfile": fmt.Sprintf(`FROM %s +USER root +ARG WORKDIR=/ +WORKDIR $WORKDIR +ENV FOO=bar +RUN echo $FOO > /root/foo.txt +RUN date --utc > /root/date.txt`, testImageAlpine), ".devcontainer/devcontainer.json": `{ "name": "Test", "build": { @@ -1332,7 +1356,7 @@ func TestPushImage(t *testing.T) { envbuilderEnv("CACHE_REPO", testRepo), envbuilderEnv("GET_CACHED_IMAGE", "1"), }}) - require.ErrorContains(t, err, "error probing build cache: uncached command") + require.ErrorContains(t, err, "error probing build cache: uncached RUN command") // Then: it should fail to build the image and nothing should be pushed _, err = remote.Image(ref, remoteAuthOpt) require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") @@ -1386,7 +1410,7 @@ COPY --from=a /root/date.txt /date.txt`, testImageAlpine, testImageAlpine), envbuilderEnv("GET_CACHED_IMAGE", "1"), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), }}) - require.ErrorContains(t, err, "error probing build cache: uncached command") + require.ErrorContains(t, err, "error probing build cache: uncached RUN command") // Then: it should fail to build the image and nothing should be pushed _, err = remote.Image(ref) require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") @@ -1422,7 +1446,13 @@ COPY --from=a /root/date.txt /date.txt`, testImageAlpine, testImageAlpine), srv := createGitServer(t, gitServerOptions{ files: map[string]string{ - ".devcontainer/Dockerfile": fmt.Sprintf("FROM %s\nRUN date --utc > /root/date.txt", testImageAlpine), + ".devcontainer/Dockerfile": fmt.Sprintf(`FROM %s +USER root +ARG WORKDIR=/ +WORKDIR $WORKDIR +ENV FOO=bar +RUN echo $FOO > /root/foo.txt +RUN date --utc > /root/date.txt`, testImageAlpine), ".devcontainer/devcontainer.json": `{ "name": "Test", "build": { @@ -1448,7 +1478,13 @@ COPY --from=a /root/date.txt /date.txt`, testImageAlpine, testImageAlpine), srv := createGitServer(t, gitServerOptions{ files: map[string]string{ - ".devcontainer/Dockerfile": fmt.Sprintf("FROM %s\nRUN date --utc > /root/date.txt", testImageAlpine), + ".devcontainer/Dockerfile": fmt.Sprintf(`FROM %s +USER root +ARG WORKDIR=/ +WORKDIR $WORKDIR +ENV FOO=bar +RUN echo $FOO > /root/foo.txt +RUN date --utc > /root/date.txt`, testImageAlpine), ".devcontainer/devcontainer.json": `{ "name": "Test", "build": { From c288d8c21a51417a666df1e82e04dc39db57d747 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Jul 2024 18:04:54 +0300 Subject: [PATCH 08/81] chore: bump google.golang.org/grpc from 1.64.0 to 1.64.1 (#271) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> (cherry picked from commit 309464b0d86baed09861b56edddea5a44d41345a) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fb9a0496..c2dd1a30 100644 --- a/go.mod +++ b/go.mod @@ -283,7 +283,7 @@ require ( google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect - google.golang.org/grpc v1.64.0 // indirect + google.golang.org/grpc v1.64.1 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/DataDog/dd-trace-go.v1 v1.64.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index 3a91b326..fe8ebf87 100644 --- a/go.sum +++ b/go.sum @@ -984,8 +984,8 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= -google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From 851c57d1b16040b4b8d9559c50b7980c5df9897f Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 22 Jul 2024 13:51:15 +0100 Subject: [PATCH 09/81] fix(envbuilder): add /product_uuid and /product_name to IgnorePaths by default (#274) Signed-off-by: Cian Johnston (cherry picked from commit 244c3ec71854c529ab75c4fb739463f7f63b62c5) --- envbuilder.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/envbuilder.go b/envbuilder.go index 10773dd5..62c4279e 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -89,7 +89,11 @@ func Run(ctx context.Context, options Options) error { // Once the legacy environment variables are phased out, this can be // reinstated to the previous default values. if len(options.IgnorePaths) == 0 { - options.IgnorePaths = []string{"/var/run"} + options.IgnorePaths = []string{ + "/var/run", + // KinD adds these paths to pods, so ignore them by default. + "/product_uuid", "/product_name", + } } if options.InitScript == "" { options.InitScript = "sleep infinity" From 3c4a4580664d28d990844c0ee49a06e2c6596b56 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 23 Jul 2024 11:37:26 +0100 Subject: [PATCH 10/81] chore: refactor options to separate package (#278) (cherry picked from commit b272a97d1ffc8dc8f216e8cfe4d968d3b322c35b) --- Makefile | 4 +- cmd/envbuilder/main.go | 32 +- envbuilder.go | 276 ++++++++---------- envbuilder_internal_test.go | 16 +- git.go | 4 +- git_test.go | 22 +- integration/integration_test.go | 123 ++++---- options.go => options/options.go | 22 +- options_test.go => options/options_test.go | 31 +- {testdata => options/testdata}/options.golden | 0 scripts/docsgen/main.go | 4 +- 11 files changed, 274 insertions(+), 260 deletions(-) rename options.go => options/options.go (97%) rename options_test.go => options/options_test.go (88%) rename {testdata => options/testdata}/options.golden (100%) diff --git a/Makefile b/Makefile index 28827efc..8bd3f6b5 100644 --- a/Makefile +++ b/Makefile @@ -33,10 +33,10 @@ build: scripts/envbuilder-$(GOARCH) update-golden-files: .gen-golden .gen-golden: $(GOLDEN_FILES) $(GO_SRC_FILES) $(GO_TEST_FILES) - go test . -update + go test ./options -update @touch "$@" -docs: options.go +docs: options/options.go options/options_test.go go run ./scripts/docsgen/main.go .PHONY: test diff --git a/cmd/envbuilder/main.go b/cmd/envbuilder/main.go index 77405536..24d63fd5 100644 --- a/cmd/envbuilder/main.go +++ b/cmd/envbuilder/main.go @@ -8,6 +8,8 @@ import ( "slices" "strings" + "github.com/coder/envbuilder/options" + "github.com/coder/coder/v2/codersdk" "github.com/coder/envbuilder" "github.com/coder/envbuilder/internal/log" @@ -23,47 +25,47 @@ func main() { cmd := envbuilderCmd() err := cmd.Invoke().WithOS().Run() if err != nil { - fmt.Fprintf(os.Stderr, "error: %v", err) + _, _ = fmt.Fprintf(os.Stderr, "error: %v", err) os.Exit(1) } } func envbuilderCmd() serpent.Command { - var options envbuilder.Options + var o options.Options cmd := serpent.Command{ Use: "envbuilder", - Options: options.CLI(), + Options: o.CLI(), Handler: func(inv *serpent.Invocation) error { - options.Logger = log.New(os.Stderr, options.Verbose) - if options.CoderAgentURL != "" { - if options.CoderAgentToken == "" { + o.Logger = log.New(os.Stderr, o.Verbose) + if o.CoderAgentURL != "" { + if o.CoderAgentToken == "" { return errors.New("CODER_AGENT_URL must be set if CODER_AGENT_TOKEN is set") } - u, err := url.Parse(options.CoderAgentURL) + u, err := url.Parse(o.CoderAgentURL) if err != nil { return fmt.Errorf("unable to parse CODER_AGENT_URL as URL: %w", err) } - coderLog, closeLogs, err := log.Coder(inv.Context(), u, options.CoderAgentToken) + coderLog, closeLogs, err := log.Coder(inv.Context(), u, o.CoderAgentToken) if err == nil { - options.Logger = log.Wrap(options.Logger, coderLog) + o.Logger = log.Wrap(o.Logger, coderLog) defer closeLogs() // This adds the envbuilder subsystem. // If telemetry is enabled in a Coder deployment, // this will be reported and help us understand // envbuilder usage. - if !slices.Contains(options.CoderAgentSubsystem, string(codersdk.AgentSubsystemEnvbuilder)) { - options.CoderAgentSubsystem = append(options.CoderAgentSubsystem, string(codersdk.AgentSubsystemEnvbuilder)) - _ = os.Setenv("CODER_AGENT_SUBSYSTEM", strings.Join(options.CoderAgentSubsystem, ",")) + if !slices.Contains(o.CoderAgentSubsystem, string(codersdk.AgentSubsystemEnvbuilder)) { + o.CoderAgentSubsystem = append(o.CoderAgentSubsystem, string(codersdk.AgentSubsystemEnvbuilder)) + _ = os.Setenv("CODER_AGENT_SUBSYSTEM", strings.Join(o.CoderAgentSubsystem, ",")) } } else { // Failure to log to Coder should cause a fatal error. - options.Logger(log.LevelError, "unable to send logs to Coder: %s", err.Error()) + o.Logger(log.LevelError, "unable to send logs to Coder: %s", err.Error()) } } - err := envbuilder.Run(inv.Context(), options) + err := envbuilder.Run(inv.Context(), o) if err != nil { - options.Logger(log.LevelError, "error: %s", err) + o.Logger(log.LevelError, "error: %s", err) } return err }, diff --git a/envbuilder.go b/envbuilder.go index 62c4279e..3cbdab04 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -24,8 +24,7 @@ import ( "syscall" "time" - "github.com/kballard/go-shellquote" - "github.com/mattn/go-isatty" + "github.com/coder/envbuilder/options" "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/GoogleContainerTools/kaniko/pkg/creds" @@ -46,6 +45,8 @@ import ( "github.com/go-git/go-git/v5/plumbing/transport" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/kballard/go-shellquote" + "github.com/mattn/go-isatty" "github.com/sirupsen/logrus" "github.com/tailscale/hujson" "golang.org/x/xerrors" @@ -83,45 +84,45 @@ type DockerConfig configfile.ConfigFile // Logger is the logf to use for all operations. // Filesystem is the filesystem to use for all operations. // Defaults to the host filesystem. -func Run(ctx context.Context, options Options) error { +func Run(ctx context.Context, opts options.Options) error { // Temporarily removed these from the default settings to prevent conflicts // between current and legacy environment variables that add default values. // Once the legacy environment variables are phased out, this can be // reinstated to the previous default values. - if len(options.IgnorePaths) == 0 { - options.IgnorePaths = []string{ + if len(opts.IgnorePaths) == 0 { + opts.IgnorePaths = []string{ "/var/run", // KinD adds these paths to pods, so ignore them by default. "/product_uuid", "/product_name", } } - if options.InitScript == "" { - options.InitScript = "sleep infinity" + if opts.InitScript == "" { + opts.InitScript = "sleep infinity" } - if options.InitCommand == "" { - options.InitCommand = "/bin/sh" + if opts.InitCommand == "" { + opts.InitCommand = "/bin/sh" } - if options.CacheRepo == "" && options.PushImage { + if opts.CacheRepo == "" && opts.PushImage { return fmt.Errorf("--cache-repo must be set when using --push-image") } // Default to the shell! - initArgs := []string{"-c", options.InitScript} - if options.InitArgs != "" { + initArgs := []string{"-c", opts.InitScript} + if opts.InitArgs != "" { var err error - initArgs, err = shellquote.Split(options.InitArgs) + initArgs, err = shellquote.Split(opts.InitArgs) if err != nil { return fmt.Errorf("parse init args: %w", err) } } - if options.Filesystem == nil { - options.Filesystem = &osfsWithChmod{osfs.New("/")} + if opts.Filesystem == nil { + opts.Filesystem = &osfsWithChmod{osfs.New("/")} } - if options.WorkspaceFolder == "" { - f, err := DefaultWorkspaceFolder(options.GitURL) + if opts.WorkspaceFolder == "" { + f, err := DefaultWorkspaceFolder(opts.GitURL) if err != nil { return err } - options.WorkspaceFolder = f + opts.WorkspaceFolder = f } stageNumber := 0 @@ -129,22 +130,22 @@ func Run(ctx context.Context, options Options) error { now := time.Now() stageNumber++ stageNum := stageNumber - options.Logger(log.LevelInfo, "#%d: %s", stageNum, fmt.Sprintf(format, args...)) + opts.Logger(log.LevelInfo, "#%d: %s", stageNum, fmt.Sprintf(format, args...)) return func(format string, args ...any) { - options.Logger(log.LevelInfo, "#%d: %s [%s]", stageNum, fmt.Sprintf(format, args...), time.Since(now)) + opts.Logger(log.LevelInfo, "#%d: %s [%s]", stageNum, fmt.Sprintf(format, args...), time.Since(now)) } } - options.Logger(log.LevelInfo, "%s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder")) + opts.Logger(log.LevelInfo, "%s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder")) var caBundle []byte - if options.SSLCertBase64 != "" { + if opts.SSLCertBase64 != "" { certPool, err := x509.SystemCertPool() if err != nil { return xerrors.Errorf("get global system cert pool: %w", err) } - data, err := base64.StdEncoding.DecodeString(options.SSLCertBase64) + data, err := base64.StdEncoding.DecodeString(opts.SSLCertBase64) if err != nil { return xerrors.Errorf("base64 decode ssl cert: %w", err) } @@ -155,8 +156,8 @@ func Run(ctx context.Context, options Options) error { caBundle = data } - if options.DockerConfigBase64 != "" { - decoded, err := base64.StdEncoding.DecodeString(options.DockerConfigBase64) + if opts.DockerConfigBase64 != "" { + decoded, err := base64.StdEncoding.DecodeString(opts.DockerConfigBase64) if err != nil { return fmt.Errorf("decode docker config: %w", err) } @@ -177,10 +178,10 @@ func Run(ctx context.Context, options Options) error { var fallbackErr error var cloned bool - if options.GitURL != "" { + if opts.GitURL != "" { endStage := startStage("📦 Cloning %s to %s...", - newColor(color.FgCyan).Sprintf(options.GitURL), - newColor(color.FgCyan).Sprintf(options.WorkspaceFolder), + newColor(color.FgCyan).Sprintf(opts.GitURL), + newColor(color.FgCyan).Sprintf(opts.WorkspaceFolder), ) reader, writer := io.Pipe() @@ -198,28 +199,28 @@ func Run(ctx context.Context, options Options) error { if line == "" { continue } - options.Logger(log.LevelInfo, "#1: %s", strings.TrimSpace(line)) + opts.Logger(log.LevelInfo, "#1: %s", strings.TrimSpace(line)) } } }() cloneOpts := CloneRepoOptions{ - Path: options.WorkspaceFolder, - Storage: options.Filesystem, - Insecure: options.Insecure, + Path: opts.WorkspaceFolder, + Storage: opts.Filesystem, + Insecure: opts.Insecure, Progress: writer, - SingleBranch: options.GitCloneSingleBranch, - Depth: int(options.GitCloneDepth), + SingleBranch: opts.GitCloneSingleBranch, + Depth: int(opts.GitCloneDepth), CABundle: caBundle, } - cloneOpts.RepoAuth = SetupRepoAuth(&options) - if options.GitHTTPProxyURL != "" { + cloneOpts.RepoAuth = SetupRepoAuth(&opts) + if opts.GitHTTPProxyURL != "" { cloneOpts.ProxyOptions = transport.ProxyOptions{ - URL: options.GitHTTPProxyURL, + URL: opts.GitHTTPProxyURL, } } - cloneOpts.RepoURL = options.GitURL + cloneOpts.RepoURL = opts.GitURL cloned, fallbackErr = CloneRepo(ctx, cloneOpts) if fallbackErr == nil { @@ -229,19 +230,19 @@ func Run(ctx context.Context, options Options) error { endStage("📦 The repository already exists!") } } else { - options.Logger(log.LevelError, "Failed to clone repository: %s", fallbackErr.Error()) - options.Logger(log.LevelError, "Falling back to the default image...") + opts.Logger(log.LevelError, "Failed to clone repository: %s", fallbackErr.Error()) + opts.Logger(log.LevelError, "Falling back to the default image...") } } defaultBuildParams := func() (*devcontainer.Compiled, error) { dockerfile := filepath.Join(MagicDir, "Dockerfile") - file, err := options.Filesystem.OpenFile(dockerfile, os.O_CREATE|os.O_WRONLY, 0o644) + file, err := opts.Filesystem.OpenFile(dockerfile, os.O_CREATE|os.O_WRONLY, 0o644) if err != nil { return nil, err } defer file.Close() - if options.FallbackImage == "" { + if opts.FallbackImage == "" { if fallbackErr != nil { return nil, xerrors.Errorf("%s: %w", fallbackErr.Error(), ErrNoFallbackImage) } @@ -249,7 +250,7 @@ func Run(ctx context.Context, options Options) error { // don't support parsing a multiline error. return nil, ErrNoFallbackImage } - content := "FROM " + options.FallbackImage + content := "FROM " + opts.FallbackImage _, err = file.Write([]byte(content)) if err != nil { return nil, err @@ -267,19 +268,19 @@ func Run(ctx context.Context, options Options) error { devcontainerPath string ) - if options.DockerfilePath == "" { + if opts.DockerfilePath == "" { // Only look for a devcontainer if a Dockerfile wasn't specified. // devcontainer is a standard, so it's reasonable to be the default. var devcontainerDir string var err error - devcontainerPath, devcontainerDir, err = findDevcontainerJSON(options) + devcontainerPath, devcontainerDir, err = findDevcontainerJSON(opts) if err != nil { - options.Logger(log.LevelError, "Failed to locate devcontainer.json: %s", err.Error()) - options.Logger(log.LevelError, "Falling back to the default image...") + opts.Logger(log.LevelError, "Failed to locate devcontainer.json: %s", err.Error()) + opts.Logger(log.LevelError, "Falling back to the default image...") } else { // We know a devcontainer exists. // Let's parse it and use it! - file, err := options.Filesystem.Open(devcontainerPath) + file, err := opts.Filesystem.Open(devcontainerPath) if err != nil { return fmt.Errorf("open devcontainer.json: %w", err) } @@ -296,32 +297,32 @@ func Run(ctx context.Context, options Options) error { if err != nil { return fmt.Errorf("no Dockerfile or image found: %w", err) } - options.Logger(log.LevelInfo, "No Dockerfile or image specified; falling back to the default image...") + opts.Logger(log.LevelInfo, "No Dockerfile or image specified; falling back to the default image...") fallbackDockerfile = defaultParams.DockerfilePath } - buildParams, err = devContainer.Compile(options.Filesystem, devcontainerDir, MagicDir, fallbackDockerfile, options.WorkspaceFolder, false, os.LookupEnv) + buildParams, err = devContainer.Compile(opts.Filesystem, devcontainerDir, MagicDir, fallbackDockerfile, opts.WorkspaceFolder, false, os.LookupEnv) if err != nil { return fmt.Errorf("compile devcontainer.json: %w", err) } scripts = devContainer.LifecycleScripts } else { - options.Logger(log.LevelError, "Failed to parse devcontainer.json: %s", err.Error()) - options.Logger(log.LevelError, "Falling back to the default image...") + opts.Logger(log.LevelError, "Failed to parse devcontainer.json: %s", err.Error()) + opts.Logger(log.LevelError, "Falling back to the default image...") } } } else { // If a Dockerfile was specified, we use that. - dockerfilePath := filepath.Join(options.WorkspaceFolder, options.DockerfilePath) + dockerfilePath := filepath.Join(opts.WorkspaceFolder, opts.DockerfilePath) // If the dockerfilePath is specified and deeper than the base of WorkspaceFolder AND the BuildContextPath is // not defined, show a warning dockerfileDir := filepath.Dir(dockerfilePath) - if dockerfileDir != filepath.Clean(options.WorkspaceFolder) && options.BuildContextPath == "" { - options.Logger(log.LevelWarn, "given dockerfile %q is below %q and no custom build context has been defined", dockerfilePath, options.WorkspaceFolder) - options.Logger(log.LevelWarn, "\t-> set BUILD_CONTEXT_PATH to %q to fix", dockerfileDir) + if dockerfileDir != filepath.Clean(opts.WorkspaceFolder) && opts.BuildContextPath == "" { + opts.Logger(log.LevelWarn, "given dockerfile %q is below %q and no custom build context has been defined", dockerfilePath, opts.WorkspaceFolder) + opts.Logger(log.LevelWarn, "\t-> set BUILD_CONTEXT_PATH to %q to fix", dockerfileDir) } - dockerfile, err := options.Filesystem.Open(dockerfilePath) + dockerfile, err := opts.Filesystem.Open(dockerfilePath) if err == nil { content, err := io.ReadAll(dockerfile) if err != nil { @@ -330,7 +331,7 @@ func Run(ctx context.Context, options Options) error { buildParams = &devcontainer.Compiled{ DockerfilePath: dockerfilePath, DockerfileContent: string(content), - BuildContext: filepath.Join(options.WorkspaceFolder, options.BuildContextPath), + BuildContext: filepath.Join(opts.WorkspaceFolder, opts.BuildContextPath), } } } @@ -347,17 +348,17 @@ func Run(ctx context.Context, options Options) error { HijackLogrus(func(entry *logrus.Entry) { for _, line := range strings.Split(entry.Message, "\r") { - options.Logger(log.LevelInfo, "#%d: %s", stageNumber, color.HiBlackString(line)) + opts.Logger(log.LevelInfo, "#%d: %s", stageNumber, color.HiBlackString(line)) } }) var closeAfterBuild func() // Allows quick testing of layer caching using a local directory! - if options.LayerCacheDir != "" { + if opts.LayerCacheDir != "" { cfg := &configuration.Configuration{ Storage: configuration.Storage{ "filesystem": configuration.Parameters{ - "rootdirectory": options.LayerCacheDir, + "rootdirectory": opts.LayerCacheDir, }, }, } @@ -380,31 +381,31 @@ func Run(ctx context.Context, options Options) error { go func() { err := srv.Serve(listener) if err != nil && !errors.Is(err, http.ErrServerClosed) { - options.Logger(log.LevelError, "Failed to serve registry: %s", err.Error()) + opts.Logger(log.LevelError, "Failed to serve registry: %s", err.Error()) } }() closeAfterBuild = func() { _ = srv.Close() _ = listener.Close() } - if options.CacheRepo != "" { - options.Logger(log.LevelWarn, "Overriding cache repo with local registry...") + if opts.CacheRepo != "" { + opts.Logger(log.LevelWarn, "Overriding cache repo with local registry...") } - options.CacheRepo = fmt.Sprintf("localhost:%d/local/cache", tcpAddr.Port) + opts.CacheRepo = fmt.Sprintf("localhost:%d/local/cache", tcpAddr.Port) } - // IgnorePaths in the Kaniko options doesn't properly ignore paths. + // IgnorePaths in the Kaniko opts doesn't properly ignore paths. // So we add them to the default ignore list. See: // https://github.com/GoogleContainerTools/kaniko/blob/63be4990ca5a60bdf06ddc4d10aa4eca0c0bc714/cmd/executor/cmd/root.go#L136 ignorePaths := append([]string{ MagicDir, - options.WorkspaceFolder, + opts.WorkspaceFolder, // See: https://github.com/coder/envbuilder/issues/37 "/etc/resolv.conf", - }, options.IgnorePaths...) + }, opts.IgnorePaths...) - if options.LayerCacheDir != "" { - ignorePaths = append(ignorePaths, options.LayerCacheDir) + if opts.LayerCacheDir != "" { + ignorePaths = append(ignorePaths, opts.LayerCacheDir) } for _, ignorePath := range ignorePaths { @@ -417,7 +418,7 @@ func Run(ctx context.Context, options Options) error { // In order to allow 'resuming' envbuilder, embed the binary into the image // if it is being pushed - if options.PushImage { + if opts.PushImage { exePath, err := os.Executable() if err != nil { return xerrors.Errorf("get exe path: %w", err) @@ -443,10 +444,10 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) // ignorePrefixes is a superset of ignorePaths that we pass to kaniko's // IgnoreList. ignorePrefixes := append([]string{"/dev", "/proc", "/sys"}, ignorePaths...) - restoreMounts, err := ebutil.TempRemount(options.Logger, tempRemountDest, ignorePrefixes...) + restoreMounts, err := ebutil.TempRemount(opts.Logger, tempRemountDest, ignorePrefixes...) defer func() { // restoreMounts should never be nil if err := restoreMounts(); err != nil { - options.Logger(log.LevelError, "restore mounts: %s", err.Error()) + opts.Logger(log.LevelError, "restore mounts: %s", err.Error()) } }() if err != nil { @@ -455,8 +456,8 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) skippedRebuild := false build := func() (v1.Image, error) { - _, err := options.Filesystem.Stat(MagicFile) - if err == nil && options.SkipRebuild { + _, err := opts.Filesystem.Stat(MagicFile) + if err == nil && opts.SkipRebuild { endStage := startStage("🏗️ Skipping build because of cache...") imageRef, err := devcontainer.ImageFromDockerfile(buildParams.DockerfileContent) if err != nil { @@ -479,7 +480,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) // It's possible that the container will already have files in it, and // we don't want to merge a new container with the old one. - if err := maybeDeleteFilesystem(options.Logger, options.ForceSafe); err != nil { + if err := maybeDeleteFilesystem(opts.Logger, opts.ForceSafe); err != nil { return nil, fmt.Errorf("delete filesystem: %w", err) } @@ -492,18 +493,18 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) go func() { scanner := bufio.NewScanner(stdoutReader) for scanner.Scan() { - options.Logger(log.LevelInfo, "%s", scanner.Text()) + opts.Logger(log.LevelInfo, "%s", scanner.Text()) } }() go func() { scanner := bufio.NewScanner(stderrReader) for scanner.Scan() { - options.Logger(log.LevelInfo, "%s", scanner.Text()) + opts.Logger(log.LevelInfo, "%s", scanner.Text()) } }() cacheTTL := time.Hour * 24 * 7 - if options.CacheTTLDays != 0 { - cacheTTL = time.Hour * 24 * time.Duration(options.CacheTTLDays) + if opts.CacheTTLDays != 0 { + cacheTTL = time.Hour * 24 * time.Duration(opts.CacheTTLDays) } // At this point we have all the context, we can now build! @@ -512,10 +513,10 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) registryMirror = strings.Split(val, ";") } var destinations []string - if options.CacheRepo != "" { - destinations = append(destinations, options.CacheRepo) + if opts.CacheRepo != "" { + destinations = append(destinations, opts.CacheRepo) } - opts := &config.KanikoOptions{ + kOpts := &config.KanikoOptions{ // Boilerplate! CustomPlatform: platforms.Format(platforms.Normalize(platforms.DefaultSpec())), SnapshotMode: "redo", @@ -523,7 +524,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) RunStdout: stdoutWriter, RunStderr: stderrWriter, Destinations: destinations, - NoPush: !options.PushImage || len(destinations) == 0, + NoPush: !opts.PushImage || len(destinations) == 0, CacheRunLayers: true, CacheCopyLayers: true, CompressedCaching: true, @@ -535,18 +536,18 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) CacheOptions: config.CacheOptions{ // Cache for a week by default! CacheTTL: cacheTTL, - CacheDir: options.BaseImageCacheDir, + CacheDir: opts.BaseImageCacheDir, }, ForceUnpack: true, BuildArgs: buildParams.BuildArgs, - CacheRepo: options.CacheRepo, - Cache: options.CacheRepo != "" || options.BaseImageCacheDir != "", + CacheRepo: opts.CacheRepo, + Cache: opts.CacheRepo != "" || opts.BaseImageCacheDir != "", DockerfilePath: buildParams.DockerfilePath, DockerfileContent: buildParams.DockerfileContent, RegistryOptions: config.RegistryOptions{ - Insecure: options.Insecure, - InsecurePull: options.Insecure, - SkipTLSVerify: options.Insecure, + Insecure: opts.Insecure, + InsecurePull: opts.Insecure, + SkipTLSVerify: opts.Insecure, // Enables registry mirror features in Kaniko, see more in link below // https://github.com/GoogleContainerTools/kaniko?tab=readme-ov-file#flag---registry-mirror // Related to PR #114 @@ -556,12 +557,12 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) SrcContext: buildParams.BuildContext, // For cached image utilization, produce reproducible builds. - Reproducible: options.PushImage, + Reproducible: opts.PushImage, } - if options.GetCachedImage { + if opts.GetCachedImage { endStage := startStage("🏗️ Checking for cached image...") - image, err := executor.DoCacheProbe(opts) + image, err := executor.DoCacheProbe(kOpts) if err != nil { return nil, xerrors.Errorf("get cached image: %w", err) } @@ -570,19 +571,19 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) return nil, xerrors.Errorf("get cached image digest: %w", err) } endStage("🏗️ Found cached image!") - _, _ = fmt.Fprintf(os.Stdout, "ENVBUILDER_CACHED_IMAGE=%s@%s\n", options.CacheRepo, digest.String()) + _, _ = fmt.Fprintf(os.Stdout, "ENVBUILDER_CACHED_IMAGE=%s@%s\n", kOpts.CacheRepo, digest.String()) os.Exit(0) } endStage := startStage("🏗️ Building image...") - image, err := executor.DoBuild(opts) + image, err := executor.DoBuild(kOpts) if err != nil { return nil, xerrors.Errorf("do build: %w", err) } endStage("🏗️ Built image!") - if options.PushImage { + if opts.PushImage { endStage = startStage("🏗️ Pushing image...") - if err := executor.DoPush(image, opts); err != nil { + if err := executor.DoPush(image, kOpts); err != nil { return nil, xerrors.Errorf("do push: %w", err) } endStage("🏗️ Pushed image!") @@ -611,13 +612,13 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) fallback = true fallbackErr = err case strings.Contains(err.Error(), "unexpected status code 401 Unauthorized"): - options.Logger(log.LevelError, "Unable to pull the provided image. Ensure your registry credentials are correct!") + opts.Logger(log.LevelError, "Unable to pull the provided image. Ensure your registry credentials are correct!") } - if !fallback || options.ExitOnBuildFailure { + if !fallback || opts.ExitOnBuildFailure { return err } - options.Logger(log.LevelError, "Failed to build: %s", err) - options.Logger(log.LevelError, "Falling back to the default image...") + opts.Logger(log.LevelError, "Failed to build: %s", err) + opts.Logger(log.LevelError, "Falling back to the default image...") buildParams, err = defaultBuildParams() if err != nil { return err @@ -638,7 +639,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) // Create the magic file to indicate that this build // has already been ran before! - file, err := options.Filesystem.Create(MagicFile) + file, err := opts.Filesystem.Create(MagicFile) if err != nil { return fmt.Errorf("create magic file: %w", err) } @@ -664,10 +665,10 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) if err != nil { return fmt.Errorf("unmarshal metadata: %w", err) } - options.Logger(log.LevelInfo, "#3: 👀 Found devcontainer.json label metadata in image...") + opts.Logger(log.LevelInfo, "#3: 👀 Found devcontainer.json label metadata in image...") for _, container := range devContainer { if container.RemoteUser != "" { - options.Logger(log.LevelInfo, "#3: 🧑 Updating the user to %q!", container.RemoteUser) + opts.Logger(log.LevelInfo, "#3: 🧑 Updating the user to %q!", container.RemoteUser) configFile.Config.User = container.RemoteUser } @@ -688,11 +689,11 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) } } - // Sanitize the environment of any options! - unsetOptionsEnv() + // Sanitize the environment of any opts! + options.UnsetEnv() // Remove the Docker config secret file! - if options.DockerConfigBase64 != "" { + if opts.DockerConfigBase64 != "" { c := filepath.Join(MagicDir, "config.json") err = os.Remove(c) if err != nil { @@ -741,7 +742,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) } sort.Strings(envKeys) for _, envVar := range envKeys { - value := devcontainer.SubstituteVars(env[envVar], options.WorkspaceFolder, os.LookupEnv) + value := devcontainer.SubstituteVars(env[envVar], opts.WorkspaceFolder, os.LookupEnv) os.Setenv(envVar, value) } } @@ -751,10 +752,10 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) // in the export. We should have generated a complete set of environment // on the intial build, so exporting environment variables a second time // isn't useful anyway. - if options.ExportEnvFile != "" && !skippedRebuild { - exportEnvFile, err := os.Create(options.ExportEnvFile) + if opts.ExportEnvFile != "" && !skippedRebuild { + exportEnvFile, err := os.Create(opts.ExportEnvFile) if err != nil { - return fmt.Errorf("failed to open EXPORT_ENV_FILE %q: %w", options.ExportEnvFile, err) + return fmt.Errorf("failed to open EXPORT_ENV_FILE %q: %w", opts.ExportEnvFile, err) } envKeys := make([]string, 0, len(allEnvKeys)) @@ -774,7 +775,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) username = buildParams.User } if username == "" { - options.Logger(log.LevelWarn, "#3: no user specified, using root") + opts.Logger(log.LevelWarn, "#3: no user specified, using root") } userInfo, err := getUser(username) @@ -791,13 +792,13 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) // // We need to change the ownership of the files to the user that will // be running the init script. - if chownErr := filepath.Walk(options.WorkspaceFolder, func(path string, _ os.FileInfo, err error) error { + if chownErr := filepath.Walk(opts.WorkspaceFolder, func(path string, _ os.FileInfo, err error) error { if err != nil { return err } return os.Chown(path, userInfo.uid, userInfo.gid) }); chownErr != nil { - options.Logger(log.LevelError, "chown %q: %s", userInfo.user.HomeDir, chownErr.Error()) + opts.Logger(log.LevelError, "chown %q: %s", userInfo.user.HomeDir, chownErr.Error()) endStage("⚠️ Failed to the ownership of the workspace, you may need to fix this manually!") } else { endStage("👤 Updated the ownership of the workspace!") @@ -814,18 +815,18 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) } return os.Chown(path, userInfo.uid, userInfo.gid) }); chownErr != nil { - options.Logger(log.LevelError, "chown %q: %s", userInfo.user.HomeDir, chownErr.Error()) + opts.Logger(log.LevelError, "chown %q: %s", userInfo.user.HomeDir, chownErr.Error()) endStage("⚠️ Failed to update ownership of %s, you may need to fix this manually!", userInfo.user.HomeDir) } else { endStage("🏡 Updated ownership of %s!", userInfo.user.HomeDir) } } - err = os.MkdirAll(options.WorkspaceFolder, 0o755) + err = os.MkdirAll(opts.WorkspaceFolder, 0o755) if err != nil { return fmt.Errorf("create workspace folder: %w", err) } - err = os.Chdir(options.WorkspaceFolder) + err = os.Chdir(opts.WorkspaceFolder) if err != nil { return fmt.Errorf("change directory: %w", err) } @@ -837,7 +838,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) // exec systemd as the init command, but that doesn't mean we should // run the lifecycle scripts as root. os.Setenv("HOME", userInfo.user.HomeDir) - if err := execLifecycleScripts(ctx, options, scripts, skippedRebuild, userInfo); err != nil { + if err := execLifecycleScripts(ctx, opts, scripts, skippedRebuild, userInfo); err != nil { return err } @@ -846,11 +847,11 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) // // This is useful for hooking into the environment for a specific // init to PID 1. - if options.SetupScript != "" { + if opts.SetupScript != "" { // We execute the initialize script as the root user! os.Setenv("HOME", "/root") - options.Logger(log.LevelInfo, "=== Running the setup command %q as the root user...", options.SetupScript) + opts.Logger(log.LevelInfo, "=== Running the setup command %q as the root user...", opts.SetupScript) envKey := "ENVBUILDER_ENV" envFile := filepath.Join("/", MagicDir, "environ") @@ -860,12 +861,12 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) } _ = file.Close() - cmd := exec.CommandContext(ctx, "/bin/sh", "-c", options.SetupScript) + cmd := exec.CommandContext(ctx, "/bin/sh", "-c", opts.SetupScript) cmd.Env = append(os.Environ(), fmt.Sprintf("%s=%s", envKey, envFile), fmt.Sprintf("TARGET_USER=%s", userInfo.user.Username), ) - cmd.Dir = options.WorkspaceFolder + cmd.Dir = opts.WorkspaceFolder // This allows for a really nice and clean experience to experiement with! // e.g. docker run --it --rm -e INIT_SCRIPT bash ... if isatty.IsTerminal(os.Stdout.Fd()) && isatty.IsTerminal(os.Stdin.Fd()) { @@ -877,7 +878,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) go func() { scanner := bufio.NewScanner(&buf) for scanner.Scan() { - options.Logger(log.LevelInfo, "%s", scanner.Text()) + opts.Logger(log.LevelInfo, "%s", scanner.Text()) } }() @@ -907,7 +908,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) key := pair[0] switch key { case "INIT_COMMAND": - options.InitCommand = pair[1] + opts.InitCommand = pair[1] updatedCommand = true case "INIT_ARGS": initArgs, err = shellquote.Split(pair[1]) @@ -943,9 +944,9 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) return fmt.Errorf("set uid: %w", err) } - options.Logger(log.LevelInfo, "=== Running the init command %s %+v as the %q user...", options.InitCommand, initArgs, userInfo.user.Username) + opts.Logger(log.LevelInfo, "=== Running the init command %s %+v as the %q user...", opts.InitCommand, initArgs, userInfo.user.Username) - err = syscall.Exec(options.InitCommand, append([]string{options.InitCommand}, initArgs...), os.Environ()) + err = syscall.Exec(opts.InitCommand, append([]string{opts.InitCommand}, initArgs...), os.Environ()) if err != nil { return fmt.Errorf("exec init script: %w", err) } @@ -1039,7 +1040,7 @@ func execOneLifecycleScript( func execLifecycleScripts( ctx context.Context, - options Options, + options options.Options, scripts devcontainer.LifecycleScripts, skippedRebuild bool, userInfo userInfo, @@ -1093,25 +1094,6 @@ func createPostStartScript(path string, postStartCommand devcontainer.LifecycleS return nil } -// unsetOptionsEnv unsets all environment variables that are used -// to configure the options. -func unsetOptionsEnv() { - var o Options - for _, opt := range o.CLI() { - if opt.Env == "" { - continue - } - // Do not strip options that do not have the magic prefix! - // For example, CODER_AGENT_URL, CODER_AGENT_TOKEN, CODER_AGENT_SUBSYSTEM. - if !strings.HasPrefix(opt.Env, envPrefix) { - continue - } - // Strip both with and without prefix. - os.Unsetenv(opt.Env) - os.Unsetenv(strings.TrimPrefix(opt.Env, envPrefix)) - } -} - func newColor(value ...color.Attribute) *color.Color { c := color.New(value...) c.EnableColor() @@ -1126,7 +1108,7 @@ func (fs *osfsWithChmod) Chmod(name string, mode os.FileMode) error { return os.Chmod(name, mode) } -func findDevcontainerJSON(options Options) (string, string, error) { +func findDevcontainerJSON(options options.Options) (string, string, error) { // 0. Check if custom devcontainer directory or path is provided. if options.DevcontainerDir != "" || options.DevcontainerJSONPath != "" { devcontainerDir := options.DevcontainerDir diff --git a/envbuilder_internal_test.go b/envbuilder_internal_test.go index 65edb9cd..3af4b5e4 100644 --- a/envbuilder_internal_test.go +++ b/envbuilder_internal_test.go @@ -3,6 +3,8 @@ package envbuilder import ( "testing" + "github.com/coder/envbuilder/options" + "github.com/go-git/go-billy/v5/memfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -18,7 +20,7 @@ func TestFindDevcontainerJSON(t *testing.T) { fs := memfs.New() // when - _, _, err := findDevcontainerJSON(Options{ + _, _, err := findDevcontainerJSON(options.Options{ Filesystem: fs, WorkspaceFolder: "/workspace", }) @@ -36,7 +38,7 @@ func TestFindDevcontainerJSON(t *testing.T) { require.NoError(t, err) // when - _, _, err = findDevcontainerJSON(Options{ + _, _, err = findDevcontainerJSON(options.Options{ Filesystem: fs, WorkspaceFolder: "/workspace", }) @@ -56,7 +58,7 @@ func TestFindDevcontainerJSON(t *testing.T) { require.NoError(t, err) // when - devcontainerPath, devcontainerDir, err := findDevcontainerJSON(Options{ + devcontainerPath, devcontainerDir, err := findDevcontainerJSON(options.Options{ Filesystem: fs, WorkspaceFolder: "/workspace", }) @@ -78,7 +80,7 @@ func TestFindDevcontainerJSON(t *testing.T) { require.NoError(t, err) // when - devcontainerPath, devcontainerDir, err := findDevcontainerJSON(Options{ + devcontainerPath, devcontainerDir, err := findDevcontainerJSON(options.Options{ Filesystem: fs, WorkspaceFolder: "/workspace", DevcontainerDir: "experimental-devcontainer", @@ -101,7 +103,7 @@ func TestFindDevcontainerJSON(t *testing.T) { require.NoError(t, err) // when - devcontainerPath, devcontainerDir, err := findDevcontainerJSON(Options{ + devcontainerPath, devcontainerDir, err := findDevcontainerJSON(options.Options{ Filesystem: fs, WorkspaceFolder: "/workspace", DevcontainerJSONPath: "experimental.json", @@ -124,7 +126,7 @@ func TestFindDevcontainerJSON(t *testing.T) { require.NoError(t, err) // when - devcontainerPath, devcontainerDir, err := findDevcontainerJSON(Options{ + devcontainerPath, devcontainerDir, err := findDevcontainerJSON(options.Options{ Filesystem: fs, WorkspaceFolder: "/workspace", }) @@ -146,7 +148,7 @@ func TestFindDevcontainerJSON(t *testing.T) { require.NoError(t, err) // when - devcontainerPath, devcontainerDir, err := findDevcontainerJSON(Options{ + devcontainerPath, devcontainerDir, err := findDevcontainerJSON(options.Options{ Filesystem: fs, WorkspaceFolder: "/workspace", }) diff --git a/git.go b/git.go index f28cab8d..eb75b654 100644 --- a/git.go +++ b/git.go @@ -9,6 +9,8 @@ import ( "os" "strings" + "github.com/coder/envbuilder/options" + giturls "github.com/chainguard-dev/git-urls" "github.com/coder/envbuilder/internal/log" "github.com/go-git/go-billy/v5" @@ -177,7 +179,7 @@ func LogHostKeyCallback(logger log.Func) gossh.HostKeyCallback { // If SSH_KNOWN_HOSTS is not set, the SSH auth method will be configured // to accept and log all host keys. Otherwise, host key checking will be // performed as usual. -func SetupRepoAuth(options *Options) transport.AuthMethod { +func SetupRepoAuth(options *options.Options) transport.AuthMethod { if options.GitURL == "" { options.Logger(log.LevelInfo, "#1: ❔ No Git URL supplied!") return nil diff --git a/git_test.go b/git_test.go index 38efee1a..842cf2c9 100644 --- a/git_test.go +++ b/git_test.go @@ -12,6 +12,8 @@ import ( "regexp" "testing" + "github.com/coder/envbuilder/options" + "github.com/coder/envbuilder" "github.com/coder/envbuilder/internal/log" "github.com/coder/envbuilder/testutil/gittest" @@ -265,7 +267,7 @@ func TestCloneRepoSSH(t *testing.T) { func TestSetupRepoAuth(t *testing.T) { t.Setenv("SSH_AUTH_SOCK", "") t.Run("Empty", func(t *testing.T) { - opts := &envbuilder.Options{ + opts := &options.Options{ Logger: testLog(t), } auth := envbuilder.SetupRepoAuth(opts) @@ -273,7 +275,7 @@ func TestSetupRepoAuth(t *testing.T) { }) t.Run("HTTP/NoAuth", func(t *testing.T) { - opts := &envbuilder.Options{ + opts := &options.Options{ GitURL: "http://host.tld/repo", Logger: testLog(t), } @@ -282,7 +284,7 @@ func TestSetupRepoAuth(t *testing.T) { }) t.Run("HTTP/BasicAuth", func(t *testing.T) { - opts := &envbuilder.Options{ + opts := &options.Options{ GitURL: "http://host.tld/repo", GitUsername: "user", GitPassword: "pass", @@ -296,7 +298,7 @@ func TestSetupRepoAuth(t *testing.T) { }) t.Run("HTTPS/BasicAuth", func(t *testing.T) { - opts := &envbuilder.Options{ + opts := &options.Options{ GitURL: "https://host.tld/repo", GitUsername: "user", GitPassword: "pass", @@ -311,7 +313,7 @@ func TestSetupRepoAuth(t *testing.T) { t.Run("SSH/WithScheme", func(t *testing.T) { kPath := writeTestPrivateKey(t) - opts := &envbuilder.Options{ + opts := &options.Options{ GitURL: "ssh://host.tld/repo", GitSSHPrivateKeyPath: kPath, Logger: testLog(t), @@ -323,7 +325,7 @@ func TestSetupRepoAuth(t *testing.T) { t.Run("SSH/NoScheme", func(t *testing.T) { kPath := writeTestPrivateKey(t) - opts := &envbuilder.Options{ + opts := &options.Options{ GitURL: "git@host.tld:repo/path", GitSSHPrivateKeyPath: kPath, Logger: testLog(t), @@ -336,7 +338,7 @@ func TestSetupRepoAuth(t *testing.T) { t.Run("SSH/OtherScheme", func(t *testing.T) { // Anything that is not https:// or http:// is treated as SSH. kPath := writeTestPrivateKey(t) - opts := &envbuilder.Options{ + opts := &options.Options{ GitURL: "git://git@host.tld:repo/path", GitSSHPrivateKeyPath: kPath, Logger: testLog(t), @@ -348,7 +350,7 @@ func TestSetupRepoAuth(t *testing.T) { t.Run("SSH/GitUsername", func(t *testing.T) { kPath := writeTestPrivateKey(t) - opts := &envbuilder.Options{ + opts := &options.Options{ GitURL: "host.tld:12345/repo/path", GitSSHPrivateKeyPath: kPath, GitUsername: "user", @@ -361,7 +363,7 @@ func TestSetupRepoAuth(t *testing.T) { t.Run("SSH/PrivateKey", func(t *testing.T) { kPath := writeTestPrivateKey(t) - opts := &envbuilder.Options{ + opts := &options.Options{ GitURL: "ssh://git@host.tld:repo/path", GitSSHPrivateKeyPath: kPath, Logger: testLog(t), @@ -376,7 +378,7 @@ func TestSetupRepoAuth(t *testing.T) { }) t.Run("SSH/NoAuthMethods", func(t *testing.T) { - opts := &envbuilder.Options{ + opts := &options.Options{ GitURL: "ssh://git@host.tld:repo/path", Logger: testLog(t), } diff --git a/integration/integration_test.go b/integration/integration_test.go index 29723573..e62bac02 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -21,11 +21,14 @@ import ( "testing" "time" + "github.com/coder/envbuilder/options" + "github.com/coder/envbuilder" "github.com/coder/envbuilder/devcontainer/features" "github.com/coder/envbuilder/testutil/gittest" "github.com/coder/envbuilder/testutil/mwtest" "github.com/coder/envbuilder/testutil/registrytest" + clitypes "github.com/docker/cli/cli/config/types" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" @@ -71,7 +74,7 @@ func TestInitScriptInitCommand(t *testing.T) { "Dockerfile": fmt.Sprintf("FROM %s\nRUN unlink /bin/sh", testImageAlpine), }, }) - _, err := runEnvbuilder(t, options{env: []string{ + _, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), envbuilderEnv("INIT_SCRIPT", fmt.Sprintf(`wget -O - %q`, initSrv.URL)), @@ -85,7 +88,7 @@ func TestInitScriptInitCommand(t *testing.T) { } require.NoError(t, ctx.Err(), "init script did not execute for prefixed env vars") - _, err = runEnvbuilder(t, options{env: []string{ + _, err = runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), fmt.Sprintf(`INIT_SCRIPT=wget -O - %q`, initSrv.URL), @@ -129,7 +132,7 @@ RUN printf "%%s\n" \ "Dockerfile": dockerFile, }, }) - _, err := runEnvbuilder(t, options{env: []string{ + _, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), }}) @@ -160,7 +163,7 @@ RUN mkdir -p /myapp/somedir \ "Dockerfile": dockerFile, }, }) - _, err := runEnvbuilder(t, options{env: []string{ + _, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), }}) @@ -178,7 +181,7 @@ func TestForceSafe(t *testing.T) { "Dockerfile": "FROM " + testImageAlpine, }, }) - _, err := runEnvbuilder(t, options{env: []string{ + _, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), "KANIKO_DIR=/not/envbuilder", envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), @@ -194,7 +197,7 @@ func TestForceSafe(t *testing.T) { "Dockerfile": "FROM " + testImageAlpine, }, }) - _, err := runEnvbuilder(t, options{env: []string{ + _, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), "KANIKO_DIR=/not/envbuilder", envbuilderEnv("FORCE_SAFE", "true"), @@ -213,7 +216,7 @@ func TestFailsGitAuth(t *testing.T) { username: "kyle", password: "testing", }) - _, err := runEnvbuilder(t, options{env: []string{ + _, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), }}) require.ErrorContains(t, err, "authentication required") @@ -228,7 +231,7 @@ func TestSucceedsGitAuth(t *testing.T) { username: "kyle", password: "testing", }) - ctr, err := runEnvbuilder(t, options{env: []string{ + ctr, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), envbuilderEnv("GIT_USERNAME", "kyle"), @@ -252,7 +255,7 @@ func TestSucceedsGitAuthInURL(t *testing.T) { u, err := url.Parse(srv.URL) require.NoError(t, err) u.User = url.UserPassword("kyle", "testing") - ctr, err := runEnvbuilder(t, options{env: []string{ + ctr, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", u.String()), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), }}) @@ -330,7 +333,7 @@ func TestBuildFromDevcontainerWithFeatures(t *testing.T) { ".devcontainer/feature3/install.sh": "echo $GRAPE > /test3output", }, }) - ctr, err := runEnvbuilder(t, options{env: []string{ + ctr, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), }}) require.NoError(t, err) @@ -352,7 +355,7 @@ func TestBuildFromDockerfile(t *testing.T) { "Dockerfile": "FROM " + testImageAlpine, }, }) - ctr, err := runEnvbuilder(t, options{env: []string{ + ctr, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), envbuilderEnv("DOCKER_CONFIG_BASE64", base64.StdEncoding.EncodeToString([]byte(`{"experimental": "enabled"}`))), @@ -374,7 +377,7 @@ func TestBuildPrintBuildOutput(t *testing.T) { "Dockerfile": "FROM " + testImageAlpine + "\nRUN echo hello", }, }) - ctr, err := runEnvbuilder(t, options{env: []string{ + ctr, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), }}) @@ -408,7 +411,7 @@ func TestBuildIgnoreVarRunSecrets(t *testing.T) { require.NoError(t, err) t.Run("ReadWrite", func(t *testing.T) { - ctr, err := runEnvbuilder(t, options{ + ctr, err := runEnvbuilder(t, runOpts{ env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), @@ -422,7 +425,7 @@ func TestBuildIgnoreVarRunSecrets(t *testing.T) { }) t.Run("ReadOnly", func(t *testing.T) { - ctr, err := runEnvbuilder(t, options{ + ctr, err := runEnvbuilder(t, runOpts{ env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), @@ -443,7 +446,7 @@ func TestBuildWithSetupScript(t *testing.T) { "Dockerfile": "FROM " + testImageAlpine, }, }) - ctr, err := runEnvbuilder(t, options{env: []string{ + ctr, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), envbuilderEnv("SETUP_SCRIPT", "echo \"INIT_ARGS=-c 'echo hi > /wow && sleep infinity'\" >> $ENVBUILDER_ENV"), @@ -469,7 +472,7 @@ func TestBuildFromDevcontainerInCustomPath(t *testing.T) { ".devcontainer/custom/Dockerfile": "FROM " + testImageUbuntu, }, }) - ctr, err := runEnvbuilder(t, options{env: []string{ + ctr, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DEVCONTAINER_DIR", ".devcontainer/custom"), }}) @@ -494,7 +497,7 @@ func TestBuildFromDevcontainerInSubfolder(t *testing.T) { ".devcontainer/subfolder/Dockerfile": "FROM " + testImageUbuntu, }, }) - ctr, err := runEnvbuilder(t, options{env: []string{ + ctr, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), }}) require.NoError(t, err) @@ -518,7 +521,7 @@ func TestBuildFromDevcontainerInRoot(t *testing.T) { "Dockerfile": "FROM " + testImageUbuntu, }, }) - ctr, err := runEnvbuilder(t, options{env: []string{ + ctr, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), }}) require.NoError(t, err) @@ -534,7 +537,7 @@ func TestBuildCustomCertificates(t *testing.T) { }, tls: true, }) - ctr, err := runEnvbuilder(t, options{env: []string{ + ctr, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), envbuilderEnv("SSL_CERT_BASE64", base64.StdEncoding.EncodeToString(pem.EncodeToMemory(&pem.Block{ @@ -555,7 +558,7 @@ func TestBuildStopStartCached(t *testing.T) { "Dockerfile": "FROM " + testImageAlpine, }, }) - ctr, err := runEnvbuilder(t, options{env: []string{ + ctr, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), envbuilderEnv("SKIP_REBUILD", "true"), @@ -586,7 +589,7 @@ func TestCloneFailsFallback(t *testing.T) { t.Parallel() t.Run("BadRepo", func(t *testing.T) { t.Parallel() - _, err := runEnvbuilder(t, options{env: []string{ + _, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", "bad-value"), }}) require.ErrorContains(t, err, envbuilder.ErrNoFallbackImage.Error()) @@ -603,7 +606,7 @@ func TestBuildFailsFallback(t *testing.T) { "Dockerfile": "bad syntax", }, }) - _, err := runEnvbuilder(t, options{env: []string{ + _, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), }}) @@ -619,7 +622,7 @@ func TestBuildFailsFallback(t *testing.T) { RUN exit 1`, }, }) - _, err := runEnvbuilder(t, options{env: []string{ + _, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), }}) @@ -633,7 +636,7 @@ RUN exit 1`, ".devcontainer/devcontainer.json": "not json", }, }) - _, err := runEnvbuilder(t, options{env: []string{ + _, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), }}) require.ErrorContains(t, err, envbuilder.ErrNoFallbackImage.Error()) @@ -645,7 +648,7 @@ RUN exit 1`, ".devcontainer/devcontainer.json": "{}", }, }) - ctr, err := runEnvbuilder(t, options{env: []string{ + ctr, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("FALLBACK_IMAGE", testImageAlpine), }}) @@ -663,7 +666,7 @@ func TestExitBuildOnFailure(t *testing.T) { "Dockerfile": "bad syntax", }, }) - _, err := runEnvbuilder(t, options{env: []string{ + _, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), envbuilderEnv("FALLBACK_IMAGE", testImageAlpine), @@ -697,7 +700,7 @@ func TestContainerEnv(t *testing.T) { ".devcontainer/Dockerfile": "FROM " + testImageAlpine + "\nENV FROM_DOCKERFILE=foo", }, }) - ctr, err := runEnvbuilder(t, options{env: []string{ + ctr, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("EXPORT_ENV_FILE", "/env"), }}) @@ -730,7 +733,7 @@ func TestUnsetOptionsEnv(t *testing.T) { ".devcontainer/Dockerfile": "FROM " + testImageAlpine + "\nENV FROM_DOCKERFILE=foo", }, }) - ctr, err := runEnvbuilder(t, options{env: []string{ + ctr, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), "GIT_URL", srv.URL, envbuilderEnv("GIT_PASSWORD", "supersecret"), @@ -741,13 +744,13 @@ func TestUnsetOptionsEnv(t *testing.T) { require.NoError(t, err) output := execContainer(t, ctr, "cat /root/env.txt") - var os envbuilder.Options + var os options.Options for _, s := range strings.Split(strings.TrimSpace(output), "\n") { for _, o := range os.CLI() { if strings.HasPrefix(s, o.Env) { assert.Fail(t, "environment variable should be stripped when running init script", s) } - optWithoutPrefix := strings.TrimPrefix(o.Env, envbuilder.WithEnvPrefix("")) + optWithoutPrefix := strings.TrimPrefix(o.Env, options.WithEnvPrefix("")) if strings.HasPrefix(s, optWithoutPrefix) { assert.Fail(t, "environment variable should be stripped when running init script", s) } @@ -777,7 +780,7 @@ func TestLifecycleScripts(t *testing.T) { ".devcontainer/Dockerfile": "FROM " + testImageAlpine + "\nUSER nobody", }, }) - ctr, err := runEnvbuilder(t, options{env: []string{ + ctr, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), }}) require.NoError(t, err) @@ -816,7 +819,7 @@ RUN chmod +x /bin/init.sh USER nobody`, }, }) - ctr, err := runEnvbuilder(t, options{env: []string{ + ctr, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("POST_START_SCRIPT_PATH", "/tmp/post-start.sh"), envbuilderEnv("INIT_COMMAND", "/bin/init.sh"), @@ -850,7 +853,7 @@ func TestPrivateRegistry(t *testing.T) { "Dockerfile": "FROM " + image, }, }) - _, err := runEnvbuilder(t, options{env: []string{ + _, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), }}) @@ -879,7 +882,7 @@ func TestPrivateRegistry(t *testing.T) { }) require.NoError(t, err) - _, err = runEnvbuilder(t, options{env: []string{ + _, err = runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), envbuilderEnv("DOCKER_CONFIG_BASE64", base64.StdEncoding.EncodeToString(config)), @@ -911,7 +914,7 @@ func TestPrivateRegistry(t *testing.T) { }) require.NoError(t, err) - _, err = runEnvbuilder(t, options{env: []string{ + _, err = runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), envbuilderEnv("DOCKER_CONFIG_BASE64", base64.StdEncoding.EncodeToString(config)), @@ -968,7 +971,7 @@ func setupPassthroughRegistry(t *testing.T, image string, opts *setupPassthrough } func TestNoMethodFails(t *testing.T) { - _, err := runEnvbuilder(t, options{env: []string{}}) + _, err := runEnvbuilder(t, runOpts{env: []string{}}) require.ErrorContains(t, err, envbuilder.ErrNoFallbackImage.Error()) } @@ -1042,7 +1045,7 @@ COPY %s .`, testImageAlpine, inclFile) srv := createGitServer(t, gitServerOptions{ files: tc.files, }) - _, err := runEnvbuilder(t, options{env: []string{ + _, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DOCKERFILE_PATH", tc.dockerfilePath), envbuilderEnv("BUILD_CONTEXT_PATH", tc.buildContextPath), @@ -1090,7 +1093,7 @@ RUN date --utc > /root/date.txt`, testImageAlpine), require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") // When: we run envbuilder with GET_CACHED_IMAGE - _, err = runEnvbuilder(t, options{env: []string{ + _, err = runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), envbuilderEnv("GET_CACHED_IMAGE", "1"), @@ -1101,7 +1104,7 @@ RUN date --utc > /root/date.txt`, testImageAlpine), require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") // When: we run envbuilder with PUSH_IMAGE set - _, err = runEnvbuilder(t, options{env: []string{ + _, err = runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), }}) @@ -1112,7 +1115,7 @@ RUN date --utc > /root/date.txt`, testImageAlpine), require.ErrorContains(t, err, "MANIFEST_UNKNOWN", "expected image to not be present before build + push") // Then: re-running envbuilder with GET_CACHED_IMAGE should succeed - _, err = runEnvbuilder(t, options{env: []string{ + _, err = runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), envbuilderEnv("GET_CACHED_IMAGE", "1"), @@ -1150,7 +1153,7 @@ RUN date --utc > /root/date.txt`, testImageAlpine), require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") // When: we run envbuilder with GET_CACHED_IMAGE - _, err = runEnvbuilder(t, options{env: []string{ + _, err = runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), envbuilderEnv("GET_CACHED_IMAGE", "1"), @@ -1161,7 +1164,7 @@ RUN date --utc > /root/date.txt`, testImageAlpine), require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") // When: we run envbuilder with PUSH_IMAGE set - _, err = runEnvbuilder(t, options{env: []string{ + _, err = runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), envbuilderEnv("PUSH_IMAGE", "1"), @@ -1184,7 +1187,7 @@ RUN date --utc > /root/date.txt`, testImageAlpine), } // Then: re-running envbuilder with GET_CACHED_IMAGE should succeed - ctrID, err := runEnvbuilder(t, options{env: []string{ + ctrID, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), envbuilderEnv("GET_CACHED_IMAGE", "1"), @@ -1283,7 +1286,7 @@ RUN date --utc > /root/date.txt`, testImageAlpine), require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") // When: we run envbuilder with GET_CACHED_IMAGE - _, err = runEnvbuilder(t, options{env: []string{ + _, err = runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), envbuilderEnv("GET_CACHED_IMAGE", "1"), @@ -1294,7 +1297,7 @@ RUN date --utc > /root/date.txt`, testImageAlpine), require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") // When: we run envbuilder with PUSH_IMAGE set - _, err = runEnvbuilder(t, options{env: []string{ + _, err = runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), envbuilderEnv("PUSH_IMAGE", "1"), @@ -1307,7 +1310,7 @@ RUN date --utc > /root/date.txt`, testImageAlpine), require.NoError(t, err, "expected image to be present after build + push") // Then: re-running envbuilder with GET_CACHED_IMAGE should succeed - _, err = runEnvbuilder(t, options{env: []string{ + _, err = runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), envbuilderEnv("GET_CACHED_IMAGE", "1"), @@ -1351,7 +1354,7 @@ RUN date --utc > /root/date.txt`, testImageAlpine), require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") // When: we run envbuilder with GET_CACHED_IMAGE - _, err = runEnvbuilder(t, options{env: []string{ + _, err = runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), envbuilderEnv("GET_CACHED_IMAGE", "1"), @@ -1362,7 +1365,7 @@ RUN date --utc > /root/date.txt`, testImageAlpine), require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") // When: we run envbuilder with PUSH_IMAGE set - _, err = runEnvbuilder(t, options{env: []string{ + _, err = runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), envbuilderEnv("PUSH_IMAGE", "1"), @@ -1404,7 +1407,7 @@ COPY --from=a /root/date.txt /date.txt`, testImageAlpine, testImageAlpine), require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") // When: we run envbuilder with GET_CACHED_IMAGE - _, err = runEnvbuilder(t, options{env: []string{ + _, err = runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), envbuilderEnv("GET_CACHED_IMAGE", "1"), @@ -1416,7 +1419,7 @@ COPY --from=a /root/date.txt /date.txt`, testImageAlpine, testImageAlpine), require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") // When: we run envbuilder with PUSH_IMAGE set - ctrID, err := runEnvbuilder(t, options{env: []string{ + ctrID, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), envbuilderEnv("PUSH_IMAGE", "1"), @@ -1432,7 +1435,7 @@ COPY --from=a /root/date.txt /date.txt`, testImageAlpine, testImageAlpine), require.NoError(t, err, "expected image to be present after build + push") // Then: re-running envbuilder with GET_CACHED_IMAGE should succeed - _, err = runEnvbuilder(t, options{env: []string{ + _, err = runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), envbuilderEnv("GET_CACHED_IMAGE", "1"), @@ -1463,7 +1466,7 @@ RUN date --utc > /root/date.txt`, testImageAlpine), }) // When: we run envbuilder with PUSH_IMAGE set but no cache repo set - _, err := runEnvbuilder(t, options{env: []string{ + _, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("PUSH_IMAGE", "1"), }}) @@ -1499,7 +1502,7 @@ RUN date --utc > /root/date.txt`, testImageAlpine), notRegURL := strings.TrimPrefix(notRegSrv.URL, "http://") + "/test" // When: we run envbuilder with PUSH_IMAGE set - _, err := runEnvbuilder(t, options{env: []string{ + _, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", notRegURL), envbuilderEnv("PUSH_IMAGE", "1"), @@ -1535,7 +1538,7 @@ USER test // Run envbuilder with a Docker volume mounted to homedir volName := fmt.Sprintf("%s%d-home", t.Name(), time.Now().Unix()) - ctr, err := runEnvbuilder(t, options{env: []string{ + ctr, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), }, volumes: map[string]string{volName: "/home/test"}}) require.NoError(t, err) @@ -1656,7 +1659,7 @@ func cleanOldEnvbuilders() { } } -type options struct { +type runOpts struct { binds []string env []string volumes map[string]string @@ -1664,7 +1667,7 @@ type options struct { // runEnvbuilder starts the envbuilder container with the given environment // variables and returns the container ID. -func runEnvbuilder(t *testing.T, options options) (string, error) { +func runEnvbuilder(t *testing.T, opts runOpts) (string, error) { t.Helper() ctx := context.Background() cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) @@ -1673,7 +1676,7 @@ func runEnvbuilder(t *testing.T, options options) (string, error) { cli.Close() }) mounts := make([]mount.Mount, 0) - for volName, volPath := range options.volumes { + for volName, volPath := range opts.volumes { mounts = append(mounts, mount.Mount{ Type: mount.TypeVolume, Source: volName, @@ -1689,13 +1692,13 @@ func runEnvbuilder(t *testing.T, options options) (string, error) { } ctr, err := cli.ContainerCreate(ctx, &container.Config{ Image: "envbuilder:latest", - Env: options.env, + Env: opts.env, Labels: map[string]string{ testContainerLabel: "true", }, }, &container.HostConfig{ NetworkMode: container.NetworkMode("host"), - Binds: options.binds, + Binds: opts.binds, Mounts: mounts, }, nil, nil, "") require.NoError(t, err) @@ -1784,5 +1787,5 @@ func streamContainerLogs(t *testing.T, cli *client.Client, containerID string) ( } func envbuilderEnv(env string, value string) string { - return fmt.Sprintf("%s=%s", envbuilder.WithEnvPrefix(env), value) + return fmt.Sprintf("%s=%s", options.WithEnvPrefix(env), value) } diff --git a/options.go b/options/options.go similarity index 97% rename from options.go rename to options/options.go index 76eddb60..dd5ee8b9 100644 --- a/options.go +++ b/options/options.go @@ -1,6 +1,7 @@ -package envbuilder +package options import ( + "os" "strings" "github.com/coder/envbuilder/internal/log" @@ -493,3 +494,22 @@ func skipDeprecatedOptions(options []serpent.Option) []serpent.Option { return activeOptions } + +// UnsetEnv unsets all environment variables that are used +// to configure the options. +func UnsetEnv() { + var o Options + for _, opt := range o.CLI() { + if opt.Env == "" { + continue + } + // Do not strip options that do not have the magic prefix! + // For example, CODER_AGENT_URL, CODER_AGENT_TOKEN, CODER_AGENT_SUBSYSTEM. + if !strings.HasPrefix(opt.Env, envPrefix) { + continue + } + // Strip both with and without prefix. + _ = os.Unsetenv(opt.Env) + _ = os.Unsetenv(strings.TrimPrefix(opt.Env, envPrefix)) + } +} diff --git a/options_test.go b/options/options_test.go similarity index 88% rename from options_test.go rename to options/options_test.go index e32af9e6..bf7a216c 100644 --- a/options_test.go +++ b/options/options_test.go @@ -1,4 +1,4 @@ -package envbuilder_test +package options_test import ( "bytes" @@ -6,7 +6,8 @@ import ( "os" "testing" - "github.com/coder/envbuilder" + "github.com/coder/envbuilder/options" + "github.com/coder/serpent" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -16,50 +17,50 @@ import ( func TestEnvOptionParsing(t *testing.T) { t.Run("string", func(t *testing.T) { const val = "setup.sh" - t.Setenv(envbuilder.WithEnvPrefix("SETUP_SCRIPT"), val) + t.Setenv(options.WithEnvPrefix("SETUP_SCRIPT"), val) o := runCLI() require.Equal(t, o.SetupScript, val) }) t.Run("int", func(t *testing.T) { - t.Setenv(envbuilder.WithEnvPrefix("CACHE_TTL_DAYS"), "7") + t.Setenv(options.WithEnvPrefix("CACHE_TTL_DAYS"), "7") o := runCLI() require.Equal(t, o.CacheTTLDays, int64(7)) }) t.Run("string array", func(t *testing.T) { - t.Setenv(envbuilder.WithEnvPrefix("IGNORE_PATHS"), "/var,/temp") + t.Setenv(options.WithEnvPrefix("IGNORE_PATHS"), "/var,/temp") o := runCLI() require.Equal(t, o.IgnorePaths, []string{"/var", "/temp"}) }) t.Run("bool", func(t *testing.T) { t.Run("lowercase", func(t *testing.T) { - t.Setenv(envbuilder.WithEnvPrefix("SKIP_REBUILD"), "true") - t.Setenv(envbuilder.WithEnvPrefix("GIT_CLONE_SINGLE_BRANCH"), "false") + t.Setenv(options.WithEnvPrefix("SKIP_REBUILD"), "true") + t.Setenv(options.WithEnvPrefix("GIT_CLONE_SINGLE_BRANCH"), "false") o := runCLI() require.True(t, o.SkipRebuild) require.False(t, o.GitCloneSingleBranch) }) t.Run("uppercase", func(t *testing.T) { - t.Setenv(envbuilder.WithEnvPrefix("SKIP_REBUILD"), "TRUE") - t.Setenv(envbuilder.WithEnvPrefix("GIT_CLONE_SINGLE_BRANCH"), "FALSE") + t.Setenv(options.WithEnvPrefix("SKIP_REBUILD"), "TRUE") + t.Setenv(options.WithEnvPrefix("GIT_CLONE_SINGLE_BRANCH"), "FALSE") o := runCLI() require.True(t, o.SkipRebuild) require.False(t, o.GitCloneSingleBranch) }) t.Run("numeric", func(t *testing.T) { - t.Setenv(envbuilder.WithEnvPrefix("SKIP_REBUILD"), "1") - t.Setenv(envbuilder.WithEnvPrefix("GIT_CLONE_SINGLE_BRANCH"), "0") + t.Setenv(options.WithEnvPrefix("SKIP_REBUILD"), "1") + t.Setenv(options.WithEnvPrefix("GIT_CLONE_SINGLE_BRANCH"), "0") o := runCLI() require.True(t, o.SkipRebuild) require.False(t, o.GitCloneSingleBranch) }) t.Run("empty", func(t *testing.T) { - t.Setenv(envbuilder.WithEnvPrefix("GIT_CLONE_SINGLE_BRANCH"), "") + t.Setenv(options.WithEnvPrefix("GIT_CLONE_SINGLE_BRANCH"), "") o := runCLI() require.False(t, o.GitCloneSingleBranch) }) @@ -142,7 +143,7 @@ var updateCLIOutputGoldenFiles = flag.Bool("update", false, "update options CLI // TestCLIOutput tests that the default CLI output is as expected. func TestCLIOutput(t *testing.T) { - var o envbuilder.Options + var o options.Options cmd := serpent.Command{ Use: "envbuilder", Options: o.CLI(), @@ -171,8 +172,8 @@ func TestCLIOutput(t *testing.T) { } } -func runCLI() envbuilder.Options { - var o envbuilder.Options +func runCLI() options.Options { + var o options.Options cmd := serpent.Command{ Options: o.CLI(), Handler: func(inv *serpent.Invocation) error { diff --git a/testdata/options.golden b/options/testdata/options.golden similarity index 100% rename from testdata/options.golden rename to options/testdata/options.golden diff --git a/scripts/docsgen/main.go b/scripts/docsgen/main.go index c79995cf..83d992c4 100644 --- a/scripts/docsgen/main.go +++ b/scripts/docsgen/main.go @@ -5,7 +5,7 @@ import ( "os" "strings" - "github.com/coder/envbuilder" + "github.com/coder/envbuilder/options" ) const ( @@ -26,7 +26,7 @@ func main() { panic("start or end section comments not found in the file.") } - var options envbuilder.Options + var options options.Options mkd := "\n## Environment Variables\n\n" + options.Markdown() modifiedContent := readmeContent[:startIndex+len(startSection)] + mkd + readmeContent[endIndex:] From b6cc003b82fb4292be1aaa435c3f545900fec2e3 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 23 Jul 2024 11:58:26 +0100 Subject: [PATCH 11/81] chore: extract git operations to separate package (#279) (cherry picked from commit e296431bbd87ac361fcaf61dfd7eef0efe0ce961) --- envbuilder.go | 7 ++++--- git.go => git/git.go | 2 +- git_test.go => git/git_test.go | 37 +++++++++++++++++----------------- 3 files changed, 24 insertions(+), 22 deletions(-) rename git.go => git/git.go (99%) rename git_test.go => git/git_test.go (91%) diff --git a/envbuilder.go b/envbuilder.go index 3cbdab04..7c5c380a 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -24,6 +24,7 @@ import ( "syscall" "time" + "github.com/coder/envbuilder/git" "github.com/coder/envbuilder/options" "github.com/GoogleContainerTools/kaniko/pkg/config" @@ -204,7 +205,7 @@ func Run(ctx context.Context, opts options.Options) error { } }() - cloneOpts := CloneRepoOptions{ + cloneOpts := git.CloneRepoOptions{ Path: opts.WorkspaceFolder, Storage: opts.Filesystem, Insecure: opts.Insecure, @@ -214,7 +215,7 @@ func Run(ctx context.Context, opts options.Options) error { CABundle: caBundle, } - cloneOpts.RepoAuth = SetupRepoAuth(&opts) + cloneOpts.RepoAuth = git.SetupRepoAuth(&opts) if opts.GitHTTPProxyURL != "" { cloneOpts.ProxyOptions = transport.ProxyOptions{ URL: opts.GitHTTPProxyURL, @@ -222,7 +223,7 @@ func Run(ctx context.Context, opts options.Options) error { } cloneOpts.RepoURL = opts.GitURL - cloned, fallbackErr = CloneRepo(ctx, cloneOpts) + cloned, fallbackErr = git.CloneRepo(ctx, cloneOpts) if fallbackErr == nil { if cloned { endStage("📦 Cloned repository!") diff --git a/git.go b/git/git.go similarity index 99% rename from git.go rename to git/git.go index eb75b654..019c68ef 100644 --- a/git.go +++ b/git/git.go @@ -1,4 +1,4 @@ -package envbuilder +package git import ( "context" diff --git a/git_test.go b/git/git_test.go similarity index 91% rename from git_test.go rename to git/git_test.go index 842cf2c9..a65e90f8 100644 --- a/git_test.go +++ b/git/git_test.go @@ -1,4 +1,4 @@ -package envbuilder_test +package git_test import ( "context" @@ -12,9 +12,10 @@ import ( "regexp" "testing" + "github.com/coder/envbuilder/git" + "github.com/coder/envbuilder/options" - "github.com/coder/envbuilder" "github.com/coder/envbuilder/internal/log" "github.com/coder/envbuilder/testutil/gittest" "github.com/coder/envbuilder/testutil/mwtest" @@ -90,7 +91,7 @@ func TestCloneRepo(t *testing.T) { clientFS := memfs.New() // A repo already exists! _ = gittest.NewRepo(t, clientFS) - cloned, err := envbuilder.CloneRepo(context.Background(), envbuilder.CloneRepoOptions{ + cloned, err := git.CloneRepo(context.Background(), git.CloneRepoOptions{ Path: "/", RepoURL: srv.URL, Storage: clientFS, @@ -108,7 +109,7 @@ func TestCloneRepo(t *testing.T) { srv := httptest.NewServer(authMW(gittest.NewServer(srvFS))) clientFS := memfs.New() - cloned, err := envbuilder.CloneRepo(context.Background(), envbuilder.CloneRepoOptions{ + cloned, err := git.CloneRepo(context.Background(), git.CloneRepoOptions{ Path: "/workspace", RepoURL: srv.URL, Storage: clientFS, @@ -145,7 +146,7 @@ func TestCloneRepo(t *testing.T) { authURL.User = url.UserPassword(tc.username, tc.password) clientFS := memfs.New() - cloned, err := envbuilder.CloneRepo(context.Background(), envbuilder.CloneRepoOptions{ + cloned, err := git.CloneRepo(context.Background(), git.CloneRepoOptions{ Path: "/workspace", RepoURL: authURL.String(), Storage: clientFS, @@ -184,7 +185,7 @@ func TestCloneRepoSSH(t *testing.T) { gitURL := tr.String() clientFS := memfs.New() - cloned, err := envbuilder.CloneRepo(context.Background(), envbuilder.CloneRepoOptions{ + cloned, err := git.CloneRepo(context.Background(), git.CloneRepoOptions{ Path: "/workspace", RepoURL: gitURL, Storage: clientFS, @@ -216,7 +217,7 @@ func TestCloneRepoSSH(t *testing.T) { clientFS := memfs.New() anotherKey := randKeygen(t) - cloned, err := envbuilder.CloneRepo(context.Background(), envbuilder.CloneRepoOptions{ + cloned, err := git.CloneRepo(context.Background(), git.CloneRepoOptions{ Path: "/workspace", RepoURL: gitURL, Storage: clientFS, @@ -246,7 +247,7 @@ func TestCloneRepoSSH(t *testing.T) { gitURL := tr.String() clientFS := memfs.New() - cloned, err := envbuilder.CloneRepo(context.Background(), envbuilder.CloneRepoOptions{ + cloned, err := git.CloneRepo(context.Background(), git.CloneRepoOptions{ Path: "/workspace", RepoURL: gitURL, Storage: clientFS, @@ -270,7 +271,7 @@ func TestSetupRepoAuth(t *testing.T) { opts := &options.Options{ Logger: testLog(t), } - auth := envbuilder.SetupRepoAuth(opts) + auth := git.SetupRepoAuth(opts) require.Nil(t, auth) }) @@ -279,7 +280,7 @@ func TestSetupRepoAuth(t *testing.T) { GitURL: "http://host.tld/repo", Logger: testLog(t), } - auth := envbuilder.SetupRepoAuth(opts) + auth := git.SetupRepoAuth(opts) require.Nil(t, auth) }) @@ -290,7 +291,7 @@ func TestSetupRepoAuth(t *testing.T) { GitPassword: "pass", Logger: testLog(t), } - auth := envbuilder.SetupRepoAuth(opts) + auth := git.SetupRepoAuth(opts) ba, ok := auth.(*githttp.BasicAuth) require.True(t, ok) require.Equal(t, opts.GitUsername, ba.Username) @@ -304,7 +305,7 @@ func TestSetupRepoAuth(t *testing.T) { GitPassword: "pass", Logger: testLog(t), } - auth := envbuilder.SetupRepoAuth(opts) + auth := git.SetupRepoAuth(opts) ba, ok := auth.(*githttp.BasicAuth) require.True(t, ok) require.Equal(t, opts.GitUsername, ba.Username) @@ -318,7 +319,7 @@ func TestSetupRepoAuth(t *testing.T) { GitSSHPrivateKeyPath: kPath, Logger: testLog(t), } - auth := envbuilder.SetupRepoAuth(opts) + auth := git.SetupRepoAuth(opts) _, ok := auth.(*gitssh.PublicKeys) require.True(t, ok) }) @@ -330,7 +331,7 @@ func TestSetupRepoAuth(t *testing.T) { GitSSHPrivateKeyPath: kPath, Logger: testLog(t), } - auth := envbuilder.SetupRepoAuth(opts) + auth := git.SetupRepoAuth(opts) _, ok := auth.(*gitssh.PublicKeys) require.True(t, ok) }) @@ -343,7 +344,7 @@ func TestSetupRepoAuth(t *testing.T) { GitSSHPrivateKeyPath: kPath, Logger: testLog(t), } - auth := envbuilder.SetupRepoAuth(opts) + auth := git.SetupRepoAuth(opts) _, ok := auth.(*gitssh.PublicKeys) require.True(t, ok) }) @@ -356,7 +357,7 @@ func TestSetupRepoAuth(t *testing.T) { GitUsername: "user", Logger: testLog(t), } - auth := envbuilder.SetupRepoAuth(opts) + auth := git.SetupRepoAuth(opts) _, ok := auth.(*gitssh.PublicKeys) require.True(t, ok) }) @@ -368,7 +369,7 @@ func TestSetupRepoAuth(t *testing.T) { GitSSHPrivateKeyPath: kPath, Logger: testLog(t), } - auth := envbuilder.SetupRepoAuth(opts) + auth := git.SetupRepoAuth(opts) pk, ok := auth.(*gitssh.PublicKeys) require.True(t, ok) require.NotNil(t, pk.Signer) @@ -382,7 +383,7 @@ func TestSetupRepoAuth(t *testing.T) { GitURL: "ssh://git@host.tld:repo/path", Logger: testLog(t), } - auth := envbuilder.SetupRepoAuth(opts) + auth := git.SetupRepoAuth(opts) require.Nil(t, auth) // TODO: actually test SSH_AUTH_SOCK }) } From 9aa5fe9517cc014f7b2dea6842d24a4e751b5895 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Jul 2024 13:21:07 +0100 Subject: [PATCH 12/81] chore: bump github.com/google/nftables from 0.1.1-0.20230115205135-9aa6fdf5a28c to 0.2.0 (#270) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Cian Johnston (cherry picked from commit ac3c34c7b6b117daec49dc8f70f69c935ef7c8fa) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c2dd1a30..6eb8be9f 100644 --- a/go.mod +++ b/go.mod @@ -147,7 +147,7 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.2 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/google/nftables v0.1.1-0.20230115205135-9aa6fdf5a28c // indirect + github.com/google/nftables v0.2.0 // indirect github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect github.com/gorilla/handlers v1.5.1 // indirect github.com/gorilla/mux v1.8.1 // indirect diff --git a/go.sum b/go.sum index fe8ebf87..f51d27b3 100644 --- a/go.sum +++ b/go.sum @@ -386,8 +386,8 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/nftables v0.1.1-0.20230115205135-9aa6fdf5a28c h1:06RMfw+TMMHtRuUOroMeatRCCgSMWXCJQeABvHU69YQ= -github.com/google/nftables v0.1.1-0.20230115205135-9aa6fdf5a28c/go.mod h1:BVIYo3cdnT4qSylnYqcd5YtmXhr51cJPGtnLBe/uLBU= +github.com/google/nftables v0.2.0 h1:PbJwaBmbVLzpeldoeUKGkE2RjstrjPKMl6oLrfEJ6/8= +github.com/google/nftables v0.2.0/go.mod h1:Beg6V6zZ3oEn0JuiUQ4wqwuyqqzasOltcoXPtgLbFp4= github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b h1:h9U78+dx9a4BKdQkBBos92HalKpaGKHrp+3Uo6yTodo= github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= From 6fc1cfa85fa86fce51eb302a1653c6034c13314c Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 29 Jul 2024 17:20:55 +0100 Subject: [PATCH 13/81] chore: extract constants package (#282) Signed-off-by: Cian Johnston (cherry picked from commit cc0b4c5cdf7af69033cb2da0ece9c11eef3b296e) --- constants/constants.go | 31 ++++++++++++++++++ envbuilder.go | 58 ++++++++++----------------------- envbuilder_test.go | 6 ++-- integration/integration_test.go | 16 ++++----- 4 files changed, 60 insertions(+), 51 deletions(-) create mode 100644 constants/constants.go diff --git a/constants/constants.go b/constants/constants.go new file mode 100644 index 00000000..ccdfcb8c --- /dev/null +++ b/constants/constants.go @@ -0,0 +1,31 @@ +package constants + +import ( + "errors" + "path/filepath" +) + +const ( + // WorkspacesDir is the path to the directory where + // all workspaces are stored by default. + WorkspacesDir = "/workspaces" + + // EmptyWorkspaceDir is the path to a workspace that has + // nothing going on... it's empty! + EmptyWorkspaceDir = WorkspacesDir + "/empty" + + // MagicDir is where all envbuilder related files are stored. + // This is a special directory that must not be modified + // by the user or images. + MagicDir = "/.envbuilder" +) + +var ( + ErrNoFallbackImage = errors.New("no fallback image has been specified") + + // MagicFile is a file that is created in the workspace + // when envbuilder has already been run. This is used + // to skip building when a container is restarting. + // e.g. docker stop -> docker start + MagicFile = filepath.Join(MagicDir, "built") +) diff --git a/envbuilder.go b/envbuilder.go index 7c5c380a..cafc4fcd 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -24,6 +24,7 @@ import ( "syscall" "time" + "github.com/coder/envbuilder/constants" "github.com/coder/envbuilder/git" "github.com/coder/envbuilder/options" @@ -53,31 +54,6 @@ import ( "golang.org/x/xerrors" ) -const ( - // WorkspacesDir is the path to the directory where - // all workspaces are stored by default. - WorkspacesDir = "/workspaces" - - // EmptyWorkspaceDir is the path to a workspace that has - // nothing going on... it's empty! - EmptyWorkspaceDir = WorkspacesDir + "/empty" - - // MagicDir is where all envbuilder related files are stored. - // This is a special directory that must not be modified - // by the user or images. - MagicDir = "/.envbuilder" -) - -var ( - ErrNoFallbackImage = errors.New("no fallback image has been specified") - - // MagicFile is a file that is created in the workspace - // when envbuilder has already been run. This is used - // to skip building when a container is restarting. - // e.g. docker stop -> docker start - MagicFile = filepath.Join(MagicDir, "built") -) - // DockerConfig represents the Docker configuration file. type DockerConfig configfile.ConfigFile @@ -171,7 +147,7 @@ func Run(ctx context.Context, opts options.Options) error { if err != nil { return fmt.Errorf("parse docker config: %w", err) } - err = os.WriteFile(filepath.Join(MagicDir, "config.json"), decoded, 0o644) + err = os.WriteFile(filepath.Join(constants.MagicDir, "config.json"), decoded, 0o644) if err != nil { return fmt.Errorf("write docker config: %w", err) } @@ -237,7 +213,7 @@ func Run(ctx context.Context, opts options.Options) error { } defaultBuildParams := func() (*devcontainer.Compiled, error) { - dockerfile := filepath.Join(MagicDir, "Dockerfile") + dockerfile := filepath.Join(constants.MagicDir, "Dockerfile") file, err := opts.Filesystem.OpenFile(dockerfile, os.O_CREATE|os.O_WRONLY, 0o644) if err != nil { return nil, err @@ -245,11 +221,11 @@ func Run(ctx context.Context, opts options.Options) error { defer file.Close() if opts.FallbackImage == "" { if fallbackErr != nil { - return nil, xerrors.Errorf("%s: %w", fallbackErr.Error(), ErrNoFallbackImage) + return nil, xerrors.Errorf("%s: %w", fallbackErr.Error(), constants.ErrNoFallbackImage) } // We can't use errors.Join here because our tests // don't support parsing a multiline error. - return nil, ErrNoFallbackImage + return nil, constants.ErrNoFallbackImage } content := "FROM " + opts.FallbackImage _, err = file.Write([]byte(content)) @@ -259,7 +235,7 @@ func Run(ctx context.Context, opts options.Options) error { return &devcontainer.Compiled{ DockerfilePath: dockerfile, DockerfileContent: content, - BuildContext: MagicDir, + BuildContext: constants.MagicDir, }, nil } @@ -301,7 +277,7 @@ func Run(ctx context.Context, opts options.Options) error { opts.Logger(log.LevelInfo, "No Dockerfile or image specified; falling back to the default image...") fallbackDockerfile = defaultParams.DockerfilePath } - buildParams, err = devContainer.Compile(opts.Filesystem, devcontainerDir, MagicDir, fallbackDockerfile, opts.WorkspaceFolder, false, os.LookupEnv) + buildParams, err = devContainer.Compile(opts.Filesystem, devcontainerDir, constants.MagicDir, fallbackDockerfile, opts.WorkspaceFolder, false, os.LookupEnv) if err != nil { return fmt.Errorf("compile devcontainer.json: %w", err) } @@ -399,7 +375,7 @@ func Run(ctx context.Context, opts options.Options) error { // So we add them to the default ignore list. See: // https://github.com/GoogleContainerTools/kaniko/blob/63be4990ca5a60bdf06ddc4d10aa4eca0c0bc714/cmd/executor/cmd/root.go#L136 ignorePaths := append([]string{ - MagicDir, + constants.MagicDir, opts.WorkspaceFolder, // See: https://github.com/coder/envbuilder/issues/37 "/etc/resolv.conf", @@ -441,7 +417,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) } // temp move of all ro mounts - tempRemountDest := filepath.Join("/", MagicDir, "mnt") + tempRemountDest := filepath.Join("/", constants.MagicDir, "mnt") // ignorePrefixes is a superset of ignorePaths that we pass to kaniko's // IgnoreList. ignorePrefixes := append([]string{"/dev", "/proc", "/sys"}, ignorePaths...) @@ -457,7 +433,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) skippedRebuild := false build := func() (v1.Image, error) { - _, err := opts.Filesystem.Stat(MagicFile) + _, err := opts.Filesystem.Stat(constants.MagicFile) if err == nil && opts.SkipRebuild { endStage := startStage("🏗️ Skipping build because of cache...") imageRef, err := devcontainer.ImageFromDockerfile(buildParams.DockerfileContent) @@ -640,7 +616,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) // Create the magic file to indicate that this build // has already been ran before! - file, err := opts.Filesystem.Create(MagicFile) + file, err := opts.Filesystem.Create(constants.MagicFile) if err != nil { return fmt.Errorf("create magic file: %w", err) } @@ -695,7 +671,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) // Remove the Docker config secret file! if opts.DockerConfigBase64 != "" { - c := filepath.Join(MagicDir, "config.json") + c := filepath.Join(constants.MagicDir, "config.json") err = os.Remove(c) if err != nil { if !errors.Is(err, fs.ErrNotExist) { @@ -855,7 +831,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) opts.Logger(log.LevelInfo, "=== Running the setup command %q as the root user...", opts.SetupScript) envKey := "ENVBUILDER_ENV" - envFile := filepath.Join("/", MagicDir, "environ") + envFile := filepath.Join("/", constants.MagicDir, "environ") file, err := os.Create(envFile) if err != nil { return fmt.Errorf("create environ file: %w", err) @@ -958,7 +934,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) // for a given repository URL. func DefaultWorkspaceFolder(repoURL string) (string, error) { if repoURL == "" { - return EmptyWorkspaceDir, nil + return constants.EmptyWorkspaceDir, nil } parsed, err := giturls.Parse(repoURL) if err != nil { @@ -967,7 +943,7 @@ func DefaultWorkspaceFolder(repoURL string) (string, error) { name := strings.Split(parsed.Path, "/") hasOwnerAndRepo := len(name) >= 2 if !hasOwnerAndRepo { - return EmptyWorkspaceDir, nil + return constants.EmptyWorkspaceDir, nil } repo := strings.TrimSuffix(name[len(name)-1], ".git") return fmt.Sprintf("/workspaces/%s", repo), nil @@ -1180,7 +1156,7 @@ func findDevcontainerJSON(options options.Options) (string, string, error) { // folks from unwittingly deleting their entire root directory. func maybeDeleteFilesystem(logger log.Func, force bool) error { kanikoDir, ok := os.LookupEnv("KANIKO_DIR") - if !ok || strings.TrimSpace(kanikoDir) != MagicDir { + if !ok || strings.TrimSpace(kanikoDir) != constants.MagicDir { if force { bailoutSecs := 10 logger(log.LevelWarn, "WARNING! BYPASSING SAFETY CHECK! THIS WILL DELETE YOUR ROOT FILESYSTEM!") @@ -1190,7 +1166,7 @@ func maybeDeleteFilesystem(logger log.Func, force bool) error { <-time.After(time.Second) } } else { - logger(log.LevelError, "KANIKO_DIR is not set to %s. Bailing!\n", MagicDir) + logger(log.LevelError, "KANIKO_DIR is not set to %s. Bailing!\n", constants.MagicDir) logger(log.LevelError, "To bypass this check, set FORCE_SAFE=true.") return errors.New("safety check failed") } diff --git a/envbuilder_test.go b/envbuilder_test.go index 6af599c9..0545bfce 100644 --- a/envbuilder_test.go +++ b/envbuilder_test.go @@ -4,6 +4,8 @@ import ( "testing" "github.com/coder/envbuilder" + "github.com/coder/envbuilder/constants" + "github.com/stretchr/testify/require" ) @@ -38,7 +40,7 @@ func TestDefaultWorkspaceFolder(t *testing.T) { { name: "empty", gitURL: "", - expected: envbuilder.EmptyWorkspaceDir, + expected: constants.EmptyWorkspaceDir, }, } for _, tt := range successTests { @@ -66,7 +68,7 @@ func TestDefaultWorkspaceFolder(t *testing.T) { t.Run(tt.name, func(t *testing.T) { dir, err := envbuilder.DefaultWorkspaceFolder(tt.invalidURL) require.NoError(t, err) - require.Equal(t, envbuilder.EmptyWorkspaceDir, dir) + require.Equal(t, constants.EmptyWorkspaceDir, dir) }) } } diff --git a/integration/integration_test.go b/integration/integration_test.go index e62bac02..b3fe7bef 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -21,10 +21,10 @@ import ( "testing" "time" - "github.com/coder/envbuilder/options" - "github.com/coder/envbuilder" + "github.com/coder/envbuilder/constants" "github.com/coder/envbuilder/devcontainer/features" + "github.com/coder/envbuilder/options" "github.com/coder/envbuilder/testutil/gittest" "github.com/coder/envbuilder/testutil/mwtest" "github.com/coder/envbuilder/testutil/registrytest" @@ -366,7 +366,7 @@ func TestBuildFromDockerfile(t *testing.T) { require.Equal(t, "hello", strings.TrimSpace(output)) // Verify that the Docker configuration secret file is removed - output = execContainer(t, ctr, "stat "+filepath.Join(envbuilder.MagicDir, "config.json")) + output = execContainer(t, ctr, "stat "+filepath.Join(constants.MagicDir, "config.json")) require.Contains(t, output, "No such file or directory") } @@ -592,7 +592,7 @@ func TestCloneFailsFallback(t *testing.T) { _, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", "bad-value"), }}) - require.ErrorContains(t, err, envbuilder.ErrNoFallbackImage.Error()) + require.ErrorContains(t, err, constants.ErrNoFallbackImage.Error()) }) } @@ -610,7 +610,7 @@ func TestBuildFailsFallback(t *testing.T) { envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), }}) - require.ErrorContains(t, err, envbuilder.ErrNoFallbackImage.Error()) + require.ErrorContains(t, err, constants.ErrNoFallbackImage.Error()) require.ErrorContains(t, err, "dockerfile parse error") }) t.Run("FailsBuild", func(t *testing.T) { @@ -626,7 +626,7 @@ RUN exit 1`, envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), }}) - require.ErrorContains(t, err, envbuilder.ErrNoFallbackImage.Error()) + require.ErrorContains(t, err, constants.ErrNoFallbackImage.Error()) }) t.Run("BadDevcontainer", func(t *testing.T) { t.Parallel() @@ -639,7 +639,7 @@ RUN exit 1`, _, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), }}) - require.ErrorContains(t, err, envbuilder.ErrNoFallbackImage.Error()) + require.ErrorContains(t, err, constants.ErrNoFallbackImage.Error()) }) t.Run("NoImageOrDockerfile", func(t *testing.T) { t.Parallel() @@ -972,7 +972,7 @@ func setupPassthroughRegistry(t *testing.T, image string, opts *setupPassthrough func TestNoMethodFails(t *testing.T) { _, err := runEnvbuilder(t, runOpts{env: []string{}}) - require.ErrorContains(t, err, envbuilder.ErrNoFallbackImage.Error()) + require.ErrorContains(t, err, constants.ErrNoFallbackImage.Error()) } func TestDockerfileBuildContext(t *testing.T) { From f7c2dc159d554bbfad360b7ecc88bdfca28446fc Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 29 Jul 2024 17:35:26 +0100 Subject: [PATCH 14/81] chore: extract option defaults to options package (#283) Signed-off-by: Cian Johnston (cherry picked from commit 09ce456ca8be4ef187824b59d0f329834bc6c885) --- envbuilder.go | 59 +---------------------- envbuilder_test.go | 73 ----------------------------- internal/chmodfs/chmodfs.go | 21 +++++++++ options/defaults.go | 58 +++++++++++++++++++++++ options/defaults_test.go | 93 +++++++++++++++++++++++++++++++++++++ 5 files changed, 174 insertions(+), 130 deletions(-) create mode 100644 internal/chmodfs/chmodfs.go create mode 100644 options/defaults.go create mode 100644 options/defaults_test.go diff --git a/envbuilder.go b/envbuilder.go index cafc4fcd..2a00c84c 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -32,7 +32,6 @@ import ( "github.com/GoogleContainerTools/kaniko/pkg/creds" "github.com/GoogleContainerTools/kaniko/pkg/executor" "github.com/GoogleContainerTools/kaniko/pkg/util" - giturls "github.com/chainguard-dev/git-urls" "github.com/coder/envbuilder/devcontainer" "github.com/coder/envbuilder/internal/ebutil" "github.com/coder/envbuilder/internal/log" @@ -42,8 +41,6 @@ import ( _ "github.com/distribution/distribution/v3/registry/storage/driver/filesystem" "github.com/docker/cli/cli/config/configfile" "github.com/fatih/color" - "github.com/go-git/go-billy/v5" - "github.com/go-git/go-billy/v5/osfs" "github.com/go-git/go-git/v5/plumbing/transport" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" @@ -62,23 +59,8 @@ type DockerConfig configfile.ConfigFile // Filesystem is the filesystem to use for all operations. // Defaults to the host filesystem. func Run(ctx context.Context, opts options.Options) error { - // Temporarily removed these from the default settings to prevent conflicts - // between current and legacy environment variables that add default values. - // Once the legacy environment variables are phased out, this can be - // reinstated to the previous default values. - if len(opts.IgnorePaths) == 0 { - opts.IgnorePaths = []string{ - "/var/run", - // KinD adds these paths to pods, so ignore them by default. - "/product_uuid", "/product_name", - } - } - if opts.InitScript == "" { - opts.InitScript = "sleep infinity" - } - if opts.InitCommand == "" { - opts.InitCommand = "/bin/sh" - } + opts.SetDefaults() + if opts.CacheRepo == "" && opts.PushImage { return fmt.Errorf("--cache-repo must be set when using --push-image") } @@ -91,16 +73,6 @@ func Run(ctx context.Context, opts options.Options) error { return fmt.Errorf("parse init args: %w", err) } } - if opts.Filesystem == nil { - opts.Filesystem = &osfsWithChmod{osfs.New("/")} - } - if opts.WorkspaceFolder == "" { - f, err := DefaultWorkspaceFolder(opts.GitURL) - if err != nil { - return err - } - opts.WorkspaceFolder = f - } stageNumber := 0 startStage := func(format string, args ...any) func(format string, args ...any) { @@ -930,25 +902,6 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) return nil } -// DefaultWorkspaceFolder returns the default workspace folder -// for a given repository URL. -func DefaultWorkspaceFolder(repoURL string) (string, error) { - if repoURL == "" { - return constants.EmptyWorkspaceDir, nil - } - parsed, err := giturls.Parse(repoURL) - if err != nil { - return "", err - } - name := strings.Split(parsed.Path, "/") - hasOwnerAndRepo := len(name) >= 2 - if !hasOwnerAndRepo { - return constants.EmptyWorkspaceDir, nil - } - repo := strings.TrimSuffix(name[len(name)-1], ".git") - return fmt.Sprintf("/workspaces/%s", repo), nil -} - type userInfo struct { uid int gid int @@ -1077,14 +1030,6 @@ func newColor(value ...color.Attribute) *color.Color { return c } -type osfsWithChmod struct { - billy.Filesystem -} - -func (fs *osfsWithChmod) Chmod(name string, mode os.FileMode) error { - return os.Chmod(name, mode) -} - func findDevcontainerJSON(options options.Options) (string, string, error) { // 0. Check if custom devcontainer directory or path is provided. if options.DevcontainerDir != "" || options.DevcontainerJSONPath != "" { diff --git a/envbuilder_test.go b/envbuilder_test.go index 0545bfce..aa4205c7 100644 --- a/envbuilder_test.go +++ b/envbuilder_test.go @@ -1,74 +1 @@ package envbuilder_test - -import ( - "testing" - - "github.com/coder/envbuilder" - "github.com/coder/envbuilder/constants" - - "github.com/stretchr/testify/require" -) - -func TestDefaultWorkspaceFolder(t *testing.T) { - t.Parallel() - - successTests := []struct { - name string - gitURL string - expected string - }{ - { - name: "HTTP", - gitURL: "https://github.com/coder/envbuilder.git", - expected: "/workspaces/envbuilder", - }, - { - name: "SSH", - gitURL: "git@github.com:coder/envbuilder.git", - expected: "/workspaces/envbuilder", - }, - { - name: "username and password", - gitURL: "https://username:password@github.com/coder/envbuilder.git", - expected: "/workspaces/envbuilder", - }, - { - name: "fragment", - gitURL: "https://github.com/coder/envbuilder.git#feature-branch", - expected: "/workspaces/envbuilder", - }, - { - name: "empty", - gitURL: "", - expected: constants.EmptyWorkspaceDir, - }, - } - for _, tt := range successTests { - t.Run(tt.name, func(t *testing.T) { - dir, err := envbuilder.DefaultWorkspaceFolder(tt.gitURL) - require.NoError(t, err) - require.Equal(t, tt.expected, dir) - }) - } - - invalidTests := []struct { - name string - invalidURL string - }{ - { - name: "simple text", - invalidURL: "not a valid URL", - }, - { - name: "website URL", - invalidURL: "www.google.com", - }, - } - for _, tt := range invalidTests { - t.Run(tt.name, func(t *testing.T) { - dir, err := envbuilder.DefaultWorkspaceFolder(tt.invalidURL) - require.NoError(t, err) - require.Equal(t, constants.EmptyWorkspaceDir, dir) - }) - } -} diff --git a/internal/chmodfs/chmodfs.go b/internal/chmodfs/chmodfs.go new file mode 100644 index 00000000..1242417a --- /dev/null +++ b/internal/chmodfs/chmodfs.go @@ -0,0 +1,21 @@ +package chmodfs + +import ( + "os" + + "github.com/go-git/go-billy/v5" +) + +func New(fs billy.Filesystem) billy.Filesystem { + return &osfsWithChmod{ + Filesystem: fs, + } +} + +type osfsWithChmod struct { + billy.Filesystem +} + +func (fs *osfsWithChmod) Chmod(name string, mode os.FileMode) error { + return os.Chmod(name, mode) +} diff --git a/options/defaults.go b/options/defaults.go new file mode 100644 index 00000000..18bf12a8 --- /dev/null +++ b/options/defaults.go @@ -0,0 +1,58 @@ +package options + +import ( + "fmt" + "strings" + + "github.com/go-git/go-billy/v5/osfs" + + giturls "github.com/chainguard-dev/git-urls" + "github.com/coder/envbuilder/constants" + "github.com/coder/envbuilder/internal/chmodfs" +) + +// DefaultWorkspaceFolder returns the default workspace folder +// for a given repository URL. +func DefaultWorkspaceFolder(repoURL string) string { + if repoURL == "" { + return constants.EmptyWorkspaceDir + } + parsed, err := giturls.Parse(repoURL) + if err != nil { + return constants.EmptyWorkspaceDir + } + name := strings.Split(parsed.Path, "/") + hasOwnerAndRepo := len(name) >= 2 + if !hasOwnerAndRepo { + return constants.EmptyWorkspaceDir + } + repo := strings.TrimSuffix(name[len(name)-1], ".git") + return fmt.Sprintf("/workspaces/%s", repo) +} + +func (o *Options) SetDefaults() { + // Temporarily removed these from the default settings to prevent conflicts + // between current and legacy environment variables that add default values. + // Once the legacy environment variables are phased out, this can be + // reinstated to the previous default values. + if len(o.IgnorePaths) == 0 { + o.IgnorePaths = []string{ + "/var/run", + // KinD adds these paths to pods, so ignore them by default. + "/product_uuid", "/product_name", + } + } + if o.InitScript == "" { + o.InitScript = "sleep infinity" + } + if o.InitCommand == "" { + o.InitCommand = "/bin/sh" + } + + if o.Filesystem == nil { + o.Filesystem = chmodfs.New(osfs.New("/")) + } + if o.WorkspaceFolder == "" { + o.WorkspaceFolder = DefaultWorkspaceFolder(o.GitURL) + } +} diff --git a/options/defaults_test.go b/options/defaults_test.go new file mode 100644 index 00000000..156ae16b --- /dev/null +++ b/options/defaults_test.go @@ -0,0 +1,93 @@ +package options_test + +import ( + "testing" + + "github.com/coder/envbuilder/internal/chmodfs" + "github.com/go-git/go-billy/v5/osfs" + + "github.com/stretchr/testify/assert" + + "github.com/coder/envbuilder/constants" + "github.com/coder/envbuilder/options" + "github.com/stretchr/testify/require" +) + +func TestDefaultWorkspaceFolder(t *testing.T) { + t.Parallel() + + successTests := []struct { + name string + gitURL string + expected string + }{ + { + name: "HTTP", + gitURL: "https://github.com/coder/envbuilder.git", + expected: "/workspaces/envbuilder", + }, + { + name: "SSH", + gitURL: "git@github.com:coder/envbuilder.git", + expected: "/workspaces/envbuilder", + }, + { + name: "username and password", + gitURL: "https://username:password@github.com/coder/envbuilder.git", + expected: "/workspaces/envbuilder", + }, + { + name: "fragment", + gitURL: "https://github.com/coder/envbuilder.git#feature-branch", + expected: "/workspaces/envbuilder", + }, + { + name: "empty", + gitURL: "", + expected: constants.EmptyWorkspaceDir, + }, + } + for _, tt := range successTests { + t.Run(tt.name, func(t *testing.T) { + dir := options.DefaultWorkspaceFolder(tt.gitURL) + require.Equal(t, tt.expected, dir) + }) + } + + invalidTests := []struct { + name string + invalidURL string + }{ + { + name: "simple text", + invalidURL: "not a valid URL", + }, + { + name: "website URL", + invalidURL: "www.google.com", + }, + } + for _, tt := range invalidTests { + t.Run(tt.name, func(t *testing.T) { + dir := options.DefaultWorkspaceFolder(tt.invalidURL) + require.Equal(t, constants.EmptyWorkspaceDir, dir) + }) + } +} + +func TestOptions_SetDefaults(t *testing.T) { + t.Parallel() + + expected := options.Options{ + InitScript: "sleep infinity", + InitCommand: "/bin/sh", + IgnorePaths: []string{"/var/run", "/product_uuid", "/product_name"}, + Filesystem: chmodfs.New(osfs.New("/")), + GitURL: "", + WorkspaceFolder: constants.EmptyWorkspaceDir, + } + + var actual options.Options + actual.SetDefaults() + assert.Equal(t, expected, actual) +} From 2ee05d155ac6c28181a7260e18386f703df924b9 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 29 Jul 2024 17:56:31 +0100 Subject: [PATCH 15/81] chore: move log package out of internal (#289) (cherry picked from commit 039314ed0e2df17d673d39461418e44cce90be6f) --- cmd/envbuilder/main.go | 2 +- envbuilder.go | 2 +- git/git.go | 2 +- git/git_test.go | 2 +- internal/ebutil/remount.go | 2 +- internal/ebutil/remount_internal_test.go | 2 +- {internal/log => log}/coder.go | 0 {internal/log => log}/coder_internal_test.go | 0 {internal/log => log}/log.go | 0 {internal/log => log}/log_test.go | 2 +- options/options.go | 2 +- 11 files changed, 8 insertions(+), 8 deletions(-) rename {internal/log => log}/coder.go (100%) rename {internal/log => log}/coder_internal_test.go (100%) rename {internal/log => log}/log.go (100%) rename {internal/log => log}/log_test.go (92%) diff --git a/cmd/envbuilder/main.go b/cmd/envbuilder/main.go index 24d63fd5..fcbf26d0 100644 --- a/cmd/envbuilder/main.go +++ b/cmd/envbuilder/main.go @@ -12,7 +12,7 @@ import ( "github.com/coder/coder/v2/codersdk" "github.com/coder/envbuilder" - "github.com/coder/envbuilder/internal/log" + "github.com/coder/envbuilder/log" "github.com/coder/serpent" // *Never* remove this. Certificates are not bundled as part diff --git a/envbuilder.go b/envbuilder.go index 2a00c84c..98adab03 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -34,7 +34,7 @@ import ( "github.com/GoogleContainerTools/kaniko/pkg/util" "github.com/coder/envbuilder/devcontainer" "github.com/coder/envbuilder/internal/ebutil" - "github.com/coder/envbuilder/internal/log" + "github.com/coder/envbuilder/log" "github.com/containerd/containerd/platforms" "github.com/distribution/distribution/v3/configuration" "github.com/distribution/distribution/v3/registry/handlers" diff --git a/git/git.go b/git/git.go index 019c68ef..199e350b 100644 --- a/git/git.go +++ b/git/git.go @@ -12,7 +12,7 @@ import ( "github.com/coder/envbuilder/options" giturls "github.com/chainguard-dev/git-urls" - "github.com/coder/envbuilder/internal/log" + "github.com/coder/envbuilder/log" "github.com/go-git/go-billy/v5" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" diff --git a/git/git_test.go b/git/git_test.go index a65e90f8..08ab1f93 100644 --- a/git/git_test.go +++ b/git/git_test.go @@ -16,7 +16,7 @@ import ( "github.com/coder/envbuilder/options" - "github.com/coder/envbuilder/internal/log" + "github.com/coder/envbuilder/log" "github.com/coder/envbuilder/testutil/gittest" "github.com/coder/envbuilder/testutil/mwtest" "github.com/go-git/go-billy/v5" diff --git a/internal/ebutil/remount.go b/internal/ebutil/remount.go index 21de0a3a..c6c6e6ed 100644 --- a/internal/ebutil/remount.go +++ b/internal/ebutil/remount.go @@ -9,7 +9,7 @@ import ( "sync" "syscall" - "github.com/coder/envbuilder/internal/log" + "github.com/coder/envbuilder/log" "github.com/hashicorp/go-multierror" "github.com/prometheus/procfs" ) diff --git a/internal/ebutil/remount_internal_test.go b/internal/ebutil/remount_internal_test.go index f6b68170..8ff0440d 100644 --- a/internal/ebutil/remount_internal_test.go +++ b/internal/ebutil/remount_internal_test.go @@ -8,7 +8,7 @@ import ( "testing" time "time" - "github.com/coder/envbuilder/internal/log" + "github.com/coder/envbuilder/log" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" diff --git a/internal/log/coder.go b/log/coder.go similarity index 100% rename from internal/log/coder.go rename to log/coder.go diff --git a/internal/log/coder_internal_test.go b/log/coder_internal_test.go similarity index 100% rename from internal/log/coder_internal_test.go rename to log/coder_internal_test.go diff --git a/internal/log/log.go b/log/log.go similarity index 100% rename from internal/log/log.go rename to log/log.go diff --git a/internal/log/log_test.go b/log/log_test.go similarity index 92% rename from internal/log/log_test.go rename to log/log_test.go index acf6247c..adeff7b1 100644 --- a/internal/log/log_test.go +++ b/log/log_test.go @@ -4,7 +4,7 @@ import ( "strings" "testing" - "github.com/coder/envbuilder/internal/log" + "github.com/coder/envbuilder/log" "github.com/stretchr/testify/require" ) diff --git a/options/options.go b/options/options.go index dd5ee8b9..5771b506 100644 --- a/options/options.go +++ b/options/options.go @@ -4,7 +4,7 @@ import ( "os" "strings" - "github.com/coder/envbuilder/internal/log" + "github.com/coder/envbuilder/log" "github.com/coder/serpent" "github.com/go-git/go-billy/v5" ) From 5e10073106fceb98a86c5799d8257c7c96b743a3 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 31 Jul 2024 12:59:20 +0100 Subject: [PATCH 16/81] extract RunCacheProbe function (#284) Builds on top of #282 and #283: - Extracts the logic for --get-cached-image to a separate function - Also pulls out some other common logic shared between Run and RunCacheProbe. (cherry picked from commit cacbcb8fef6c380cb6f3e9169d03d2ca00d99d39) --- cmd/envbuilder/main.go | 14 + envbuilder.go | 599 ++++++++++++++++++++++++++++++++--------- envbuilder_test.go | 1 - log/log.go | 29 ++ 4 files changed, 522 insertions(+), 121 deletions(-) delete mode 100644 envbuilder_test.go diff --git a/cmd/envbuilder/main.go b/cmd/envbuilder/main.go index fcbf26d0..1910568e 100644 --- a/cmd/envbuilder/main.go +++ b/cmd/envbuilder/main.go @@ -36,6 +36,7 @@ func envbuilderCmd() serpent.Command { Use: "envbuilder", Options: o.CLI(), Handler: func(inv *serpent.Invocation) error { + o.SetDefaults() o.Logger = log.New(os.Stderr, o.Verbose) if o.CoderAgentURL != "" { if o.CoderAgentToken == "" { @@ -63,6 +64,19 @@ func envbuilderCmd() serpent.Command { } } + if o.GetCachedImage { + img, err := envbuilder.RunCacheProbe(inv.Context(), o) + if err != nil { + o.Logger(log.LevelError, "error: %s", err) + } + digest, err := img.Digest() + if err != nil { + return fmt.Errorf("get cached image digest: %w", err) + } + _, _ = fmt.Fprintf(inv.Stdout, "ENVBUILDER_CACHED_IMAGE=%s@%s\n", o.CacheRepo, digest.String()) + return nil + } + err := envbuilder.Run(inv.Context(), o) if err != nil { o.Logger(log.LevelError, "error: %s", err) diff --git a/envbuilder.go b/envbuilder.go index 98adab03..dc2ead8e 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -21,6 +21,7 @@ import ( "sort" "strconv" "strings" + "sync" "syscall" "time" @@ -59,7 +60,10 @@ type DockerConfig configfile.ConfigFile // Filesystem is the filesystem to use for all operations. // Defaults to the host filesystem. func Run(ctx context.Context, opts options.Options) error { - opts.SetDefaults() + defer options.UnsetEnv() + if opts.GetCachedImage { + return fmt.Errorf("developer error: use RunCacheProbe instead") + } if opts.CacheRepo == "" && opts.PushImage { return fmt.Errorf("--cache-repo must be set when using --push-image") @@ -89,41 +93,20 @@ func Run(ctx context.Context, opts options.Options) error { opts.Logger(log.LevelInfo, "%s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder")) var caBundle []byte - if opts.SSLCertBase64 != "" { - certPool, err := x509.SystemCertPool() - if err != nil { - return xerrors.Errorf("get global system cert pool: %w", err) - } - data, err := base64.StdEncoding.DecodeString(opts.SSLCertBase64) - if err != nil { - return xerrors.Errorf("base64 decode ssl cert: %w", err) - } - ok := certPool.AppendCertsFromPEM(data) - if !ok { - return xerrors.Errorf("failed to append the ssl cert to the global pool: %s", data) - } - caBundle = data + caBundle, err := initCABundle(opts.SSLCertBase64) + if err != nil { + return err } - if opts.DockerConfigBase64 != "" { - decoded, err := base64.StdEncoding.DecodeString(opts.DockerConfigBase64) - if err != nil { - return fmt.Errorf("decode docker config: %w", err) - } - var configFile DockerConfig - decoded, err = hujson.Standardize(decoded) - if err != nil { - return fmt.Errorf("humanize json for docker config: %w", err) - } - err = json.Unmarshal(decoded, &configFile) - if err != nil { - return fmt.Errorf("parse docker config: %w", err) - } - err = os.WriteFile(filepath.Join(constants.MagicDir, "config.json"), decoded, 0o644) - if err != nil { - return fmt.Errorf("write docker config: %w", err) - } + cleanupDockerConfigJSON, err := initDockerConfigJSON(opts.DockerConfigBase64) + if err != nil { + return err } + defer func() { + if err := cleanupDockerConfigJSON(); err != nil { + opts.Logger(log.LevelError, "failed to cleanup docker config JSON: %w", err) + } + }() // best effort var fallbackErr error var cloned bool @@ -301,46 +284,16 @@ func Run(ctx context.Context, opts options.Options) error { } }) - var closeAfterBuild func() - // Allows quick testing of layer caching using a local directory! if opts.LayerCacheDir != "" { - cfg := &configuration.Configuration{ - Storage: configuration.Storage{ - "filesystem": configuration.Parameters{ - "rootdirectory": opts.LayerCacheDir, - }, - }, + if opts.CacheRepo != "" { + opts.Logger(log.LevelWarn, "Overriding cache repo with local registry...") } - cfg.Log.Level = "error" - - // Spawn an in-memory registry to cache built layers... - registry := handlers.NewApp(ctx, cfg) - - listener, err := net.Listen("tcp", "127.0.0.1:0") + localRegistry, closeLocalRegistry, err := serveLocalRegistry(ctx, opts.Logger, opts.LayerCacheDir) if err != nil { return err } - tcpAddr, ok := listener.Addr().(*net.TCPAddr) - if !ok { - return fmt.Errorf("listener addr was of wrong type: %T", listener.Addr()) - } - srv := &http.Server{ - Handler: registry, - } - go func() { - err := srv.Serve(listener) - if err != nil && !errors.Is(err, http.ErrServerClosed) { - opts.Logger(log.LevelError, "Failed to serve registry: %s", err.Error()) - } - }() - closeAfterBuild = func() { - _ = srv.Close() - _ = listener.Close() - } - if opts.CacheRepo != "" { - opts.Logger(log.LevelWarn, "Overriding cache repo with local registry...") - } - opts.CacheRepo = fmt.Sprintf("localhost:%d/local/cache", tcpAddr.Port) + defer closeLocalRegistry() + opts.CacheRepo = localRegistry } // IgnorePaths in the Kaniko opts doesn't properly ignore paths. @@ -404,6 +357,10 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) } skippedRebuild := false + stdoutWriter, closeStdout := log.Writer(opts.Logger) + defer closeStdout() + stderrWriter, closeStderr := log.Writer(opts.Logger) + defer closeStderr() build := func() (v1.Image, error) { _, err := opts.Filesystem.Stat(constants.MagicFile) if err == nil && opts.SkipRebuild { @@ -432,25 +389,26 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) if err := maybeDeleteFilesystem(opts.Logger, opts.ForceSafe); err != nil { return nil, fmt.Errorf("delete filesystem: %w", err) } - - stdoutReader, stdoutWriter := io.Pipe() - stderrReader, stderrWriter := io.Pipe() - defer stdoutReader.Close() - defer stdoutWriter.Close() - defer stderrReader.Close() - defer stderrWriter.Close() - go func() { - scanner := bufio.NewScanner(stdoutReader) - for scanner.Scan() { - opts.Logger(log.LevelInfo, "%s", scanner.Text()) - } - }() - go func() { - scanner := bufio.NewScanner(stderrReader) - for scanner.Scan() { - opts.Logger(log.LevelInfo, "%s", scanner.Text()) - } - }() + /* + stdoutReader, stdoutWriter := io.Pipe() + stderrReader, stderrWriter := io.Pipe() + defer stdoutReader.Close() + defer stdoutWriter.Close() + defer stderrReader.Close() + defer stderrWriter.Close() + go func() { + scanner := bufio.NewScanner(stdoutReader) + for scanner.Scan() { + opts.Logger(log.LevelInfo, "%s", scanner.Text()) + } + }() + go func() { + scanner := bufio.NewScanner(stderrReader) + for scanner.Scan() { + opts.Logger(log.LevelInfo, "%s", scanner.Text()) + } + }() + */ cacheTTL := time.Hour * 24 * 7 if opts.CacheTTLDays != 0 { cacheTTL = time.Hour * 24 * time.Duration(opts.CacheTTLDays) @@ -483,7 +441,6 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) // https://github.com/klauspost/compress/blob/67a538e2b4df11f8ec7139388838a13bce84b5d5/zstd/encoder_options.go#L188 CompressionLevel: 3, CacheOptions: config.CacheOptions{ - // Cache for a week by default! CacheTTL: cacheTTL, CacheDir: opts.BaseImageCacheDir, }, @@ -509,21 +466,6 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) Reproducible: opts.PushImage, } - if opts.GetCachedImage { - endStage := startStage("🏗️ Checking for cached image...") - image, err := executor.DoCacheProbe(kOpts) - if err != nil { - return nil, xerrors.Errorf("get cached image: %w", err) - } - digest, err := image.Digest() - if err != nil { - return nil, xerrors.Errorf("get cached image digest: %w", err) - } - endStage("🏗️ Found cached image!") - _, _ = fmt.Fprintf(os.Stdout, "ENVBUILDER_CACHED_IMAGE=%s@%s\n", kOpts.CacheRepo, digest.String()) - os.Exit(0) - } - endStage := startStage("🏗️ Building image...") image, err := executor.DoBuild(kOpts) if err != nil { @@ -578,10 +520,6 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) return fmt.Errorf("build with kaniko: %w", err) } - if closeAfterBuild != nil { - closeAfterBuild() - } - if err := restoreMounts(); err != nil { return fmt.Errorf("restore mounts: %w", err) } @@ -642,16 +580,8 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) options.UnsetEnv() // Remove the Docker config secret file! - if opts.DockerConfigBase64 != "" { - c := filepath.Join(constants.MagicDir, "config.json") - err = os.Remove(c) - if err != nil { - if !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("remove docker config: %w", err) - } else { - fmt.Fprintln(os.Stderr, "failed to remove the Docker config secret file: %w", c) - } - } + if err := cleanupDockerConfigJSON(); err != nil { + return err } environ, err := os.ReadFile("/etc/environment") @@ -902,6 +832,327 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) return nil } +// RunCacheProbe performs a 'dry-run' build of the image and checks that +// all of the resulting layers are present in options.CacheRepo. +func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) { + defer options.UnsetEnv() + if !opts.GetCachedImage { + return nil, fmt.Errorf("developer error: RunCacheProbe must be run with --get-cached-image") + } + if opts.CacheRepo == "" { + return nil, fmt.Errorf("--cache-repo must be set when using --get-cached-image") + } + + stageNumber := 0 + startStage := func(format string, args ...any) func(format string, args ...any) { + now := time.Now() + stageNumber++ + stageNum := stageNumber + opts.Logger(log.LevelInfo, "#%d: %s", stageNum, fmt.Sprintf(format, args...)) + + return func(format string, args ...any) { + opts.Logger(log.LevelInfo, "#%d: %s [%s]", stageNum, fmt.Sprintf(format, args...), time.Since(now)) + } + } + + opts.Logger(log.LevelInfo, "%s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder")) + + caBundle, err := initCABundle(opts.SSLCertBase64) + if err != nil { + return nil, err + } + + cleanupDockerConfigJSON, err := initDockerConfigJSON(opts.DockerConfigBase64) + if err != nil { + return nil, err + } + defer func() { + if err := cleanupDockerConfigJSON(); err != nil { + opts.Logger(log.LevelError, "failed to cleanup docker config JSON: %w", err) + } + }() // best effort + + var fallbackErr error + var cloned bool + if opts.GitURL != "" { + endStage := startStage("📦 Cloning %s to %s...", + newColor(color.FgCyan).Sprintf(opts.GitURL), + newColor(color.FgCyan).Sprintf(opts.WorkspaceFolder), + ) + + reader, writer := io.Pipe() + defer reader.Close() + defer writer.Close() + go func() { + data := make([]byte, 4096) + for { + read, err := reader.Read(data) + if err != nil { + return + } + content := data[:read] + for _, line := range strings.Split(string(content), "\r") { + if line == "" { + continue + } + opts.Logger(log.LevelInfo, "#1: %s", strings.TrimSpace(line)) + } + } + }() + + cloneOpts := git.CloneRepoOptions{ + Path: opts.WorkspaceFolder, + Storage: opts.Filesystem, + Insecure: opts.Insecure, + Progress: writer, + SingleBranch: opts.GitCloneSingleBranch, + Depth: int(opts.GitCloneDepth), + CABundle: caBundle, + } + + cloneOpts.RepoAuth = git.SetupRepoAuth(&opts) + if opts.GitHTTPProxyURL != "" { + cloneOpts.ProxyOptions = transport.ProxyOptions{ + URL: opts.GitHTTPProxyURL, + } + } + cloneOpts.RepoURL = opts.GitURL + + cloned, fallbackErr = git.CloneRepo(ctx, cloneOpts) + if fallbackErr == nil { + if cloned { + endStage("📦 Cloned repository!") + } else { + endStage("📦 The repository already exists!") + } + } else { + opts.Logger(log.LevelError, "Failed to clone repository: %s", fallbackErr.Error()) + opts.Logger(log.LevelError, "Falling back to the default image...") + } + } + + defaultBuildParams := func() (*devcontainer.Compiled, error) { + dockerfile := filepath.Join(constants.MagicDir, "Dockerfile") + file, err := opts.Filesystem.OpenFile(dockerfile, os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return nil, err + } + defer file.Close() + if opts.FallbackImage == "" { + if fallbackErr != nil { + return nil, fmt.Errorf("%s: %w", fallbackErr.Error(), constants.ErrNoFallbackImage) + } + // We can't use errors.Join here because our tests + // don't support parsing a multiline error. + return nil, constants.ErrNoFallbackImage + } + content := "FROM " + opts.FallbackImage + _, err = file.Write([]byte(content)) + if err != nil { + return nil, err + } + return &devcontainer.Compiled{ + DockerfilePath: dockerfile, + DockerfileContent: content, + BuildContext: constants.MagicDir, + }, nil + } + + var ( + buildParams *devcontainer.Compiled + devcontainerPath string + ) + if opts.DockerfilePath == "" { + // Only look for a devcontainer if a Dockerfile wasn't specified. + // devcontainer is a standard, so it's reasonable to be the default. + var devcontainerDir string + var err error + devcontainerPath, devcontainerDir, err = findDevcontainerJSON(opts) + if err != nil { + opts.Logger(log.LevelError, "Failed to locate devcontainer.json: %s", err.Error()) + opts.Logger(log.LevelError, "Falling back to the default image...") + } else { + // We know a devcontainer exists. + // Let's parse it and use it! + file, err := opts.Filesystem.Open(devcontainerPath) + if err != nil { + return nil, fmt.Errorf("open devcontainer.json: %w", err) + } + defer file.Close() + content, err := io.ReadAll(file) + if err != nil { + return nil, fmt.Errorf("read devcontainer.json: %w", err) + } + devContainer, err := devcontainer.Parse(content) + if err == nil { + var fallbackDockerfile string + if !devContainer.HasImage() && !devContainer.HasDockerfile() { + defaultParams, err := defaultBuildParams() + if err != nil { + return nil, fmt.Errorf("no Dockerfile or image found: %w", err) + } + opts.Logger(log.LevelInfo, "No Dockerfile or image specified; falling back to the default image...") + fallbackDockerfile = defaultParams.DockerfilePath + } + buildParams, err = devContainer.Compile(opts.Filesystem, devcontainerDir, constants.MagicDir, fallbackDockerfile, opts.WorkspaceFolder, false, os.LookupEnv) + if err != nil { + return nil, fmt.Errorf("compile devcontainer.json: %w", err) + } + } else { + opts.Logger(log.LevelError, "Failed to parse devcontainer.json: %s", err.Error()) + opts.Logger(log.LevelError, "Falling back to the default image...") + } + } + } else { + // If a Dockerfile was specified, we use that. + dockerfilePath := filepath.Join(opts.WorkspaceFolder, opts.DockerfilePath) + + // If the dockerfilePath is specified and deeper than the base of WorkspaceFolder AND the BuildContextPath is + // not defined, show a warning + dockerfileDir := filepath.Dir(dockerfilePath) + if dockerfileDir != filepath.Clean(opts.WorkspaceFolder) && opts.BuildContextPath == "" { + opts.Logger(log.LevelWarn, "given dockerfile %q is below %q and no custom build context has been defined", dockerfilePath, opts.WorkspaceFolder) + opts.Logger(log.LevelWarn, "\t-> set BUILD_CONTEXT_PATH to %q to fix", dockerfileDir) + } + + dockerfile, err := opts.Filesystem.Open(dockerfilePath) + if err == nil { + content, err := io.ReadAll(dockerfile) + if err != nil { + return nil, fmt.Errorf("read Dockerfile: %w", err) + } + buildParams = &devcontainer.Compiled{ + DockerfilePath: dockerfilePath, + DockerfileContent: string(content), + BuildContext: filepath.Join(opts.WorkspaceFolder, opts.BuildContextPath), + } + } + } + + // When probing the build cache, there is no fallback! + if buildParams == nil { + return nil, fmt.Errorf("no Dockerfile or devcontainer.json found") + } + + HijackLogrus(func(entry *logrus.Entry) { + for _, line := range strings.Split(entry.Message, "\r") { + opts.Logger(log.LevelInfo, "#%d: %s", stageNumber, color.HiBlackString(line)) + } + }) + + if opts.LayerCacheDir != "" { + if opts.CacheRepo != "" { + opts.Logger(log.LevelWarn, "Overriding cache repo with local registry...") + } + localRegistry, closeLocalRegistry, err := serveLocalRegistry(ctx, opts.Logger, opts.LayerCacheDir) + if err != nil { + return nil, err + } + defer closeLocalRegistry() + opts.CacheRepo = localRegistry + } + + // IgnorePaths in the Kaniko opts doesn't properly ignore paths. + // So we add them to the default ignore list. See: + // https://github.com/GoogleContainerTools/kaniko/blob/63be4990ca5a60bdf06ddc4d10aa4eca0c0bc714/cmd/executor/cmd/root.go#L136 + ignorePaths := append([]string{ + constants.MagicDir, + opts.WorkspaceFolder, + // See: https://github.com/coder/envbuilder/issues/37 + "/etc/resolv.conf", + }, opts.IgnorePaths...) + + if opts.LayerCacheDir != "" { + ignorePaths = append(ignorePaths, opts.LayerCacheDir) + } + + for _, ignorePath := range ignorePaths { + util.AddToDefaultIgnoreList(util.IgnoreListEntry{ + Path: ignorePath, + PrefixMatchOnly: false, + AllowedPaths: nil, + }) + } + + stdoutWriter, closeStdout := log.Writer(opts.Logger) + defer closeStdout() + stderrWriter, closeStderr := log.Writer(opts.Logger) + defer closeStderr() + cacheTTL := time.Hour * 24 * 7 + if opts.CacheTTLDays != 0 { + cacheTTL = time.Hour * 24 * time.Duration(opts.CacheTTLDays) + } + + // At this point we have all the context, we can now build! + registryMirror := []string{} + if val, ok := os.LookupEnv("KANIKO_REGISTRY_MIRROR"); ok { + registryMirror = strings.Split(val, ";") + } + var destinations []string + if opts.CacheRepo != "" { + destinations = append(destinations, opts.CacheRepo) + } + kOpts := &config.KanikoOptions{ + // Boilerplate! + CustomPlatform: platforms.Format(platforms.Normalize(platforms.DefaultSpec())), + SnapshotMode: "redo", + RunV2: true, + RunStdout: stdoutWriter, + RunStderr: stderrWriter, + Destinations: destinations, + NoPush: !opts.PushImage || len(destinations) == 0, + CacheRunLayers: true, + CacheCopyLayers: true, + CompressedCaching: true, + Compression: config.ZStd, + // Maps to "default" level, ~100-300 MB/sec according to + // benchmarks in klauspost/compress README + // https://github.com/klauspost/compress/blob/67a538e2b4df11f8ec7139388838a13bce84b5d5/zstd/encoder_options.go#L188 + CompressionLevel: 3, + CacheOptions: config.CacheOptions{ + CacheTTL: cacheTTL, + CacheDir: opts.BaseImageCacheDir, + }, + ForceUnpack: true, + BuildArgs: buildParams.BuildArgs, + CacheRepo: opts.CacheRepo, + Cache: opts.CacheRepo != "" || opts.BaseImageCacheDir != "", + DockerfilePath: buildParams.DockerfilePath, + DockerfileContent: buildParams.DockerfileContent, + RegistryOptions: config.RegistryOptions{ + Insecure: opts.Insecure, + InsecurePull: opts.Insecure, + SkipTLSVerify: opts.Insecure, + // Enables registry mirror features in Kaniko, see more in link below + // https://github.com/GoogleContainerTools/kaniko?tab=readme-ov-file#flag---registry-mirror + // Related to PR #114 + // https://github.com/coder/envbuilder/pull/114 + RegistryMirrors: registryMirror, + }, + SrcContext: buildParams.BuildContext, + + // For cached image utilization, produce reproducible builds. + Reproducible: opts.PushImage, + } + + endStage := startStage("🏗️ Checking for cached image...") + image, err := executor.DoCacheProbe(kOpts) + if err != nil { + return nil, fmt.Errorf("get cached image: %w", err) + } + endStage("🏗️ Found cached image!") + + // Sanitize the environment of any opts! + options.UnsetEnv() + + // Remove the Docker config secret file! + if err := cleanupDockerConfigJSON(); err != nil { + return nil, err + } + + return image, nil +} + type userInfo struct { uid int gid int @@ -1123,17 +1374,125 @@ func maybeDeleteFilesystem(logger log.Func, force bool) error { func copyFile(src, dst string) error { content, err := os.ReadFile(src) if err != nil { - return xerrors.Errorf("read file failed: %w", err) + return fmt.Errorf("read file failed: %w", err) } err = os.MkdirAll(filepath.Dir(dst), 0o755) if err != nil { - return xerrors.Errorf("mkdir all failed: %w", err) + return fmt.Errorf("mkdir all failed: %w", err) } err = os.WriteFile(dst, content, 0o644) if err != nil { - return xerrors.Errorf("write file failed: %w", err) + return fmt.Errorf("write file failed: %w", err) } return nil } + +func initCABundle(sslCertBase64 string) ([]byte, error) { + if sslCertBase64 == "" { + return []byte{}, nil + } + certPool, err := x509.SystemCertPool() + if err != nil { + return nil, fmt.Errorf("get global system cert pool: %w", err) + } + data, err := base64.StdEncoding.DecodeString(sslCertBase64) + if err != nil { + return nil, fmt.Errorf("base64 decode ssl cert: %w", err) + } + ok := certPool.AppendCertsFromPEM(data) + if !ok { + return nil, fmt.Errorf("failed to append the ssl cert to the global pool: %s", data) + } + return data, nil +} + +func initDockerConfigJSON(dockerConfigBase64 string) (func() error, error) { + var cleanupOnce sync.Once + noop := func() error { return nil } + if dockerConfigBase64 == "" { + return noop, nil + } + cfgPath := filepath.Join(constants.MagicDir, "config.json") + decoded, err := base64.StdEncoding.DecodeString(dockerConfigBase64) + if err != nil { + return noop, fmt.Errorf("decode docker config: %w", err) + } + var configFile DockerConfig + decoded, err = hujson.Standardize(decoded) + if err != nil { + return noop, fmt.Errorf("humanize json for docker config: %w", err) + } + err = json.Unmarshal(decoded, &configFile) + if err != nil { + return noop, fmt.Errorf("parse docker config: %w", err) + } + err = os.WriteFile(cfgPath, decoded, 0o644) + if err != nil { + return noop, fmt.Errorf("write docker config: %w", err) + } + cleanup := func() error { + var cleanupErr error + cleanupOnce.Do(func() { + // Remove the Docker config secret file! + if cleanupErr = os.Remove(cfgPath); err != nil { + if !errors.Is(err, fs.ErrNotExist) { + cleanupErr = fmt.Errorf("remove docker config: %w", cleanupErr) + } + _, _ = fmt.Fprintf(os.Stderr, "failed to remove the Docker config secret file: %s\n", cleanupErr) + } + }) + return cleanupErr + } + return cleanup, err +} + +// Allows quick testing of layer caching using a local directory! +func serveLocalRegistry(ctx context.Context, logf log.Func, layerCacheDir string) (string, func(), error) { + noop := func() {} + if layerCacheDir == "" { + return "", noop, nil + } + cfg := &configuration.Configuration{ + Storage: configuration.Storage{ + "filesystem": configuration.Parameters{ + "rootdirectory": layerCacheDir, + }, + }, + } + cfg.Log.Level = "error" + + // Spawn an in-memory registry to cache built layers... + registry := handlers.NewApp(ctx, cfg) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return "", nil, fmt.Errorf("start listener for in-memory registry: %w", err) + } + tcpAddr, ok := listener.Addr().(*net.TCPAddr) + if !ok { + return "", noop, fmt.Errorf("listener addr was of wrong type: %T", listener.Addr()) + } + srv := &http.Server{ + Handler: registry, + } + done := make(chan struct{}) + go func() { + defer close(done) + err := srv.Serve(listener) + if err != nil && !errors.Is(err, http.ErrServerClosed) { + logf(log.LevelError, "Failed to serve registry: %s", err.Error()) + } + }() + var closeOnce sync.Once + closer := func() { + closeOnce.Do(func() { + _ = srv.Close() + _ = listener.Close() + <-done + }) + } + addr := fmt.Sprintf("localhost:%d/local/cache", tcpAddr.Port) + return addr, closer, nil +} diff --git a/envbuilder_test.go b/envbuilder_test.go deleted file mode 100644 index aa4205c7..00000000 --- a/envbuilder_test.go +++ /dev/null @@ -1 +0,0 @@ -package envbuilder_test diff --git a/log/log.go b/log/log.go index da308266..8519d6b0 100644 --- a/log/log.go +++ b/log/log.go @@ -1,6 +1,7 @@ package log import ( + "bufio" "fmt" "io" "strings" @@ -45,3 +46,31 @@ func Wrap(fs ...Func) Func { } } } + +// Writer returns an io.Writer that logs all writes in a separate goroutine. +// It is the responsibility of the caller to call the returned +// function to stop the goroutine. +func Writer(logf Func) (io.Writer, func()) { + pipeReader, pipeWriter := io.Pipe() + doneCh := make(chan struct{}) + go func() { + defer pipeWriter.Close() + defer pipeReader.Close() + scanner := bufio.NewScanner(pipeReader) + for { + select { + case <-doneCh: + return + default: + if !scanner.Scan() { + return + } + logf(LevelInfo, "%s", scanner.Text()) + } + } + }() + closer := func() { + close(doneCh) + } + return pipeWriter, closer +} From 961bab762ee5239fabf73738b252a9a1dee45e6a Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 1 Aug 2024 18:07:29 +0300 Subject: [PATCH 17/81] feat: implement repo-mode (#290) (cherry picked from commit 9073748fb309b2b4cfbc9c84f1d60b446b12965b) --- README.md | 1 + constants/constants.go | 4 + envbuilder.go | 230 +++++++++++------------- envbuilder_internal_test.go | 308 +++++++++++++++++--------------- git/git.go | 100 +++++++++++ git/git_test.go | 67 +++++++ options/options.go | 41 +++++ options/testdata/options.golden | 7 + 8 files changed, 485 insertions(+), 273 deletions(-) diff --git a/README.md b/README.md index 34237c58..51642345 100644 --- a/README.md +++ b/README.md @@ -388,5 +388,6 @@ On MacOS or Windows systems, we recommend either using a VM or the provided `.de | `--coder-agent-subsystem` | `CODER_AGENT_SUBSYSTEM` | | Coder agent subsystems to report when forwarding logs. The envbuilder subsystem is always included. | | `--push-image` | `ENVBUILDER_PUSH_IMAGE` | | Push the built image to a remote registry. This option forces a reproducible build. | | `--get-cached-image` | `ENVBUILDER_GET_CACHED_IMAGE` | | Print the digest of the cached image, if available. Exits with an error if not found. | +| `--remote-repo-build-mode` | `ENVBUILDER_REMOTE_REPO_BUILD_MODE` | `false` | Use the remote repository as the source of truth when building the image. Enabling this option ignores user changes to local files and they will not be reflected in the image. This can be used to improving cache utilization when multiple users are building working on the same repository. | | `--verbose` | `ENVBUILDER_VERBOSE` | | Enable verbose logging. | diff --git a/constants/constants.go b/constants/constants.go index ccdfcb8c..042660dd 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -28,4 +28,8 @@ var ( // to skip building when a container is restarting. // e.g. docker stop -> docker start MagicFile = filepath.Join(MagicDir, "built") + + // MagicFile is the location of the build context when + // using remote build mode. + MagicRemoteRepoDir = filepath.Join(MagicDir, "repo") ) diff --git a/envbuilder.go b/envbuilder.go index dc2ead8e..c65ae62f 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -4,7 +4,6 @@ import ( "bufio" "bytes" "context" - "crypto/x509" "encoding/base64" "encoding/json" "errors" @@ -42,7 +41,6 @@ import ( _ "github.com/distribution/distribution/v3/registry/storage/driver/filesystem" "github.com/docker/cli/cli/config/configfile" "github.com/fatih/color" - "github.com/go-git/go-git/v5/plumbing/transport" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/kballard/go-shellquote" @@ -92,12 +90,6 @@ func Run(ctx context.Context, opts options.Options) error { opts.Logger(log.LevelInfo, "%s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder")) - var caBundle []byte - caBundle, err := initCABundle(opts.SSLCertBase64) - if err != nil { - return err - } - cleanupDockerConfigJSON, err := initDockerConfigJSON(opts.DockerConfigBase64) if err != nil { return err @@ -108,51 +100,23 @@ func Run(ctx context.Context, opts options.Options) error { } }() // best effort + buildTimeWorkspaceFolder := opts.WorkspaceFolder var fallbackErr error var cloned bool if opts.GitURL != "" { + cloneOpts, err := git.CloneOptionsFromOptions(opts) + if err != nil { + return fmt.Errorf("git clone options: %w", err) + } + endStage := startStage("📦 Cloning %s to %s...", newColor(color.FgCyan).Sprintf(opts.GitURL), - newColor(color.FgCyan).Sprintf(opts.WorkspaceFolder), + newColor(color.FgCyan).Sprintf(cloneOpts.Path), ) - reader, writer := io.Pipe() - defer reader.Close() - defer writer.Close() - go func() { - data := make([]byte, 4096) - for { - read, err := reader.Read(data) - if err != nil { - return - } - content := data[:read] - for _, line := range strings.Split(string(content), "\r") { - if line == "" { - continue - } - opts.Logger(log.LevelInfo, "#1: %s", strings.TrimSpace(line)) - } - } - }() - - cloneOpts := git.CloneRepoOptions{ - Path: opts.WorkspaceFolder, - Storage: opts.Filesystem, - Insecure: opts.Insecure, - Progress: writer, - SingleBranch: opts.GitCloneSingleBranch, - Depth: int(opts.GitCloneDepth), - CABundle: caBundle, - } - - cloneOpts.RepoAuth = git.SetupRepoAuth(&opts) - if opts.GitHTTPProxyURL != "" { - cloneOpts.ProxyOptions = transport.ProxyOptions{ - URL: opts.GitHTTPProxyURL, - } - } - cloneOpts.RepoURL = opts.GitURL + w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNumber, line) }) + defer w.Close() + cloneOpts.Progress = w cloned, fallbackErr = git.CloneRepo(ctx, cloneOpts) if fallbackErr == nil { @@ -165,6 +129,34 @@ func Run(ctx context.Context, opts options.Options) error { opts.Logger(log.LevelError, "Failed to clone repository: %s", fallbackErr.Error()) opts.Logger(log.LevelError, "Falling back to the default image...") } + + // Always clone the repo in remote repo build mode into a location that + // we control that isn't affected by the users changes. + if opts.RemoteRepoBuildMode { + cloneOpts, err := git.CloneOptionsFromOptions(opts) + if err != nil { + return fmt.Errorf("git clone options: %w", err) + } + cloneOpts.Path = constants.MagicRemoteRepoDir + + endStage := startStage("📦 Remote repo build mode enabled, cloning %s to %s for build context...", + newColor(color.FgCyan).Sprintf(opts.GitURL), + newColor(color.FgCyan).Sprintf(cloneOpts.Path), + ) + + w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNumber, line) }) + defer w.Close() + cloneOpts.Progress = w + + fallbackErr = git.ShallowCloneRepo(ctx, cloneOpts) + if fallbackErr == nil { + endStage("📦 Cloned repository!") + buildTimeWorkspaceFolder = cloneOpts.Path + } else { + opts.Logger(log.LevelError, "Failed to clone repository for remote repo mode: %s", err.Error()) + opts.Logger(log.LevelError, "Falling back to the default image...") + } + } } defaultBuildParams := func() (*devcontainer.Compiled, error) { @@ -205,7 +197,7 @@ func Run(ctx context.Context, opts options.Options) error { // devcontainer is a standard, so it's reasonable to be the default. var devcontainerDir string var err error - devcontainerPath, devcontainerDir, err = findDevcontainerJSON(opts) + devcontainerPath, devcontainerDir, err = findDevcontainerJSON(buildTimeWorkspaceFolder, opts) if err != nil { opts.Logger(log.LevelError, "Failed to locate devcontainer.json: %s", err.Error()) opts.Logger(log.LevelError, "Falling back to the default image...") @@ -244,13 +236,13 @@ func Run(ctx context.Context, opts options.Options) error { } } else { // If a Dockerfile was specified, we use that. - dockerfilePath := filepath.Join(opts.WorkspaceFolder, opts.DockerfilePath) + dockerfilePath := filepath.Join(buildTimeWorkspaceFolder, opts.DockerfilePath) // If the dockerfilePath is specified and deeper than the base of WorkspaceFolder AND the BuildContextPath is // not defined, show a warning dockerfileDir := filepath.Dir(dockerfilePath) - if dockerfileDir != filepath.Clean(opts.WorkspaceFolder) && opts.BuildContextPath == "" { - opts.Logger(log.LevelWarn, "given dockerfile %q is below %q and no custom build context has been defined", dockerfilePath, opts.WorkspaceFolder) + if dockerfileDir != filepath.Clean(buildTimeWorkspaceFolder) && opts.BuildContextPath == "" { + opts.Logger(log.LevelWarn, "given dockerfile %q is below %q and no custom build context has been defined", dockerfilePath, buildTimeWorkspaceFolder) opts.Logger(log.LevelWarn, "\t-> set BUILD_CONTEXT_PATH to %q to fix", dockerfileDir) } @@ -263,7 +255,7 @@ func Run(ctx context.Context, opts options.Options) error { buildParams = &devcontainer.Compiled{ DockerfilePath: dockerfilePath, DockerfileContent: string(content), - BuildContext: filepath.Join(opts.WorkspaceFolder, opts.BuildContextPath), + BuildContext: filepath.Join(buildTimeWorkspaceFolder, opts.BuildContextPath), } } } @@ -552,10 +544,10 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) if err != nil { return fmt.Errorf("unmarshal metadata: %w", err) } - opts.Logger(log.LevelInfo, "#3: 👀 Found devcontainer.json label metadata in image...") + opts.Logger(log.LevelInfo, "#%d: 👀 Found devcontainer.json label metadata in image...", stageNumber) for _, container := range devContainer { if container.RemoteUser != "" { - opts.Logger(log.LevelInfo, "#3: 🧑 Updating the user to %q!", container.RemoteUser) + opts.Logger(log.LevelInfo, "#%d: 🧑 Updating the user to %q!", stageNumber, container.RemoteUser) configFile.Config.User = container.RemoteUser } @@ -654,7 +646,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) username = buildParams.User } if username == "" { - opts.Logger(log.LevelWarn, "#3: no user specified, using root") + opts.Logger(log.LevelWarn, "#%d: no user specified, using root", stageNumber) } userInfo, err := getUser(username) @@ -857,11 +849,6 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) opts.Logger(log.LevelInfo, "%s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder")) - caBundle, err := initCABundle(opts.SSLCertBase64) - if err != nil { - return nil, err - } - cleanupDockerConfigJSON, err := initDockerConfigJSON(opts.DockerConfigBase64) if err != nil { return nil, err @@ -872,51 +859,23 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) } }() // best effort + buildTimeWorkspaceFolder := opts.WorkspaceFolder var fallbackErr error var cloned bool if opts.GitURL != "" { + cloneOpts, err := git.CloneOptionsFromOptions(opts) + if err != nil { + return nil, fmt.Errorf("git clone options: %w", err) + } + endStage := startStage("📦 Cloning %s to %s...", newColor(color.FgCyan).Sprintf(opts.GitURL), - newColor(color.FgCyan).Sprintf(opts.WorkspaceFolder), + newColor(color.FgCyan).Sprintf(cloneOpts.Path), ) - reader, writer := io.Pipe() - defer reader.Close() - defer writer.Close() - go func() { - data := make([]byte, 4096) - for { - read, err := reader.Read(data) - if err != nil { - return - } - content := data[:read] - for _, line := range strings.Split(string(content), "\r") { - if line == "" { - continue - } - opts.Logger(log.LevelInfo, "#1: %s", strings.TrimSpace(line)) - } - } - }() - - cloneOpts := git.CloneRepoOptions{ - Path: opts.WorkspaceFolder, - Storage: opts.Filesystem, - Insecure: opts.Insecure, - Progress: writer, - SingleBranch: opts.GitCloneSingleBranch, - Depth: int(opts.GitCloneDepth), - CABundle: caBundle, - } - - cloneOpts.RepoAuth = git.SetupRepoAuth(&opts) - if opts.GitHTTPProxyURL != "" { - cloneOpts.ProxyOptions = transport.ProxyOptions{ - URL: opts.GitHTTPProxyURL, - } - } - cloneOpts.RepoURL = opts.GitURL + w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNumber, line) }) + defer w.Close() + cloneOpts.Progress = w cloned, fallbackErr = git.CloneRepo(ctx, cloneOpts) if fallbackErr == nil { @@ -929,6 +888,34 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) opts.Logger(log.LevelError, "Failed to clone repository: %s", fallbackErr.Error()) opts.Logger(log.LevelError, "Falling back to the default image...") } + + // Always clone the repo in remote repo build mode into a location that + // we control that isn't affected by the users changes. + if opts.RemoteRepoBuildMode { + cloneOpts, err := git.CloneOptionsFromOptions(opts) + if err != nil { + return nil, fmt.Errorf("git clone options: %w", err) + } + cloneOpts.Path = constants.MagicRemoteRepoDir + + endStage := startStage("📦 Remote repo build mode enabled, cloning %s to %s for build context...", + newColor(color.FgCyan).Sprintf(opts.GitURL), + newColor(color.FgCyan).Sprintf(cloneOpts.Path), + ) + + w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNumber, line) }) + defer w.Close() + cloneOpts.Progress = w + + fallbackErr = git.ShallowCloneRepo(ctx, cloneOpts) + if fallbackErr == nil { + endStage("📦 Cloned repository!") + buildTimeWorkspaceFolder = cloneOpts.Path + } else { + opts.Logger(log.LevelError, "Failed to clone repository for remote repo mode: %s", err.Error()) + opts.Logger(log.LevelError, "Falling back to the default image...") + } + } } defaultBuildParams := func() (*devcontainer.Compiled, error) { @@ -967,7 +954,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) // devcontainer is a standard, so it's reasonable to be the default. var devcontainerDir string var err error - devcontainerPath, devcontainerDir, err = findDevcontainerJSON(opts) + devcontainerPath, devcontainerDir, err = findDevcontainerJSON(buildTimeWorkspaceFolder, opts) if err != nil { opts.Logger(log.LevelError, "Failed to locate devcontainer.json: %s", err.Error()) opts.Logger(log.LevelError, "Falling back to the default image...") @@ -1005,13 +992,13 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) } } else { // If a Dockerfile was specified, we use that. - dockerfilePath := filepath.Join(opts.WorkspaceFolder, opts.DockerfilePath) + dockerfilePath := filepath.Join(buildTimeWorkspaceFolder, opts.DockerfilePath) // If the dockerfilePath is specified and deeper than the base of WorkspaceFolder AND the BuildContextPath is // not defined, show a warning dockerfileDir := filepath.Dir(dockerfilePath) - if dockerfileDir != filepath.Clean(opts.WorkspaceFolder) && opts.BuildContextPath == "" { - opts.Logger(log.LevelWarn, "given dockerfile %q is below %q and no custom build context has been defined", dockerfilePath, opts.WorkspaceFolder) + if dockerfileDir != filepath.Clean(buildTimeWorkspaceFolder) && opts.BuildContextPath == "" { + opts.Logger(log.LevelWarn, "given dockerfile %q is below %q and no custom build context has been defined", dockerfilePath, buildTimeWorkspaceFolder) opts.Logger(log.LevelWarn, "\t-> set BUILD_CONTEXT_PATH to %q to fix", dockerfileDir) } @@ -1024,7 +1011,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) buildParams = &devcontainer.Compiled{ DockerfilePath: dockerfilePath, DockerfileContent: string(content), - BuildContext: filepath.Join(opts.WorkspaceFolder, opts.BuildContextPath), + BuildContext: filepath.Join(buildTimeWorkspaceFolder, opts.BuildContextPath), } } } @@ -1281,7 +1268,11 @@ func newColor(value ...color.Attribute) *color.Color { return c } -func findDevcontainerJSON(options options.Options) (string, string, error) { +func findDevcontainerJSON(workspaceFolder string, options options.Options) (string, string, error) { + if workspaceFolder == "" { + workspaceFolder = options.WorkspaceFolder + } + // 0. Check if custom devcontainer directory or path is provided. if options.DevcontainerDir != "" || options.DevcontainerJSONPath != "" { devcontainerDir := options.DevcontainerDir @@ -1291,7 +1282,7 @@ func findDevcontainerJSON(options options.Options) (string, string, error) { // If `devcontainerDir` is not an absolute path, assume it is relative to the workspace folder. if !filepath.IsAbs(devcontainerDir) { - devcontainerDir = filepath.Join(options.WorkspaceFolder, devcontainerDir) + devcontainerDir = filepath.Join(workspaceFolder, devcontainerDir) } // An absolute location always takes a precedence. @@ -1310,20 +1301,20 @@ func findDevcontainerJSON(options options.Options) (string, string, error) { return devcontainerPath, devcontainerDir, nil } - // 1. Check `options.WorkspaceFolder`/.devcontainer/devcontainer.json. - location := filepath.Join(options.WorkspaceFolder, ".devcontainer", "devcontainer.json") + // 1. Check `workspaceFolder`/.devcontainer/devcontainer.json. + location := filepath.Join(workspaceFolder, ".devcontainer", "devcontainer.json") if _, err := options.Filesystem.Stat(location); err == nil { return location, filepath.Dir(location), nil } - // 2. Check `options.WorkspaceFolder`/devcontainer.json. - location = filepath.Join(options.WorkspaceFolder, "devcontainer.json") + // 2. Check `workspaceFolder`/devcontainer.json. + location = filepath.Join(workspaceFolder, "devcontainer.json") if _, err := options.Filesystem.Stat(location); err == nil { return location, filepath.Dir(location), nil } - // 3. Check every folder: `options.WorkspaceFolder`/.devcontainer//devcontainer.json. - devcontainerDir := filepath.Join(options.WorkspaceFolder, ".devcontainer") + // 3. Check every folder: `workspaceFolder`/.devcontainer//devcontainer.json. + devcontainerDir := filepath.Join(workspaceFolder, ".devcontainer") fileInfos, err := options.Filesystem.ReadDir(devcontainerDir) if err != nil { @@ -1389,25 +1380,6 @@ func copyFile(src, dst string) error { return nil } -func initCABundle(sslCertBase64 string) ([]byte, error) { - if sslCertBase64 == "" { - return []byte{}, nil - } - certPool, err := x509.SystemCertPool() - if err != nil { - return nil, fmt.Errorf("get global system cert pool: %w", err) - } - data, err := base64.StdEncoding.DecodeString(sslCertBase64) - if err != nil { - return nil, fmt.Errorf("base64 decode ssl cert: %w", err) - } - ok := certPool.AppendCertsFromPEM(data) - if !ok { - return nil, fmt.Errorf("failed to append the ssl cert to the global pool: %s", data) - } - return data, nil -} - func initDockerConfigJSON(dockerConfigBase64 string) (func() error, error) { var cleanupOnce sync.Once noop := func() error { return nil } diff --git a/envbuilder_internal_test.go b/envbuilder_internal_test.go index 3af4b5e4..eb756071 100644 --- a/envbuilder_internal_test.go +++ b/envbuilder_internal_test.go @@ -13,149 +13,169 @@ import ( func TestFindDevcontainerJSON(t *testing.T) { t.Parallel() - t.Run("empty filesystem", func(t *testing.T) { - t.Parallel() - - // given - fs := memfs.New() - - // when - _, _, err := findDevcontainerJSON(options.Options{ - Filesystem: fs, - WorkspaceFolder: "/workspace", + defaultWorkspaceFolder := "/workspace" + + for _, tt := range []struct { + name string + workspaceFolder string + }{ + { + name: "Default", + workspaceFolder: defaultWorkspaceFolder, + }, + { + name: "RepoMode", + workspaceFolder: "/.envbuilder/repo", + }, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + t.Run("empty filesystem", func(t *testing.T) { + t.Parallel() + + // given + fs := memfs.New() + + // when + _, _, err := findDevcontainerJSON(tt.workspaceFolder, options.Options{ + Filesystem: fs, + WorkspaceFolder: "/workspace", + }) + + // then + require.Error(t, err) + }) + + t.Run("devcontainer.json is missing", func(t *testing.T) { + t.Parallel() + + // given + fs := memfs.New() + err := fs.MkdirAll(tt.workspaceFolder+"/.devcontainer", 0o600) + require.NoError(t, err) + + // when + _, _, err = findDevcontainerJSON(tt.workspaceFolder, options.Options{ + Filesystem: fs, + WorkspaceFolder: "/workspace", + }) + + // then + require.Error(t, err) + }) + + t.Run("default configuration", func(t *testing.T) { + t.Parallel() + + // given + fs := memfs.New() + err := fs.MkdirAll(tt.workspaceFolder+"/.devcontainer", 0o600) + require.NoError(t, err) + _, err = fs.Create(tt.workspaceFolder + "/.devcontainer/devcontainer.json") + require.NoError(t, err) + + // when + devcontainerPath, devcontainerDir, err := findDevcontainerJSON(tt.workspaceFolder, options.Options{ + Filesystem: fs, + WorkspaceFolder: "/workspace", + }) + + // then + require.NoError(t, err) + assert.Equal(t, tt.workspaceFolder+"/.devcontainer/devcontainer.json", devcontainerPath) + assert.Equal(t, tt.workspaceFolder+"/.devcontainer", devcontainerDir) + }) + + t.Run("overridden .devcontainer directory", func(t *testing.T) { + t.Parallel() + + // given + fs := memfs.New() + err := fs.MkdirAll(tt.workspaceFolder+"/experimental-devcontainer", 0o600) + require.NoError(t, err) + _, err = fs.Create(tt.workspaceFolder + "/experimental-devcontainer/devcontainer.json") + require.NoError(t, err) + + // when + devcontainerPath, devcontainerDir, err := findDevcontainerJSON(tt.workspaceFolder, options.Options{ + Filesystem: fs, + WorkspaceFolder: "/workspace", + DevcontainerDir: "experimental-devcontainer", + }) + + // then + require.NoError(t, err) + assert.Equal(t, tt.workspaceFolder+"/experimental-devcontainer/devcontainer.json", devcontainerPath) + assert.Equal(t, tt.workspaceFolder+"/experimental-devcontainer", devcontainerDir) + }) + + t.Run("overridden devcontainer.json path", func(t *testing.T) { + t.Parallel() + + // given + fs := memfs.New() + err := fs.MkdirAll(tt.workspaceFolder+"/.devcontainer", 0o600) + require.NoError(t, err) + _, err = fs.Create(tt.workspaceFolder + "/.devcontainer/experimental.json") + require.NoError(t, err) + + // when + devcontainerPath, devcontainerDir, err := findDevcontainerJSON(tt.workspaceFolder, options.Options{ + Filesystem: fs, + WorkspaceFolder: "/workspace", + DevcontainerJSONPath: "experimental.json", + }) + + // then + require.NoError(t, err) + assert.Equal(t, tt.workspaceFolder+"/.devcontainer/experimental.json", devcontainerPath) + assert.Equal(t, tt.workspaceFolder+"/.devcontainer", devcontainerDir) + }) + + t.Run("devcontainer.json in workspace root", func(t *testing.T) { + t.Parallel() + + // given + fs := memfs.New() + err := fs.MkdirAll(tt.workspaceFolder+"", 0o600) + require.NoError(t, err) + _, err = fs.Create(tt.workspaceFolder + "/devcontainer.json") + require.NoError(t, err) + + // when + devcontainerPath, devcontainerDir, err := findDevcontainerJSON(tt.workspaceFolder, options.Options{ + Filesystem: fs, + WorkspaceFolder: "/workspace", + }) + + // then + require.NoError(t, err) + assert.Equal(t, tt.workspaceFolder+"/devcontainer.json", devcontainerPath) + assert.Equal(t, tt.workspaceFolder+"", devcontainerDir) + }) + + t.Run("devcontainer.json in subfolder of .devcontainer", func(t *testing.T) { + t.Parallel() + + // given + fs := memfs.New() + err := fs.MkdirAll(tt.workspaceFolder+"/.devcontainer/sample", 0o600) + require.NoError(t, err) + _, err = fs.Create(tt.workspaceFolder + "/.devcontainer/sample/devcontainer.json") + require.NoError(t, err) + + // when + devcontainerPath, devcontainerDir, err := findDevcontainerJSON(tt.workspaceFolder, options.Options{ + Filesystem: fs, + WorkspaceFolder: "/workspace", + }) + + // then + require.NoError(t, err) + assert.Equal(t, tt.workspaceFolder+"/.devcontainer/sample/devcontainer.json", devcontainerPath) + assert.Equal(t, tt.workspaceFolder+"/.devcontainer/sample", devcontainerDir) + }) }) - - // then - require.Error(t, err) - }) - - t.Run("devcontainer.json is missing", func(t *testing.T) { - t.Parallel() - - // given - fs := memfs.New() - err := fs.MkdirAll("/workspace/.devcontainer", 0o600) - require.NoError(t, err) - - // when - _, _, err = findDevcontainerJSON(options.Options{ - Filesystem: fs, - WorkspaceFolder: "/workspace", - }) - - // then - require.Error(t, err) - }) - - t.Run("default configuration", func(t *testing.T) { - t.Parallel() - - // given - fs := memfs.New() - err := fs.MkdirAll("/workspace/.devcontainer", 0o600) - require.NoError(t, err) - _, err = fs.Create("/workspace/.devcontainer/devcontainer.json") - require.NoError(t, err) - - // when - devcontainerPath, devcontainerDir, err := findDevcontainerJSON(options.Options{ - Filesystem: fs, - WorkspaceFolder: "/workspace", - }) - - // then - require.NoError(t, err) - assert.Equal(t, "/workspace/.devcontainer/devcontainer.json", devcontainerPath) - assert.Equal(t, "/workspace/.devcontainer", devcontainerDir) - }) - - t.Run("overridden .devcontainer directory", func(t *testing.T) { - t.Parallel() - - // given - fs := memfs.New() - err := fs.MkdirAll("/workspace/experimental-devcontainer", 0o600) - require.NoError(t, err) - _, err = fs.Create("/workspace/experimental-devcontainer/devcontainer.json") - require.NoError(t, err) - - // when - devcontainerPath, devcontainerDir, err := findDevcontainerJSON(options.Options{ - Filesystem: fs, - WorkspaceFolder: "/workspace", - DevcontainerDir: "experimental-devcontainer", - }) - - // then - require.NoError(t, err) - assert.Equal(t, "/workspace/experimental-devcontainer/devcontainer.json", devcontainerPath) - assert.Equal(t, "/workspace/experimental-devcontainer", devcontainerDir) - }) - - t.Run("overridden devcontainer.json path", func(t *testing.T) { - t.Parallel() - - // given - fs := memfs.New() - err := fs.MkdirAll("/workspace/.devcontainer", 0o600) - require.NoError(t, err) - _, err = fs.Create("/workspace/.devcontainer/experimental.json") - require.NoError(t, err) - - // when - devcontainerPath, devcontainerDir, err := findDevcontainerJSON(options.Options{ - Filesystem: fs, - WorkspaceFolder: "/workspace", - DevcontainerJSONPath: "experimental.json", - }) - - // then - require.NoError(t, err) - assert.Equal(t, "/workspace/.devcontainer/experimental.json", devcontainerPath) - assert.Equal(t, "/workspace/.devcontainer", devcontainerDir) - }) - - t.Run("devcontainer.json in workspace root", func(t *testing.T) { - t.Parallel() - - // given - fs := memfs.New() - err := fs.MkdirAll("/workspace", 0o600) - require.NoError(t, err) - _, err = fs.Create("/workspace/devcontainer.json") - require.NoError(t, err) - - // when - devcontainerPath, devcontainerDir, err := findDevcontainerJSON(options.Options{ - Filesystem: fs, - WorkspaceFolder: "/workspace", - }) - - // then - require.NoError(t, err) - assert.Equal(t, "/workspace/devcontainer.json", devcontainerPath) - assert.Equal(t, "/workspace", devcontainerDir) - }) - - t.Run("devcontainer.json in subfolder of .devcontainer", func(t *testing.T) { - t.Parallel() - - // given - fs := memfs.New() - err := fs.MkdirAll("/workspace/.devcontainer/sample", 0o600) - require.NoError(t, err) - _, err = fs.Create("/workspace/.devcontainer/sample/devcontainer.json") - require.NoError(t, err) - - // when - devcontainerPath, devcontainerDir, err := findDevcontainerJSON(options.Options{ - Filesystem: fs, - WorkspaceFolder: "/workspace", - }) - - // then - require.NoError(t, err) - assert.Equal(t, "/workspace/.devcontainer/sample/devcontainer.json", devcontainerPath) - assert.Equal(t, "/workspace/.devcontainer/sample", devcontainerDir) - }) + } } diff --git a/git/git.go b/git/git.go index 199e350b..d6c1371c 100644 --- a/git/git.go +++ b/git/git.go @@ -126,6 +126,41 @@ func CloneRepo(ctx context.Context, opts CloneRepoOptions) (bool, error) { return true, nil } +// ShallowCloneRepo will clone the repository at the given URL into the given path +// with a depth of 1. If the destination folder exists and is not empty, the +// clone will not be performed. +// +// The bool returned states whether the repository was cloned or not. +func ShallowCloneRepo(ctx context.Context, opts CloneRepoOptions) error { + opts.Depth = 1 + opts.SingleBranch = true + + if opts.Path == "" { + return errors.New("path is required") + } + + // Avoid clobbering the destination. + if _, err := opts.Storage.Stat(opts.Path); err == nil { + files, err := opts.Storage.ReadDir(opts.Path) + if err != nil { + return fmt.Errorf("read dir %q: %w", opts.Path, err) + } + if len(files) > 0 { + return fmt.Errorf("directory %q is not empty", opts.Path) + } + } + + cloned, err := CloneRepo(ctx, opts) + if err != nil { + return err + } + if !cloned { + return errors.New("repository already exists") + } + + return nil +} + // ReadPrivateKey attempts to read an SSH private key from path // and returns an ssh.Signer. func ReadPrivateKey(path string) (gossh.Signer, error) { @@ -253,3 +288,68 @@ func SetupRepoAuth(options *options.Options) transport.AuthMethod { } return auth } + +func CloneOptionsFromOptions(options options.Options) (CloneRepoOptions, error) { + caBundle, err := options.CABundle() + if err != nil { + return CloneRepoOptions{}, err + } + + cloneOpts := CloneRepoOptions{ + Path: options.WorkspaceFolder, + Storage: options.Filesystem, + Insecure: options.Insecure, + SingleBranch: options.GitCloneSingleBranch, + Depth: int(options.GitCloneDepth), + CABundle: caBundle, + } + + cloneOpts.RepoAuth = SetupRepoAuth(&options) + if options.GitHTTPProxyURL != "" { + cloneOpts.ProxyOptions = transport.ProxyOptions{ + URL: options.GitHTTPProxyURL, + } + } + cloneOpts.RepoURL = options.GitURL + + return cloneOpts, nil +} + +type progressWriter struct { + io.WriteCloser + r io.ReadCloser +} + +func (w *progressWriter) Close() error { + err := w.r.Close() + err2 := w.WriteCloser.Close() + if err != nil { + return err + } + return err2 +} + +func ProgressWriter(write func(line string)) io.WriteCloser { + reader, writer := io.Pipe() + go func() { + data := make([]byte, 4096) + for { + read, err := reader.Read(data) + if err != nil { + return + } + content := data[:read] + for _, line := range strings.Split(string(content), "\r") { + if line == "" { + continue + } + write(strings.TrimSpace(line)) + } + } + }() + + return &progressWriter{ + WriteCloser: writer, + r: reader, + } +} diff --git a/git/git_test.go b/git/git_test.go index 08ab1f93..14656886 100644 --- a/git/git_test.go +++ b/git/git_test.go @@ -169,6 +169,73 @@ func TestCloneRepo(t *testing.T) { } } +func TestShallowCloneRepo(t *testing.T) { + t.Parallel() + + t.Run("NotEmpty", func(t *testing.T) { + t.Parallel() + srvFS := memfs.New() + _ = gittest.NewRepo(t, srvFS, + gittest.Commit(t, "README.md", "Hello, world!", "Many wow!"), + gittest.Commit(t, "foo", "bar!", "Such commit!"), + gittest.Commit(t, "baz", "qux", "V nice!"), + ) + authMW := mwtest.BasicAuthMW("test", "test") + srv := httptest.NewServer(authMW(gittest.NewServer(srvFS))) + + clientFS := memfs.New() + // Not empty. + err := clientFS.MkdirAll("/repo", 0o500) + require.NoError(t, err) + f, err := clientFS.Create("/repo/not-empty") + require.NoError(t, err) + require.NoError(t, f.Close()) + + err = git.ShallowCloneRepo(context.Background(), git.CloneRepoOptions{ + Path: "/repo", + RepoURL: srv.URL, + Storage: clientFS, + RepoAuth: &githttp.BasicAuth{ + Username: "test", + Password: "test", + }, + }) + require.Error(t, err) + }) + t.Run("OK", func(t *testing.T) { + // 2024/08/01 13:22:08 unsupported capability: shallow + // clone "http://127.0.0.1:41499": unexpected client error: unexpected requesting "http://127.0.0.1:41499/git-upload-pack" status code: 500 + t.Skip("The gittest server doesn't support shallow cloning, skip for now...") + + t.Parallel() + srvFS := memfs.New() + _ = gittest.NewRepo(t, srvFS, + gittest.Commit(t, "README.md", "Hello, world!", "Many wow!"), + gittest.Commit(t, "foo", "bar!", "Such commit!"), + gittest.Commit(t, "baz", "qux", "V nice!"), + ) + authMW := mwtest.BasicAuthMW("test", "test") + srv := httptest.NewServer(authMW(gittest.NewServer(srvFS))) + + clientFS := memfs.New() + + err := git.ShallowCloneRepo(context.Background(), git.CloneRepoOptions{ + Path: "/repo", + RepoURL: srv.URL, + Storage: clientFS, + RepoAuth: &githttp.BasicAuth{ + Username: "test", + Password: "test", + }, + }) + require.NoError(t, err) + for _, path := range []string{"README.md", "foo", "baz"} { + _, err := clientFS.Stat(filepath.Join("/repo", path)) + require.NoError(t, err) + } + }) +} + func TestCloneRepoSSH(t *testing.T) { t.Parallel() diff --git a/options/options.go b/options/options.go index 5771b506..3432cee5 100644 --- a/options/options.go +++ b/options/options.go @@ -1,6 +1,9 @@ package options import ( + "crypto/x509" + "encoding/base64" + "fmt" "os" "strings" @@ -146,6 +149,13 @@ type Options struct { // GetCachedImage is a flag to determine if the cached image is available, // and if it is, to return it. GetCachedImage bool + + // RemoteRepoBuildMode uses the remote repository as the source of truth + // when building the image. Enabling this option ignores user changes to + // local files and they will not be reflected in the image. This can be + // used to improving cache utilization when multiple users are building + // working on the same repository. + RemoteRepoBuildMode bool } const envPrefix = "ENVBUILDER_" @@ -417,6 +427,17 @@ func (o *Options) CLI() serpent.OptionSet { Description: "Print the digest of the cached image, if available. " + "Exits with an error if not found.", }, + { + Flag: "remote-repo-build-mode", + Env: WithEnvPrefix("REMOTE_REPO_BUILD_MODE"), + Value: serpent.BoolOf(&o.RemoteRepoBuildMode), + Default: "false", + Description: "Use the remote repository as the source of truth " + + "when building the image. Enabling this option ignores user changes " + + "to local files and they will not be reflected in the image. This can " + + "be used to improving cache utilization when multiple users are building " + + "working on the same repository.", + }, { Flag: "verbose", Env: WithEnvPrefix("VERBOSE"), @@ -482,6 +503,26 @@ func (o *Options) Markdown() string { return sb.String() } +func (o *Options) CABundle() ([]byte, error) { + if o.SSLCertBase64 == "" { + return nil, nil + } + + certPool, err := x509.SystemCertPool() + if err != nil { + return nil, fmt.Errorf("get global system cert pool: %w", err) + } + data, err := base64.StdEncoding.DecodeString(o.SSLCertBase64) + if err != nil { + return nil, fmt.Errorf("base64 decode ssl cert: %w", err) + } + ok := certPool.AppendCertsFromPEM(data) + if !ok { + return nil, fmt.Errorf("failed to append the ssl cert to the global pool: %s", data) + } + return data, nil +} + func skipDeprecatedOptions(options []serpent.Option) []serpent.Option { var activeOptions []serpent.Option diff --git a/options/testdata/options.golden b/options/testdata/options.golden index d59ccd21..0bfbd64a 100644 --- a/options/testdata/options.golden +++ b/options/testdata/options.golden @@ -138,6 +138,13 @@ OPTIONS: Push the built image to a remote registry. This option forces a reproducible build. + --remote-repo-build-mode bool, $ENVBUILDER_REMOTE_REPO_BUILD_MODE (default: false) + Use the remote repository as the source of truth when building the + image. Enabling this option ignores user changes to local files and + they will not be reflected in the image. This can be used to improving + cache utilization when multiple users are building working on the same + repository. + --setup-script string, $ENVBUILDER_SETUP_SCRIPT The script to run before the init script. It runs as the root user regardless of the user specified in the devcontainer.json file. From 2304ce308af427f877fc85c60f80b12f3af63356 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 2 Aug 2024 09:52:41 +0100 Subject: [PATCH 18/81] fix: always add COPY directives in DoCacheProbe (#293) (cherry picked from commit 065bcaaeb4b04a7098c61b4a95beba95d4bc5fa1) --- envbuilder.go | 20 ++++++++++++++++++++ integration/integration_test.go | 5 +++-- options/defaults.go | 3 +++ options/defaults_test.go | 1 + options/options.go | 15 +++++++++++++++ 5 files changed, 42 insertions(+), 2 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index c65ae62f..6bebd2e4 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -1061,6 +1061,26 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) }) } + // We expect an image built and pushed by envbuilder to have the envbuilder + // binary present at a predefined path. In order to correctly replicate the + // build via executor.RunCacheProbe we need to have the *exact* copy of the + // envbuilder binary available used to build the image. + exePath := opts.BinaryPath + // Add an exception for the current running binary in kaniko ignore list + if err := util.AddAllowedPathToDefaultIgnoreList(exePath); err != nil { + return nil, xerrors.Errorf("add exe path to ignore list: %w", err) + } + // Copy the envbuilder binary into the build context. + buildParams.DockerfileContent += fmt.Sprintf(` +COPY --chmod=0755 %s %s +USER root +WORKDIR / +ENTRYPOINT [%q]`, exePath, exePath, exePath) + dst := filepath.Join(buildParams.BuildContext, exePath) + if err := copyFile(exePath, dst); err != nil { + return nil, xerrors.Errorf("copy envbuilder binary to build context: %w", err) + } + stdoutWriter, closeStdout := log.Writer(opts.Logger) defer closeStdout() stderrWriter, closeStderr := log.Writer(opts.Logger) diff --git a/integration/integration_test.go b/integration/integration_test.go index b3fe7bef..297e86a1 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -1114,13 +1114,14 @@ RUN date --utc > /root/date.txt`, testImageAlpine), _, err = remote.Image(ref) require.ErrorContains(t, err, "MANIFEST_UNKNOWN", "expected image to not be present before build + push") - // Then: re-running envbuilder with GET_CACHED_IMAGE should succeed + // Then: re-running envbuilder with GET_CACHED_IMAGE should not succeed, as + // the envbuilder binary is not present in the pushed image. _, err = runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), envbuilderEnv("GET_CACHED_IMAGE", "1"), }}) - require.NoError(t, err) + require.ErrorContains(t, err, "uncached COPY command is not supported in cache probe mode") }) t.Run("CacheAndPush", func(t *testing.T) { diff --git a/options/defaults.go b/options/defaults.go index 18bf12a8..42e48063 100644 --- a/options/defaults.go +++ b/options/defaults.go @@ -55,4 +55,7 @@ func (o *Options) SetDefaults() { if o.WorkspaceFolder == "" { o.WorkspaceFolder = DefaultWorkspaceFolder(o.GitURL) } + if o.BinaryPath == "" { + o.BinaryPath = "/.envbuilder/bin/envbuilder" + } } diff --git a/options/defaults_test.go b/options/defaults_test.go index 156ae16b..48783585 100644 --- a/options/defaults_test.go +++ b/options/defaults_test.go @@ -85,6 +85,7 @@ func TestOptions_SetDefaults(t *testing.T) { Filesystem: chmodfs.New(osfs.New("/")), GitURL: "", WorkspaceFolder: constants.EmptyWorkspaceDir, + BinaryPath: "/.envbuilder/bin/envbuilder", } var actual options.Options diff --git a/options/options.go b/options/options.go index 3432cee5..ff6a8be3 100644 --- a/options/options.go +++ b/options/options.go @@ -156,6 +156,11 @@ type Options struct { // used to improving cache utilization when multiple users are building // working on the same repository. RemoteRepoBuildMode bool + + // BinaryPath is the path to the local envbuilder binary when + // attempting to probe the build cache. This is only relevant when + // GetCachedImage is true. + BinaryPath string } const envPrefix = "ENVBUILDER_" @@ -427,6 +432,13 @@ func (o *Options) CLI() serpent.OptionSet { Description: "Print the digest of the cached image, if available. " + "Exits with an error if not found.", }, + { + Flag: "binary-path", + Env: WithEnvPrefix("BINARY_PATH"), + Value: serpent.StringOf(&o.BinaryPath), + Hidden: true, + Description: "Specify the path to an Envbuilder binary for use when probing the build cache.", + }, { Flag: "remote-repo-build-mode", Env: WithEnvPrefix("REMOTE_REPO_BUILD_MODE"), @@ -485,6 +497,9 @@ func (o *Options) Markdown() string { _, _ = sb.WriteString("| - | - | - | - |\n") for _, opt := range cliOptions { + if opt.Hidden { + continue + } d := opt.Default if d != "" { d = "`" + d + "`" From 45a17d81a800834b9afb7652c5b60cf931461258 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Aug 2024 11:20:39 +0000 Subject: [PATCH 19/81] chore: bump golang.org/x/crypto from 0.24.0 to 0.25.0 (#267) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> (cherry picked from commit 2b457fe2a5596445a6a0ba4384b3d1474a3df8aa) --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 6eb8be9f..270cb077 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a go.uber.org/mock v0.4.0 - golang.org/x/crypto v0.24.0 + golang.org/x/crypto v0.25.0 golang.org/x/mod v0.18.0 golang.org/x/sync v0.7.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 @@ -272,8 +272,8 @@ require ( golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect golang.org/x/net v0.26.0 // indirect golang.org/x/oauth2 v0.20.0 // indirect - golang.org/x/sys v0.21.0 // indirect - golang.org/x/term v0.21.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/term v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.22.0 // indirect diff --git a/go.sum b/go.sum index f51d27b3..86887005 100644 --- a/go.sum +++ b/go.sum @@ -829,8 +829,8 @@ golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= @@ -918,15 +918,15 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.1-0.20230131160137-e7d7f63158de/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 908d7c6fab2f3c8ef0350914abffee1527e0b8dd Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Fri, 2 Aug 2024 18:39:43 +0300 Subject: [PATCH 20/81] fix: only use repo mode in cache probe mode (#294) (cherry picked from commit 5063d1b2ffa02f4482308ca48dc68853ecc05b05) --- envbuilder.go | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 6bebd2e4..d0000689 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -863,35 +863,35 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) var fallbackErr error var cloned bool if opts.GitURL != "" { - cloneOpts, err := git.CloneOptionsFromOptions(opts) - if err != nil { - return nil, fmt.Errorf("git clone options: %w", err) - } + // In cache probe mode we should only attempt to clone the full + // repository if remote repo build mode isn't enabled. + if !opts.RemoteRepoBuildMode { + cloneOpts, err := git.CloneOptionsFromOptions(opts) + if err != nil { + return nil, fmt.Errorf("git clone options: %w", err) + } - endStage := startStage("📦 Cloning %s to %s...", - newColor(color.FgCyan).Sprintf(opts.GitURL), - newColor(color.FgCyan).Sprintf(cloneOpts.Path), - ) + endStage := startStage("📦 Cloning %s to %s...", + newColor(color.FgCyan).Sprintf(opts.GitURL), + newColor(color.FgCyan).Sprintf(cloneOpts.Path), + ) - w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNumber, line) }) - defer w.Close() - cloneOpts.Progress = w + w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNumber, line) }) + defer w.Close() + cloneOpts.Progress = w - cloned, fallbackErr = git.CloneRepo(ctx, cloneOpts) - if fallbackErr == nil { - if cloned { - endStage("📦 Cloned repository!") + cloned, fallbackErr = git.CloneRepo(ctx, cloneOpts) + if fallbackErr == nil { + if cloned { + endStage("📦 Cloned repository!") + } else { + endStage("📦 The repository already exists!") + } } else { - endStage("📦 The repository already exists!") + opts.Logger(log.LevelError, "Failed to clone repository: %s", fallbackErr.Error()) + opts.Logger(log.LevelError, "Falling back to the default image...") } } else { - opts.Logger(log.LevelError, "Failed to clone repository: %s", fallbackErr.Error()) - opts.Logger(log.LevelError, "Falling back to the default image...") - } - - // Always clone the repo in remote repo build mode into a location that - // we control that isn't affected by the users changes. - if opts.RemoteRepoBuildMode { cloneOpts, err := git.CloneOptionsFromOptions(opts) if err != nil { return nil, fmt.Errorf("git clone options: %w", err) From 069a0aa33a440c4e7dd4ce90b9529ea67c29eac3 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Fri, 2 Aug 2024 19:12:48 +0300 Subject: [PATCH 21/81] fix: nil ptr deref in Run and RunCacheProbe (#295) (cherry picked from commit df6597a0772c6ca766f6337ce9194a515960c70f) --- envbuilder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index d0000689..f24dd14a 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -153,7 +153,7 @@ func Run(ctx context.Context, opts options.Options) error { endStage("📦 Cloned repository!") buildTimeWorkspaceFolder = cloneOpts.Path } else { - opts.Logger(log.LevelError, "Failed to clone repository for remote repo mode: %s", err.Error()) + opts.Logger(log.LevelError, "Failed to clone repository for remote repo mode: %s", fallbackErr.Error()) opts.Logger(log.LevelError, "Falling back to the default image...") } } @@ -912,7 +912,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) endStage("📦 Cloned repository!") buildTimeWorkspaceFolder = cloneOpts.Path } else { - opts.Logger(log.LevelError, "Failed to clone repository for remote repo mode: %s", err.Error()) + opts.Logger(log.LevelError, "Failed to clone repository for remote repo mode: %s", fallbackErr.Error()) opts.Logger(log.LevelError, "Falling back to the default image...") } } From 303ce81fbc8768d03a601e838145e2fa11cac5da Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Sat, 3 Aug 2024 19:38:47 +0100 Subject: [PATCH 22/81] feat: support starting from an already-built image (#296) - Extracts 'magic directives' to constants.go. - Adds 'magic image' file to signify envbuilder should skip the destructive 'build image' stage. - Modifies logic for copying binary into built image: we now copy to build context and remove it after build finishes to avoid leaving around files owned by root:root. Also ensures files are created with consistent permissions. (cherry picked from commit 6afe89e6950e93487245ef3d43bc81597a1c747a) --- cmd/envbuilder/main.go | 1 + constants/constants.go | 29 +++++++ envbuilder.go | 166 ++++++++++++++++++++++++++--------------- go.mod | 2 +- go.sum | 4 +- 5 files changed, 140 insertions(+), 62 deletions(-) diff --git a/cmd/envbuilder/main.go b/cmd/envbuilder/main.go index 1910568e..410e0897 100644 --- a/cmd/envbuilder/main.go +++ b/cmd/envbuilder/main.go @@ -68,6 +68,7 @@ func envbuilderCmd() serpent.Command { img, err := envbuilder.RunCacheProbe(inv.Context(), o) if err != nil { o.Logger(log.LevelError, "error: %s", err) + return err } digest, err := img.Digest() if err != nil { diff --git a/constants/constants.go b/constants/constants.go index 042660dd..fefa1394 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -2,6 +2,7 @@ package constants import ( "errors" + "fmt" "path/filepath" ) @@ -32,4 +33,32 @@ var ( // MagicFile is the location of the build context when // using remote build mode. MagicRemoteRepoDir = filepath.Join(MagicDir, "repo") + + // MagicBinaryLocation is the expected location of the envbuilder binary + // inside a builder image. + MagicBinaryLocation = filepath.Join(MagicDir, "bin", "envbuilder") + + // MagicImage is a file that is created in the image when + // envbuilder has already been run. This is used to skip + // the destructive initial build step when 'resuming' envbuilder + // from a previously built image. + MagicImage = filepath.Join(MagicDir, "image") + + // MagicTempDir is a directory inside the build context inside which + // we place files referenced by MagicDirectives. + MagicTempDir = ".envbuilder.tmp" + + // MagicDirectives are directives automatically appended to Dockerfiles + // when pushing the image. These directives allow the built image to be + // 're-used'. + MagicDirectives = fmt.Sprintf(` +COPY --chmod=0755 %[1]s %[2]s +COPY --chmod=0644 %[3]s %[4]s +USER root +WORKDIR / +ENTRYPOINT [%[2]q] +`, + ".envbuilder.tmp/envbuilder", MagicBinaryLocation, + ".envbuilder.tmp/image", MagicImage, + ) ) diff --git a/envbuilder.go b/envbuilder.go index f24dd14a..215ccc48 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -27,6 +27,7 @@ import ( "github.com/coder/envbuilder/constants" "github.com/coder/envbuilder/git" "github.com/coder/envbuilder/options" + "github.com/go-git/go-billy/v5" "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/GoogleContainerTools/kaniko/pkg/creds" @@ -311,26 +312,58 @@ func Run(ctx context.Context, opts options.Options) error { } // In order to allow 'resuming' envbuilder, embed the binary into the image - // if it is being pushed + // if it is being pushed. + // As these files will be owned by root, it is considerate to clean up + // after we're done! + cleanupBuildContext := func() {} if opts.PushImage { - exePath, err := os.Executable() - if err != nil { - return xerrors.Errorf("get exe path: %w", err) + // Add exceptions in Kaniko's ignorelist for these magic files we add. + if err := util.AddAllowedPathToDefaultIgnoreList(opts.BinaryPath); err != nil { + return fmt.Errorf("add envbuilder binary to ignore list: %w", err) + } + if err := util.AddAllowedPathToDefaultIgnoreList(constants.MagicImage); err != nil { + return fmt.Errorf("add magic image file to ignore list: %w", err) } - // Add an exception for the current running binary in kaniko ignore list - if err := util.AddAllowedPathToDefaultIgnoreList(exePath); err != nil { - return xerrors.Errorf("add exe path to ignore list: %w", err) + magicTempDir := filepath.Join(buildParams.BuildContext, constants.MagicTempDir) + if err := opts.Filesystem.MkdirAll(magicTempDir, 0o755); err != nil { + return fmt.Errorf("create magic temp dir in build context: %w", err) } + // Add the magic directives that embed the binary into the built image. + buildParams.DockerfileContent += constants.MagicDirectives // Copy the envbuilder binary into the build context. - buildParams.DockerfileContent += fmt.Sprintf(` -COPY --chmod=0755 %s %s -USER root -WORKDIR / -ENTRYPOINT [%q]`, exePath, exePath, exePath) - dst := filepath.Join(buildParams.BuildContext, exePath) - if err := copyFile(exePath, dst); err != nil { - return xerrors.Errorf("copy running binary to build context: %w", err) + // External callers will need to specify the path to the desired envbuilder binary. + envbuilderBinDest := filepath.Join( + magicTempDir, + filepath.Base(constants.MagicBinaryLocation), + ) + // Also touch the magic file that signifies the image has been built! + magicImageDest := filepath.Join( + magicTempDir, + filepath.Base(constants.MagicImage), + ) + // Clean up after build! + var cleanupOnce sync.Once + cleanupBuildContext = func() { + cleanupOnce.Do(func() { + for _, path := range []string{magicImageDest, envbuilderBinDest, magicTempDir} { + if err := opts.Filesystem.Remove(path); err != nil { + opts.Logger(log.LevelWarn, "failed to clean up magic temp dir from build context: %w", err) + } + } + }) + } + defer cleanupBuildContext() + + opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, envbuilderBinDest) + if err := copyFile(opts.Filesystem, opts.BinaryPath, envbuilderBinDest, 0o755); err != nil { + return fmt.Errorf("copy envbuilder binary to build context: %w", err) + } + + opts.Logger(log.LevelDebug, "touching magic image file at %q in build context %q", magicImageDest, buildParams.BuildContext) + if err := touchFile(opts.Filesystem, magicImageDest, 0o755); err != nil { + return fmt.Errorf("touch magic image file in build context: %w", err) } + } // temp move of all ro mounts @@ -354,8 +387,10 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) stderrWriter, closeStderr := log.Writer(opts.Logger) defer closeStderr() build := func() (v1.Image, error) { - _, err := opts.Filesystem.Stat(constants.MagicFile) - if err == nil && opts.SkipRebuild { + defer cleanupBuildContext() + _, alreadyBuiltErr := opts.Filesystem.Stat(constants.MagicFile) + _, isImageErr := opts.Filesystem.Stat(constants.MagicImage) + if (alreadyBuiltErr == nil && opts.SkipRebuild) || isImageErr == nil { endStage := startStage("🏗️ Skipping build because of cache...") imageRef, err := devcontainer.ImageFromDockerfile(buildParams.DockerfileContent) if err != nil { @@ -381,26 +416,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) if err := maybeDeleteFilesystem(opts.Logger, opts.ForceSafe); err != nil { return nil, fmt.Errorf("delete filesystem: %w", err) } - /* - stdoutReader, stdoutWriter := io.Pipe() - stderrReader, stderrWriter := io.Pipe() - defer stdoutReader.Close() - defer stdoutWriter.Close() - defer stderrReader.Close() - defer stderrWriter.Close() - go func() { - scanner := bufio.NewScanner(stdoutReader) - for scanner.Scan() { - opts.Logger(log.LevelInfo, "%s", scanner.Text()) - } - }() - go func() { - scanner := bufio.NewScanner(stderrReader) - for scanner.Scan() { - opts.Logger(log.LevelInfo, "%s", scanner.Text()) - } - }() - */ + cacheTTL := time.Hour * 24 * 7 if opts.CacheTTLDays != 0 { cacheTTL = time.Hour * 24 * time.Duration(opts.CacheTTLDays) @@ -1064,23 +1080,41 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) // We expect an image built and pushed by envbuilder to have the envbuilder // binary present at a predefined path. In order to correctly replicate the // build via executor.RunCacheProbe we need to have the *exact* copy of the - // envbuilder binary available used to build the image. - exePath := opts.BinaryPath - // Add an exception for the current running binary in kaniko ignore list - if err := util.AddAllowedPathToDefaultIgnoreList(exePath); err != nil { - return nil, xerrors.Errorf("add exe path to ignore list: %w", err) - } + // envbuilder binary available used to build the image and we also need to + // add the magic directives to the Dockerfile content. + buildParams.DockerfileContent += constants.MagicDirectives + magicTempDir := filepath.Join(buildParams.BuildContext, constants.MagicTempDir) + if err := opts.Filesystem.MkdirAll(magicTempDir, 0o755); err != nil { + return nil, fmt.Errorf("create magic temp dir in build context: %w", err) + } + envbuilderBinDest := filepath.Join( + magicTempDir, + filepath.Base(constants.MagicBinaryLocation), + ) + // Copy the envbuilder binary into the build context. - buildParams.DockerfileContent += fmt.Sprintf(` -COPY --chmod=0755 %s %s -USER root -WORKDIR / -ENTRYPOINT [%q]`, exePath, exePath, exePath) - dst := filepath.Join(buildParams.BuildContext, exePath) - if err := copyFile(exePath, dst); err != nil { + opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, buildParams.BuildContext) + if err := copyFile(opts.Filesystem, opts.BinaryPath, envbuilderBinDest, 0o755); err != nil { return nil, xerrors.Errorf("copy envbuilder binary to build context: %w", err) } + // Also touch the magic file that signifies the image has been built! + magicImageDest := filepath.Join( + magicTempDir, + filepath.Base(constants.MagicImage), + ) + if err := touchFile(opts.Filesystem, magicImageDest, 0o755); err != nil { + return nil, fmt.Errorf("touch magic image file in build context: %w", err) + } + defer func() { + // Clean up after we're done! + for _, path := range []string{magicImageDest, envbuilderBinDest, magicTempDir} { + if err := opts.Filesystem.Remove(path); err != nil { + opts.Logger(log.LevelWarn, "failed to clean up magic temp dir from build context: %w", err) + } + } + }() + stdoutWriter, closeStdout := log.Writer(opts.Logger) defer closeStdout() stderrWriter, closeStderr := log.Writer(opts.Logger) @@ -1138,8 +1172,8 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) }, SrcContext: buildParams.BuildContext, - // For cached image utilization, produce reproducible builds. - Reproducible: opts.PushImage, + // When performing a cache probe, always perform reproducible snapshots. + Reproducible: true, } endStage := startStage("🏗️ Checking for cached image...") @@ -1382,24 +1416,38 @@ func maybeDeleteFilesystem(logger log.Func, force bool) error { return util.DeleteFilesystem() } -func copyFile(src, dst string) error { - content, err := os.ReadFile(src) +func copyFile(fs billy.Filesystem, src, dst string, mode fs.FileMode) error { + srcF, err := fs.Open(src) if err != nil { - return fmt.Errorf("read file failed: %w", err) + return fmt.Errorf("open src file: %w", err) } + defer srcF.Close() - err = os.MkdirAll(filepath.Dir(dst), 0o755) + err = fs.MkdirAll(filepath.Dir(dst), mode) if err != nil { - return fmt.Errorf("mkdir all failed: %w", err) + return fmt.Errorf("create destination dir failed: %w", err) } - err = os.WriteFile(dst, content, 0o644) + dstF, err := fs.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode) if err != nil { - return fmt.Errorf("write file failed: %w", err) + return fmt.Errorf("open dest file for writing: %w", err) + } + defer dstF.Close() + + if _, err := io.Copy(dstF, srcF); err != nil { + return fmt.Errorf("copy failed: %w", err) } return nil } +func touchFile(fs billy.Filesystem, dst string, mode fs.FileMode) error { + f, err := fs.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) + if err != nil { + return xerrors.Errorf("failed to touch file: %w", err) + } + return f.Close() +} + func initDockerConfigJSON(dockerConfigBase64 string) (func() error, error) { var cleanupOnce sync.Once noop := func() error { return nil } diff --git a/go.mod b/go.mod index 270cb077..e06edc53 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22.4 // There are a few options we need added to Kaniko! // See: https://github.com/GoogleContainerTools/kaniko/compare/main...coder:kaniko:main -replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240717115058-0ba2908ca4d3 +replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240803153527-10d1800455b9 // Required to import codersdk due to gvisor dependency. replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20240702054557-aa558fbe5374 diff --git a/go.sum b/go.sum index 86887005..25bdf7fc 100644 --- a/go.sum +++ b/go.sum @@ -171,8 +171,8 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352 h1:L/EjCuZxs5tOcqqCaASj/nu65TRYEFcTt8qRQfHZXX0= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352/go.mod h1:P1KoQSgnKEAG6Mnd3YlGzAophty+yKA9VV48LpfNRvo= -github.com/coder/kaniko v0.0.0-20240717115058-0ba2908ca4d3 h1:Q7L6cjKfw3DIyhKIcgCJEmgxnUTBajmMDrHxXvxgBZs= -github.com/coder/kaniko v0.0.0-20240717115058-0ba2908ca4d3/go.mod h1:YMK7BlxerzLlMwihGxNWUaFoN9LXCij4P+w/8/fNlcM= +github.com/coder/kaniko v0.0.0-20240803153527-10d1800455b9 h1:d01T5YbPN1yc1mXjIXG59YcQQoT/9idvqFErjWHfsZ4= +github.com/coder/kaniko v0.0.0-20240803153527-10d1800455b9/go.mod h1:YMK7BlxerzLlMwihGxNWUaFoN9LXCij4P+w/8/fNlcM= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 h1:3A0ES21Ke+FxEM8CXx9n47SZOKOpgSE1bbJzlE4qPVs= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0/go.mod h1:5UuS2Ts+nTToAMeOjNlnHFkPahrtDkmpydBen/3wgZc= github.com/coder/quartz v0.1.0 h1:cLL+0g5l7xTf6ordRnUMMiZtRE8Sq5LxpghS63vEXrQ= From 2a59ce08908a48f9e8236c501df6e3d37e831b18 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Mon, 5 Aug 2024 12:45:24 +0300 Subject: [PATCH 23/81] add hidden remote repo dir option to change clone path (#297) (cherry picked from commit c1f9917dfb61f124c687c978a734321fda3a7397) --- envbuilder.go | 4 ++-- options/options.go | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 215ccc48..a16f2fb4 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -138,7 +138,7 @@ func Run(ctx context.Context, opts options.Options) error { if err != nil { return fmt.Errorf("git clone options: %w", err) } - cloneOpts.Path = constants.MagicRemoteRepoDir + cloneOpts.Path = opts.RemoteRepoDir endStage := startStage("📦 Remote repo build mode enabled, cloning %s to %s for build context...", newColor(color.FgCyan).Sprintf(opts.GitURL), @@ -912,7 +912,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) if err != nil { return nil, fmt.Errorf("git clone options: %w", err) } - cloneOpts.Path = constants.MagicRemoteRepoDir + cloneOpts.Path = opts.RemoteRepoDir endStage := startStage("📦 Remote repo build mode enabled, cloning %s to %s for build context...", newColor(color.FgCyan).Sprintf(opts.GitURL), diff --git a/options/options.go b/options/options.go index ff6a8be3..d7bd66b3 100644 --- a/options/options.go +++ b/options/options.go @@ -7,6 +7,7 @@ import ( "os" "strings" + "github.com/coder/envbuilder/constants" "github.com/coder/envbuilder/log" "github.com/coder/serpent" "github.com/go-git/go-billy/v5" @@ -157,6 +158,10 @@ type Options struct { // working on the same repository. RemoteRepoBuildMode bool + // RemoteRepoDir is the destination directory for the cloned repo when using + // remote repo build mode. + RemoteRepoDir string + // BinaryPath is the path to the local envbuilder binary when // attempting to probe the build cache. This is only relevant when // GetCachedImage is true. @@ -450,6 +455,14 @@ func (o *Options) CLI() serpent.OptionSet { "be used to improving cache utilization when multiple users are building " + "working on the same repository.", }, + { + Flag: "remote-repo-dir", + Env: WithEnvPrefix("REMOTE_REPO_DIR"), + Value: serpent.StringOf(&o.RemoteRepoDir), + Default: constants.MagicRemoteRepoDir, + Hidden: true, + Description: "Specify the destination directory for the cloned repo when using remote repo build mode.", + }, { Flag: "verbose", Env: WithEnvPrefix("VERBOSE"), From e7acbad4064b54243c57f2579c8341535132affe Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 5 Aug 2024 12:53:46 +0100 Subject: [PATCH 24/81] feat: embed version info into binary (#298) (cherry picked from commit 6d5f48990f3fee9bb650166864b876ce6d29d17c) --- .github/workflows/ci.yaml | 6 +--- buildinfo/version.go | 71 +++++++++++++++++++++++++++++++++++++ envbuilder.go | 5 +-- scripts/build.sh | 14 ++++++-- scripts/lib.sh | 41 ++++++++++++++++++++++ scripts/version.sh | 74 +++++++++++++++++++++++++++++++++++++-- 6 files changed, 198 insertions(+), 13 deletions(-) create mode 100644 buildinfo/version.go create mode 100644 scripts/lib.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4e0d51cc..ecb23ed1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -99,18 +99,15 @@ jobs: - name: Build if: github.event_name == 'pull_request' run: | - VERSION=$(./scripts/version.sh)-dev-$(git rev-parse --short HEAD) BASE=ghcr.io/coder/envbuilder-preview ./scripts/build.sh \ --arch=amd64 \ - --base=$BASE \ - --tag=$VERSION + --base=$BASE - name: Build and Push if: github.ref == 'refs/heads/main' run: | - VERSION=$(./scripts/version.sh)-dev-$(git rev-parse --short HEAD) BASE=ghcr.io/coder/envbuilder-preview ./scripts/build.sh \ @@ -118,5 +115,4 @@ jobs: --arch=arm64 \ --arch=arm \ --base=$BASE \ - --tag=$VERSION \ --push diff --git a/buildinfo/version.go b/buildinfo/version.go new file mode 100644 index 00000000..86f35348 --- /dev/null +++ b/buildinfo/version.go @@ -0,0 +1,71 @@ +package buildinfo + +import ( + "fmt" + "runtime/debug" + "sync" + + "golang.org/x/mod/semver" +) + +const ( + noVersion = "v0.0.0" + develPreRelease = "devel" +) + +var ( + buildInfo *debug.BuildInfo + buildInfoValid bool + readBuildInfo sync.Once + + version string + readVersion sync.Once + + // Injected with ldflags at build time + tag string +) + +func revision() (string, bool) { + return find("vcs.revision") +} + +func find(key string) (string, bool) { + readBuildInfo.Do(func() { + buildInfo, buildInfoValid = debug.ReadBuildInfo() + }) + if !buildInfoValid { + panic("could not read build info") + } + for _, setting := range buildInfo.Settings { + if setting.Key != key { + continue + } + return setting.Value, true + } + return "", false +} + +// Version returns the semantic version of the build. +// Use golang.org/x/mod/semver to compare versions. +func Version() string { + readVersion.Do(func() { + revision, valid := revision() + if valid { + revision = "+" + revision[:7] + } + if tag == "" { + // This occurs when the tag hasn't been injected, + // like when using "go run". + // -+ + version = fmt.Sprintf("%s-%s%s", noVersion, develPreRelease, revision) + return + } + version = "v" + tag + // The tag must be prefixed with "v" otherwise the + // semver library will return an empty string. + if semver.Build(version) == "" { + version += revision + } + }) + return version +} diff --git a/envbuilder.go b/envbuilder.go index a16f2fb4..7a61159e 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -24,6 +24,7 @@ import ( "syscall" "time" + "github.com/coder/envbuilder/buildinfo" "github.com/coder/envbuilder/constants" "github.com/coder/envbuilder/git" "github.com/coder/envbuilder/options" @@ -89,7 +90,7 @@ func Run(ctx context.Context, opts options.Options) error { } } - opts.Logger(log.LevelInfo, "%s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder")) + opts.Logger(log.LevelInfo, "%s %s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder"), buildinfo.Version()) cleanupDockerConfigJSON, err := initDockerConfigJSON(opts.DockerConfigBase64) if err != nil { @@ -863,7 +864,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) } } - opts.Logger(log.LevelInfo, "%s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder")) + opts.Logger(log.LevelInfo, "%s %s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder"), buildinfo.Version()) cleanupDockerConfigJSON, err := initDockerConfigJSON(opts.DockerConfigBase64) if err != nil { diff --git a/scripts/build.sh b/scripts/build.sh index e186dc02..40545199 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -6,7 +6,7 @@ set -euo pipefail archs=() push=false base="envbuilder" -tag="latest" +tag="" for arg in "$@"; do if [[ $arg == --arch=* ]]; then @@ -30,6 +30,10 @@ if [ ${#archs[@]} -eq 0 ]; then archs=( "$current" ) fi +if [[ -z "${tag}" ]]; then + tag=$(./version.sh) +fi + # We have to use docker buildx to tag multiple images with # platforms tragically, so we have to create a builder. BUILDER_NAME="envbuilder" @@ -46,9 +50,11 @@ fi # Ensure the builder is bootstrapped and ready to use docker buildx inspect --bootstrap &> /dev/null +ldflags=(-X "'github.com/coder/envbuilder/buildinfo.tag=$tag'") + for arch in "${archs[@]}"; do echo "Building for $arch..." - GOARCH=$arch CGO_ENABLED=0 go build -o "./envbuilder-${arch}" ../cmd/envbuilder & + GOARCH=$arch CGO_ENABLED=0 go build -ldflags="${ldflags[*]}" -o "./envbuilder-${arch}" ../cmd/envbuilder & done wait @@ -62,10 +68,12 @@ else args+=( --load ) fi +# coerce semver build tags into something docker won't complain about +tag="${tag//\+/-}" docker buildx build --builder $BUILDER_NAME "${args[@]}" -t "${base}:${tag}" -t "${base}:latest" -f Dockerfile . # Check if archs contains the current. If so, then output a message! if [[ -z "${CI:-}" ]] && [[ " ${archs[*]} " =~ ${current} ]]; then docker tag "${base}:${tag}" envbuilder:latest - echo "Tagged $current as envbuilder:latest!" + echo "Tagged $current as ${base}:${tag} ${base}:latest!" fi diff --git a/scripts/lib.sh b/scripts/lib.sh new file mode 100644 index 00000000..3fbcd979 --- /dev/null +++ b/scripts/lib.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# This script is meant to be sourced by other scripts. To source this script: +# # shellcheck source=scripts/lib.sh +# source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +set -euo pipefail + +# Avoid sourcing this script multiple times to guard against when lib.sh +# is used by another sourced script, it can lead to confusing results. +if [[ ${SCRIPTS_LIB_IS_SOURCED:-0} == 1 ]]; then + return +fi +# Do not export to avoid this value being inherited by non-sourced +# scripts. +SCRIPTS_LIB_IS_SOURCED=1 + +# We have to define realpath before these otherwise it fails on Mac's bash. +SCRIPT="${BASH_SOURCE[1]:-${BASH_SOURCE[0]}}" +SCRIPT_DIR="$(realpath "$(dirname "$SCRIPT")")" + +function project_root { + # Nix sets $src in derivations! + [[ -n "${src:-}" ]] && echo "$src" && return + + # Try to use `git rev-parse --show-toplevel` to find the project root. + # If this directory is not a git repository, this command will fail. + git rev-parse --show-toplevel 2>/dev/null && return +} + +PROJECT_ROOT="$(cd "$SCRIPT_DIR" && realpath "$(project_root)")" + +# cdroot changes directory to the root of the repository. +cdroot() { + cd "$PROJECT_ROOT" || error "Could not change directory to '$PROJECT_ROOT'" +} + +# log prints a message to stderr +log() { + echo "$*" 1>&2 +} diff --git a/scripts/version.sh b/scripts/version.sh index 31968d27..17c8f727 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -1,10 +1,78 @@ #!/usr/bin/env bash +# This script generates the version string used by Envbuilder, including for dev +# versions. Note: the version returned by this script will NOT include the "v" +# prefix that is included in the Git tag. +# +# If $ENVBUILDER_RELEASE is set to "true", the returned version will equal the +# current git tag. If the current commit is not tagged, this will fail. +# +# If $ENVBUILDER_RELEASE is not set, the returned version will always be a dev +# version. + set -euo pipefail -cd "$(dirname "${BASH_SOURCE[0]}")" +# shellcheck source=scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +cdroot + +if [[ -n "${ENVBUILDER_FORCE_VERSION:-}" ]]; then + echo "${ENVBUILDER_FORCE_VERSION}" + exit 0 +fi + +# To make contributing easier, if there are no tags, we'll use a default +# version. +tag_list=$(git tag) +if [[ -z ${tag_list} ]]; then + log + log "INFO(version.sh): It appears you've checked out a fork or shallow clone of Envbuilder." + log "INFO(version.sh): By default GitHub does not include tags when forking." + log "INFO(version.sh): We will use the default version 0.0.1 for this build." + log "INFO(version.sh): To pull tags from upstream, use the following commands:" + log "INFO(version.sh): - git remote add upstream https://github.com/coder/envbuilder.git" + log "INFO(version.sh): - git fetch upstream" + log + last_tag="v0.0.1" +else + current_commit=$(git rev-parse HEAD) + # Try to find the last tag that contains the current commit + last_tag=$(git tag --contains "$current_commit" --sort=version:refname | head -n 1) + # If there is no tag that contains the current commit, + # get the latest tag sorted by semver. + if [[ -z "${last_tag}" ]]; then + last_tag=$(git tag --sort=version:refname | tail -n 1) + fi +fi + +version="${last_tag}" + +# If the HEAD has extra commits since the last tag then we are in a dev version. +# +# Dev versions are denoted by the "-dev+" suffix with a trailing commit short +# SHA. +if [[ "${ENVBUILDER_RELEASE:-}" == *t* ]]; then + # $last_tag will equal `git describe --always` if we currently have the tag + # checked out. + if [[ "${last_tag}" != "$(git describe --always)" ]]; then + # make won't exit on $(shell cmd) failures, so we have to kill it :( + if [[ "$(ps -o comm= "${PPID}" || true)" == *make* ]]; then + log "ERROR: version.sh: the current commit is not tagged with an annotated tag" + kill "${PPID}" || true + exit 1 + fi + + error "version.sh: the current commit is not tagged with an annotated tag" + fi +else + rev=$(git rev-parse --short HEAD) + version="0.0.0+dev-${rev}" + # If the git repo has uncommitted changes, mark the version string as 'dirty'. + dirty_files=$(git ls-files --other --modified --exclude-standard) + if [[ -n "${dirty_files}" ]]; then + version+="-dirty" + fi +fi -last_tag="$(git describe --tags --abbrev=0)" -version="$last_tag" # Remove the "v" prefix. echo "${version#v}" From eab2f9af7db4415109c395d6fad303000fd12610 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Aug 2024 10:18:14 +0100 Subject: [PATCH 25/81] chore: bump github.com/skeema/knownhosts from 1.2.2 to 1.3.0 (#300) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e06edc53..7622ea85 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/otiai10/copy v1.14.0 github.com/prometheus/procfs v0.15.0 github.com/sirupsen/logrus v1.9.3 - github.com/skeema/knownhosts v1.2.2 + github.com/skeema/knownhosts v1.3.0 github.com/stretchr/testify v1.9.0 github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a go.uber.org/mock v0.4.0 diff --git a/go.sum b/go.sum index 25bdf7fc..19f3f69a 100644 --- a/go.sum +++ b/go.sum @@ -680,8 +680,8 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/skeema/knownhosts v1.2.2 h1:Iug2P4fLmDw9f41PB6thxUkNUkJzB5i+1/exaj40L3A= -github.com/skeema/knownhosts v1.2.2/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo= +github.com/skeema/knownhosts v1.3.0 h1:AM+y0rI04VksttfwjkSTNQorvGqmwATnvnAHpSgc0LY= +github.com/skeema/knownhosts v1.3.0/go.mod h1:sPINvnADmT/qYH1kfv+ePMmOBTH6Tbl7b5LvTDjFK7M= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= From fd910d5a5f3519c37aa9f8e5af46a7939ff955ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Aug 2024 09:52:36 +0100 Subject: [PATCH 26/81] chore: bump github.com/google/go-containerregistry from 0.19.1 to 0.20.2 (#302) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 7622ea85..3be7733a 100644 --- a/go.mod +++ b/go.mod @@ -19,13 +19,13 @@ require ( github.com/coder/serpent v0.7.0 github.com/containerd/containerd v1.7.15 github.com/distribution/distribution/v3 v3.0.0-alpha.1 - github.com/docker/cli v26.1.0+incompatible + github.com/docker/cli v27.1.1+incompatible github.com/docker/docker v26.1.0+incompatible github.com/fatih/color v1.17.0 github.com/gliderlabs/ssh v0.3.7 github.com/go-git/go-billy/v5 v5.5.0 github.com/go-git/go-git/v5 v5.12.0 - github.com/google/go-containerregistry v0.19.1 + github.com/google/go-containerregistry v0.20.2 github.com/google/uuid v1.6.0 github.com/hashicorp/go-multierror v1.1.1 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 diff --git a/go.sum b/go.sum index 19f3f69a..e9f3dae1 100644 --- a/go.sum +++ b/go.sum @@ -230,8 +230,8 @@ github.com/distribution/distribution/v3 v3.0.0-alpha.1 h1:jn7I1gvjOvmLztH1+1cLiU github.com/distribution/distribution/v3 v3.0.0-alpha.1/go.mod h1:LCp4JZp1ZalYg0W/TN05jarCQu+h4w7xc7ZfQF4Y/cY= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/cli v26.1.0+incompatible h1:+nwRy8Ocd8cYNQ60mozDDICICD8aoFGtlPXifX/UQ3Y= -github.com/docker/cli v26.1.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v27.1.1+incompatible h1:goaZxOqs4QKxznZjjBWKONQci/MywhtRv2oNn0GkeZE= +github.com/docker/cli v27.1.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v26.1.0+incompatible h1:W1G9MPNbskA6VZWL7b3ZljTh0pXI68FpINx0GKaOdaM= @@ -380,8 +380,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-containerregistry v0.19.1 h1:yMQ62Al6/V0Z7CqIrrS1iYoA5/oQCm88DeNujc7C1KY= -github.com/google/go-containerregistry v0.19.1/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= +github.com/google/go-containerregistry v0.20.2 h1:B1wPJ1SN/S7pB+ZAimcciVD+r+yV/l/DSArMxlbwseo= +github.com/google/go-containerregistry v0.20.2/go.mod h1:z38EKdKh4h7IP2gSfUUqEvalZBqs6AoLeWfUy34nQC8= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= From ca806825d76d5a3c22039705dea2ece6d3396ab0 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 7 Aug 2024 17:37:59 +0100 Subject: [PATCH 27/81] ci: fix version script and update release.yaml (#303) --- .github/workflows/release.yaml | 17 +++++++++++++---- scripts/lib.sh | 6 ++++++ scripts/version.sh | 12 ++---------- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 3f03b2fd..c5af4938 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -19,7 +19,9 @@ jobs: name: Build and publish runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + fetch-tags: true - name: Echo Go Cache Paths id: go-cache-paths @@ -44,11 +46,18 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and Push + - name: Get version + id: get-version + env: + ENVBUILDER_RELEASE: "t" run: | - VERSION=$(./scripts/version.sh) - BASE=ghcr.io/coder/envbuilder + echo "ENVBUILDER_VERSION=$(./scripts.version.sh)" >> $GITHUB_OUTPUT + - name: Build and Push + env: + VERSION: "${{ steps.get-version.outputs.ENVBUILDER_VERSION }}" + BASE: "ghcr.io/coder/envbuilder" + run: | ./scripts/build.sh \ --arch=amd64 \ --arch=arm64 \ diff --git a/scripts/lib.sh b/scripts/lib.sh index 3fbcd979..b39c0b9d 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -39,3 +39,9 @@ cdroot() { log() { echo "$*" 1>&2 } + +# error prints an error message and returns an error exit code. +error() { + log "ERROR: $*" + exit 1 +} diff --git a/scripts/version.sh b/scripts/version.sh index 17c8f727..75dafcc4 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -54,18 +54,11 @@ if [[ "${ENVBUILDER_RELEASE:-}" == *t* ]]; then # $last_tag will equal `git describe --always` if we currently have the tag # checked out. if [[ "${last_tag}" != "$(git describe --always)" ]]; then - # make won't exit on $(shell cmd) failures, so we have to kill it :( - if [[ "$(ps -o comm= "${PPID}" || true)" == *make* ]]; then - log "ERROR: version.sh: the current commit is not tagged with an annotated tag" - kill "${PPID}" || true - exit 1 - fi - error "version.sh: the current commit is not tagged with an annotated tag" fi else - rev=$(git rev-parse --short HEAD) - version="0.0.0+dev-${rev}" + rev=$(git log -1 --format='%h' HEAD) + version+="+dev-${rev}" # If the git repo has uncommitted changes, mark the version string as 'dirty'. dirty_files=$(git ls-files --other --modified --exclude-standard) if [[ -n "${dirty_files}" ]]; then @@ -73,6 +66,5 @@ else fi fi - # Remove the "v" prefix. echo "${version#v}" From 8b7974b461ea50411f88c281504c858c8fea6bca Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 7 Aug 2024 16:10:28 +0100 Subject: [PATCH 28/81] chore(deps): update kaniko (#301) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3be7733a..972a1f29 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22.4 // There are a few options we need added to Kaniko! // See: https://github.com/GoogleContainerTools/kaniko/compare/main...coder:kaniko:main -replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240803153527-10d1800455b9 +replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240807142221-ffc5e60fca41 // Required to import codersdk due to gvisor dependency. replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20240702054557-aa558fbe5374 diff --git a/go.sum b/go.sum index e9f3dae1..cc9e8546 100644 --- a/go.sum +++ b/go.sum @@ -171,8 +171,8 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352 h1:L/EjCuZxs5tOcqqCaASj/nu65TRYEFcTt8qRQfHZXX0= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352/go.mod h1:P1KoQSgnKEAG6Mnd3YlGzAophty+yKA9VV48LpfNRvo= -github.com/coder/kaniko v0.0.0-20240803153527-10d1800455b9 h1:d01T5YbPN1yc1mXjIXG59YcQQoT/9idvqFErjWHfsZ4= -github.com/coder/kaniko v0.0.0-20240803153527-10d1800455b9/go.mod h1:YMK7BlxerzLlMwihGxNWUaFoN9LXCij4P+w/8/fNlcM= +github.com/coder/kaniko v0.0.0-20240807142221-ffc5e60fca41 h1:1Ye7AcLnuT5IDv6il7Fxo+aqpzlWfedkpraCCwx8Lyo= +github.com/coder/kaniko v0.0.0-20240807142221-ffc5e60fca41/go.mod h1:YMK7BlxerzLlMwihGxNWUaFoN9LXCij4P+w/8/fNlcM= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 h1:3A0ES21Ke+FxEM8CXx9n47SZOKOpgSE1bbJzlE4qPVs= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0/go.mod h1:5UuS2Ts+nTToAMeOjNlnHFkPahrtDkmpydBen/3wgZc= github.com/coder/quartz v0.1.0 h1:cLL+0g5l7xTf6ordRnUMMiZtRE8Sq5LxpghS63vEXrQ= From 65f22890271cf497f0eba1997b44c410642696b0 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 8 Aug 2024 11:05:33 +0100 Subject: [PATCH 29/81] chore(ci): fix issue with checkout action (#306) (cherry picked from commit f6dbfb192b21e6f724bfed7eee98f407fecc7b98) --- .github/workflows/release.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c5af4938..9b035483 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -20,8 +20,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - with: - fetch-tags: true + + # Workaround for actions/checkout#1467 + - name: Fetch tags + run: | + git fetch --tags --depth 1 --force - name: Echo Go Cache Paths id: go-cache-paths From 618ea115fe5978229d103f6b7e6e7ec69e0b5175 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 8 Aug 2024 11:15:37 +0100 Subject: [PATCH 30/81] fixup! ci: fix version script and update release.yaml (#303) --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9b035483..6c83f1e0 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -54,7 +54,7 @@ jobs: env: ENVBUILDER_RELEASE: "t" run: | - echo "ENVBUILDER_VERSION=$(./scripts.version.sh)" >> $GITHUB_OUTPUT + echo "ENVBUILDER_VERSION=$(./scripts/version.sh)" >> $GITHUB_OUTPUT - name: Build and Push env: From fda6bb56ad2c4c332a196f3857323d2b49fcee60 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 9 Aug 2024 16:43:59 +0100 Subject: [PATCH 31/81] chore(docs): s/ENVBUILDER_GIT_SSH_KEY_PATH/ENVBUILDER_GIT_SSH_PRIVATE_KEY_PATH/g (#308) (cherry picked from commit 3f054f6c132a8c2da4afa7f54a552ba8053c9357) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 51642345..e5cc3cfe 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ resource "docker_container" "dev" { If `ENVBUILDER_GIT_URL` does not start with `http://` or `https://`, envbuilder will assume SSH authentication. You have the following options: -1. Public/Private key authentication: set `ENVBUILDER_GIT_SSH_KEY_PATH` to the path of an +1. Public/Private key authentication: set `ENVBUILDER_GIT_SSH_PRIVATE_KEY_PATH` to the path of an SSH private key mounted inside the container. Envbuilder will use this SSH key to authenticate. Example: @@ -185,7 +185,7 @@ envbuilder will assume SSH authentication. You have the following options: -v /tmp/envbuilder:/workspaces \ -e ENVBUILDER_GIT_URL=git@example.com:path/to/private/repo.git \ -e ENVBUILDER_INIT_SCRIPT=bash \ - -e ENVBUILDER_GIT_SSH_KEY_PATH=/.ssh/id_rsa \ + -e ENVBUILDER_GIT_SSH_PRIVATE_KEY_PATH=/.ssh/id_rsa \ -v /home/user/id_rsa:/.ssh/id_rsa \ ghcr.io/coder/envbuilder ``` From 400275804fbe7469127935f9ba5f72a863aaf632 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Mon, 12 Aug 2024 12:52:56 +0300 Subject: [PATCH 32/81] fix: prevent git progress writer race reading stageNumber (#309) (cherry picked from commit 7c486bb5e871ab05459a486f398cba85b95f76da) --- envbuilder.go | 20 ++++++++++++++++---- git/git.go | 11 ++++++++--- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 7a61159e..9f25481e 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -116,7 +116,8 @@ func Run(ctx context.Context, opts options.Options) error { newColor(color.FgCyan).Sprintf(cloneOpts.Path), ) - w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNumber, line) }) + stageNum := stageNumber + w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNum, line) }) defer w.Close() cloneOpts.Progress = w @@ -132,6 +133,8 @@ func Run(ctx context.Context, opts options.Options) error { opts.Logger(log.LevelError, "Falling back to the default image...") } + _ = w.Close() + // Always clone the repo in remote repo build mode into a location that // we control that isn't affected by the users changes. if opts.RemoteRepoBuildMode { @@ -146,7 +149,8 @@ func Run(ctx context.Context, opts options.Options) error { newColor(color.FgCyan).Sprintf(cloneOpts.Path), ) - w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNumber, line) }) + stageNum := stageNumber + w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNum, line) }) defer w.Close() cloneOpts.Progress = w @@ -158,6 +162,8 @@ func Run(ctx context.Context, opts options.Options) error { opts.Logger(log.LevelError, "Failed to clone repository for remote repo mode: %s", fallbackErr.Error()) opts.Logger(log.LevelError, "Falling back to the default image...") } + + _ = w.Close() } } @@ -893,7 +899,8 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) newColor(color.FgCyan).Sprintf(cloneOpts.Path), ) - w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNumber, line) }) + stageNum := stageNumber + w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNum, line) }) defer w.Close() cloneOpts.Progress = w @@ -908,6 +915,8 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) opts.Logger(log.LevelError, "Failed to clone repository: %s", fallbackErr.Error()) opts.Logger(log.LevelError, "Falling back to the default image...") } + + _ = w.Close() } else { cloneOpts, err := git.CloneOptionsFromOptions(opts) if err != nil { @@ -920,7 +929,8 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) newColor(color.FgCyan).Sprintf(cloneOpts.Path), ) - w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNumber, line) }) + stageNum := stageNumber + w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNum, line) }) defer w.Close() cloneOpts.Progress = w @@ -932,6 +942,8 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) opts.Logger(log.LevelError, "Failed to clone repository for remote repo mode: %s", fallbackErr.Error()) opts.Logger(log.LevelError, "Falling back to the default image...") } + + _ = w.Close() } } diff --git a/git/git.go b/git/git.go index d6c1371c..1404f089 100644 --- a/git/git.go +++ b/git/git.go @@ -317,12 +317,14 @@ func CloneOptionsFromOptions(options options.Options) (CloneRepoOptions, error) type progressWriter struct { io.WriteCloser - r io.ReadCloser + r io.ReadCloser + done chan struct{} } func (w *progressWriter) Close() error { - err := w.r.Close() - err2 := w.WriteCloser.Close() + err := w.WriteCloser.Close() + <-w.done + err2 := w.r.Close() if err != nil { return err } @@ -331,7 +333,9 @@ func (w *progressWriter) Close() error { func ProgressWriter(write func(line string)) io.WriteCloser { reader, writer := io.Pipe() + done := make(chan struct{}) go func() { + defer close(done) data := make([]byte, 4096) for { read, err := reader.Read(data) @@ -351,5 +355,6 @@ func ProgressWriter(write func(line string)) io.WriteCloser { return &progressWriter{ WriteCloser: writer, r: reader, + done: done, } } From 59f0faedc9dcf27eabafa142f8c9b80512e461bb Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 12 Aug 2024 15:06:48 +0100 Subject: [PATCH 33/81] fix(log): increase coder rpcConnectTimeout to 30s (#313) (cherry picked from commit 1490e8a38a897c3cc5ce46e1b53d6fef3d5b52c2) --- log/coder.go | 5 +++- log/coder_internal_test.go | 49 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/log/coder.go b/log/coder.go index 38e9373e..d8b4fe0d 100644 --- a/log/coder.go +++ b/log/coder.go @@ -19,7 +19,10 @@ import ( ) var ( - rpcConnectTimeout = 10 * time.Second + // We set a relatively high connection timeout for the initial connection. + // There is an unfortunate race between the envbuilder container starting and the + // associated provisioner job completing. + rpcConnectTimeout = 30 * time.Second logSendGracePeriod = 10 * time.Second minAgentAPIV2 = "v2.9" ) diff --git a/log/coder_internal_test.go b/log/coder_internal_test.go index 22b6f249..4895150e 100644 --- a/log/coder_internal_test.go +++ b/log/coder_internal_test.go @@ -170,6 +170,55 @@ func TestCoder(t *testing.T) { require.ErrorIs(t, err, context.DeadlineExceeded) <-handlerDone }) + + // In this test, we validate that a 401 error on the initial connect + // results in a retry. When envbuilder initially attempts to connect + // using the Coder agent token, the workspace build may not yet have + // completed. + t.Run("V2Retry", func(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + token := uuid.NewString() + done := make(chan struct{}) + handlerSend := make(chan int) + handler := func(w http.ResponseWriter, r *http.Request) { + t.Logf("test handler: %s", r.URL.Path) + if r.URL.Path == "/api/v2/buildinfo" { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"version": "v2.9.0"}`)) + return + } + code := <-handlerSend + t.Logf("test handler response: %d", code) + w.WriteHeader(code) + } + srv := httptest.NewServer(http.HandlerFunc(handler)) + defer srv.Close() + + u, err := url.Parse(srv.URL) + require.NoError(t, err) + var connectError error + go func() { + defer close(handlerSend) + defer close(done) + _, _, connectError = Coder(ctx, u, token) + }() + + // Initial: unauthorized + handlerSend <- http.StatusUnauthorized + // 2nd try: still unauthorized + handlerSend <- http.StatusUnauthorized + // 3rd try: authorized + handlerSend <- http.StatusOK + + cancel() + + <-done + require.ErrorContains(t, connectError, "failed to WebSocket dial") + require.ErrorIs(t, connectError, context.Canceled) + }) } type fakeLogDest struct { From e6283db826e8665a3e9a5c7318c3030ceeccddbe Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 14 Aug 2024 21:35:00 +0100 Subject: [PATCH 34/81] fix(envbuilder): RunCacheProbe: remove references to constants.MagicDir (#315) (cherry picked from commit be15d1ac1f7f61c98877a4567d6a596e4b77df6a) --- envbuilder.go | 6 +- integration/integration_test.go | 306 ++++++++++++++++++++------------ testutil/gittest/gittest.go | 30 ++++ 3 files changed, 225 insertions(+), 117 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 9f25481e..35268bae 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -948,7 +948,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) } defaultBuildParams := func() (*devcontainer.Compiled, error) { - dockerfile := filepath.Join(constants.MagicDir, "Dockerfile") + dockerfile := filepath.Join(buildTimeWorkspaceFolder, "Dockerfile") file, err := opts.Filesystem.OpenFile(dockerfile, os.O_CREATE|os.O_WRONLY, 0o644) if err != nil { return nil, err @@ -970,7 +970,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) return &devcontainer.Compiled{ DockerfilePath: dockerfile, DockerfileContent: content, - BuildContext: constants.MagicDir, + BuildContext: buildTimeWorkspaceFolder, }, nil } @@ -1010,7 +1010,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) opts.Logger(log.LevelInfo, "No Dockerfile or image specified; falling back to the default image...") fallbackDockerfile = defaultParams.DockerfilePath } - buildParams, err = devContainer.Compile(opts.Filesystem, devcontainerDir, constants.MagicDir, fallbackDockerfile, opts.WorkspaceFolder, false, os.LookupEnv) + buildParams, err = devContainer.Compile(opts.Filesystem, devcontainerDir, buildTimeWorkspaceFolder, fallbackDockerfile, opts.WorkspaceFolder, false, os.LookupEnv) if err != nil { return nil, fmt.Errorf("compile devcontainer.json: %w", err) } diff --git a/integration/integration_test.go b/integration/integration_test.go index 297e86a1..af051473 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -38,7 +38,6 @@ import ( "github.com/docker/docker/api/types/volume" "github.com/docker/docker/client" "github.com/docker/docker/pkg/stdcopy" - "github.com/go-git/go-billy/v5/memfs" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/registry" @@ -68,8 +67,8 @@ func TestInitScriptInitCommand(t *testing.T) { w.WriteHeader(http.StatusOK) })) - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ // Let's say /bin/sh is not available and we can only use /bin/ash "Dockerfile": fmt.Sprintf("FROM %s\nRUN unlink /bin/sh", testImageAlpine), }, @@ -113,7 +112,7 @@ RUN mkdir -p /myapp/somedir \ && touch /myapp/somedir/somefile \ && chown 123:123 /myapp/somedir \ && chown 321:321 /myapp/somedir/somefile - + FROM %s COPY --from=builder /myapp /myapp RUN printf "%%s\n" \ @@ -127,8 +126,8 @@ RUN printf "%%s\n" \ /myapp/somedir/somefile \ > /tmp/got \ && diff -u /tmp/got /tmp/expected`, testImageAlpine, testImageAlpine) - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ "Dockerfile": dockerFile, }, }) @@ -158,8 +157,8 @@ RUN mkdir -p /myapp/somedir \ /myapp/somedir/somefile \ > /tmp/got \ && diff -u /tmp/got /tmp/expected`, testImageAlpine) - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ "Dockerfile": dockerFile, }, }) @@ -176,8 +175,8 @@ func TestForceSafe(t *testing.T) { t.Run("Safe", func(t *testing.T) { t.Parallel() - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ "Dockerfile": "FROM " + testImageAlpine, }, }) @@ -192,8 +191,8 @@ func TestForceSafe(t *testing.T) { // Careful with this one! t.Run("Unsafe", func(t *testing.T) { t.Parallel() - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ "Dockerfile": "FROM " + testImageAlpine, }, }) @@ -209,12 +208,12 @@ func TestForceSafe(t *testing.T) { func TestFailsGitAuth(t *testing.T) { t.Parallel() - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ "Dockerfile": "FROM " + testImageAlpine, }, - username: "kyle", - password: "testing", + Username: "kyle", + Password: "testing", }) _, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), @@ -224,12 +223,12 @@ func TestFailsGitAuth(t *testing.T) { func TestSucceedsGitAuth(t *testing.T) { t.Parallel() - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ "Dockerfile": "FROM " + testImageAlpine, }, - username: "kyle", - password: "testing", + Username: "kyle", + Password: "testing", }) ctr, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), @@ -244,12 +243,12 @@ func TestSucceedsGitAuth(t *testing.T) { func TestSucceedsGitAuthInURL(t *testing.T) { t.Parallel() - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ "Dockerfile": "FROM " + testImageAlpine, }, - username: "kyle", - password: "testing", + Username: "kyle", + Password: "testing", }) u, err := url.Parse(srv.URL) @@ -309,8 +308,8 @@ func TestBuildFromDevcontainerWithFeatures(t *testing.T) { require.NoError(t, err) // Ensures that a Git repository with a devcontainer.json is cloned and built. - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ ".devcontainer/devcontainer.json": `{ "name": "Test", "build": { @@ -350,8 +349,8 @@ func TestBuildFromDevcontainerWithFeatures(t *testing.T) { func TestBuildFromDockerfile(t *testing.T) { // Ensures that a Git repository with a Dockerfile is cloned and built. - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ "Dockerfile": "FROM " + testImageAlpine, }, }) @@ -372,8 +371,8 @@ func TestBuildFromDockerfile(t *testing.T) { func TestBuildPrintBuildOutput(t *testing.T) { // Ensures that a Git repository with a Dockerfile is cloned and built. - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ "Dockerfile": "FROM " + testImageAlpine + "\nRUN echo hello", }, }) @@ -400,8 +399,8 @@ func TestBuildPrintBuildOutput(t *testing.T) { func TestBuildIgnoreVarRunSecrets(t *testing.T) { // Ensures that a Git repository with a Dockerfile is cloned and built. - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ "Dockerfile": "FROM " + testImageAlpine, }, }) @@ -441,8 +440,8 @@ func TestBuildIgnoreVarRunSecrets(t *testing.T) { func TestBuildWithSetupScript(t *testing.T) { // Ensures that a Git repository with a Dockerfile is cloned and built. - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ "Dockerfile": "FROM " + testImageAlpine, }, }) @@ -461,8 +460,8 @@ func TestBuildFromDevcontainerInCustomPath(t *testing.T) { t.Parallel() // Ensures that a Git repository with a devcontainer.json is cloned and built. - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ ".devcontainer/custom/devcontainer.json": `{ "name": "Test", "build": { @@ -486,8 +485,8 @@ func TestBuildFromDevcontainerInSubfolder(t *testing.T) { t.Parallel() // Ensures that a Git repository with a devcontainer.json is cloned and built. - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ ".devcontainer/subfolder/devcontainer.json": `{ "name": "Test", "build": { @@ -510,8 +509,8 @@ func TestBuildFromDevcontainerInRoot(t *testing.T) { t.Parallel() // Ensures that a Git repository with a devcontainer.json is cloned and built. - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ "devcontainer.json": `{ "name": "Test", "build": { @@ -531,11 +530,11 @@ func TestBuildFromDevcontainerInRoot(t *testing.T) { } func TestBuildCustomCertificates(t *testing.T) { - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ "Dockerfile": "FROM " + testImageAlpine, }, - tls: true, + TLS: true, }) ctr, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), @@ -553,8 +552,8 @@ func TestBuildCustomCertificates(t *testing.T) { func TestBuildStopStartCached(t *testing.T) { // Ensures that a Git repository with a Dockerfile is cloned and built. - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ "Dockerfile": "FROM " + testImageAlpine, }, }) @@ -601,8 +600,8 @@ func TestBuildFailsFallback(t *testing.T) { t.Run("BadDockerfile", func(t *testing.T) { t.Parallel() // Ensures that a Git repository with a Dockerfile is cloned and built. - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ "Dockerfile": "bad syntax", }, }) @@ -616,8 +615,8 @@ func TestBuildFailsFallback(t *testing.T) { t.Run("FailsBuild", func(t *testing.T) { t.Parallel() // Ensures that a Git repository with a Dockerfile is cloned and built. - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ "Dockerfile": `FROM ` + testImageAlpine + ` RUN exit 1`, }, @@ -631,8 +630,8 @@ RUN exit 1`, t.Run("BadDevcontainer", func(t *testing.T) { t.Parallel() // Ensures that a Git repository with a Dockerfile is cloned and built. - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ ".devcontainer/devcontainer.json": "not json", }, }) @@ -643,8 +642,8 @@ RUN exit 1`, }) t.Run("NoImageOrDockerfile", func(t *testing.T) { t.Parallel() - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ ".devcontainer/devcontainer.json": "{}", }, }) @@ -661,8 +660,8 @@ RUN exit 1`, func TestExitBuildOnFailure(t *testing.T) { t.Parallel() - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ "Dockerfile": "bad syntax", }, }) @@ -680,8 +679,8 @@ func TestContainerEnv(t *testing.T) { t.Parallel() // Ensures that a Git repository with a devcontainer.json is cloned and built. - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ ".devcontainer/devcontainer.json": `{ "name": "Test", "build": { @@ -722,8 +721,8 @@ func TestUnsetOptionsEnv(t *testing.T) { t.Parallel() // Ensures that a Git repository with a devcontainer.json is cloned and built. - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ ".devcontainer/devcontainer.json": `{ "name": "Test", "build": { @@ -762,8 +761,8 @@ func TestLifecycleScripts(t *testing.T) { t.Parallel() // Ensures that a Git repository with a devcontainer.json is cloned and built. - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ ".devcontainer/devcontainer.json": `{ "name": "Test", "build": { @@ -798,8 +797,8 @@ func TestPostStartScript(t *testing.T) { t.Parallel() // Ensures that a Git repository with a devcontainer.json is cloned and built. - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ ".devcontainer/devcontainer.json": `{ "name": "Test", "build": { @@ -848,8 +847,8 @@ func TestPrivateRegistry(t *testing.T) { }) // Ensures that a Git repository with a Dockerfile is cloned and built. - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ "Dockerfile": "FROM " + image, }, }) @@ -867,8 +866,8 @@ func TestPrivateRegistry(t *testing.T) { }) // Ensures that a Git repository with a Dockerfile is cloned and built. - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ "Dockerfile": "FROM " + image, }, }) @@ -899,8 +898,8 @@ func TestPrivateRegistry(t *testing.T) { }) // Ensures that a Git repository with a Dockerfile is cloned and built. - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ "Dockerfile": "FROM " + image, }, }) @@ -1042,8 +1041,8 @@ COPY %s .`, testImageAlpine, inclFile) tc := tc t.Run(tc.name, func(t *testing.T) { - srv := createGitServer(t, gitServerOptions{ - files: tc.files, + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: tc.files, }) _, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), @@ -1066,8 +1065,8 @@ func TestPushImage(t *testing.T) { t.Run("CacheWithoutPush", func(t *testing.T) { t.Parallel() - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ ".devcontainer/Dockerfile": fmt.Sprintf(`FROM %s USER root ARG WORKDIR=/ @@ -1127,8 +1126,8 @@ RUN date --utc > /root/date.txt`, testImageAlpine), t.Run("CacheAndPush", func(t *testing.T) { t.Parallel() - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ ".devcontainer/Dockerfile": fmt.Sprintf(`FROM %s USER root ARG WORKDIR=/ @@ -1243,11 +1242,117 @@ RUN date --utc > /root/date.txt`, testImageAlpine), require.NotEmpty(t, strings.TrimSpace(out)) }) + t.Run("CacheAndPushDevcontainerOnly", func(t *testing.T) { + t.Parallel() + + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ + ".devcontainer/devcontainer.json": fmt.Sprintf(`{"image": %q}`, testImageAlpine), + }, + }) + + // Given: an empty registry + testReg := setupInMemoryRegistry(t, setupInMemoryRegistryOpts{}) + testRepo := testReg + "/test" + ref, err := name.ParseReference(testRepo + ":latest") + require.NoError(t, err) + _, err = remote.Image(ref) + require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") + + // When: we run envbuilder with GET_CACHED_IMAGE + _, err = runEnvbuilder(t, runOpts{env: []string{ + envbuilderEnv("GIT_URL", srv.URL), + envbuilderEnv("CACHE_REPO", testRepo), + envbuilderEnv("GET_CACHED_IMAGE", "1"), + }}) + require.ErrorContains(t, err, "error probing build cache: uncached COPY command") + // Then: it should fail to build the image and nothing should be pushed + _, err = remote.Image(ref) + require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") + + // When: we run envbuilder with PUSH_IMAGE set + _, err = runEnvbuilder(t, runOpts{env: []string{ + envbuilderEnv("GIT_URL", srv.URL), + envbuilderEnv("CACHE_REPO", testRepo), + envbuilderEnv("PUSH_IMAGE", "1"), + }}) + require.NoError(t, err) + + // Then: the image should be pushed + img, err := remote.Image(ref) + require.NoError(t, err, "expected image to be present after build + push") + + // Then: the image should have its directives replaced with those required + // to run envbuilder automatically + configFile, err := img.ConfigFile() + require.NoError(t, err, "expected image to return a config file") + + assert.Equal(t, "root", configFile.Config.User, "user must be root") + assert.Equal(t, "/", configFile.Config.WorkingDir, "workdir must be /") + if assert.Len(t, configFile.Config.Entrypoint, 1) { + assert.Equal(t, "/.envbuilder/bin/envbuilder", configFile.Config.Entrypoint[0], "incorrect entrypoint") + } + + // Then: re-running envbuilder with GET_CACHED_IMAGE should succeed + ctrID, err := runEnvbuilder(t, runOpts{env: []string{ + envbuilderEnv("GIT_URL", srv.URL), + envbuilderEnv("CACHE_REPO", testRepo), + envbuilderEnv("GET_CACHED_IMAGE", "1"), + }}) + require.NoError(t, err) + + // Then: the cached image ref should be emitted in the container logs + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + require.NoError(t, err) + defer cli.Close() + logs, err := cli.ContainerLogs(ctx, ctrID, container.LogsOptions{ + ShowStdout: true, + ShowStderr: true, + }) + require.NoError(t, err) + defer logs.Close() + logBytes, err := io.ReadAll(logs) + require.NoError(t, err) + require.Regexp(t, `ENVBUILDER_CACHED_IMAGE=(\S+)`, string(logBytes)) + + // When: we pull the image we just built + rc, err := cli.ImagePull(ctx, ref.String(), image.PullOptions{}) + require.NoError(t, err) + t.Cleanup(func() { _ = rc.Close() }) + _, err = io.ReadAll(rc) + require.NoError(t, err) + + // When: we run the image we just built + ctr, err := cli.ContainerCreate(ctx, &container.Config{ + Image: ref.String(), + Entrypoint: []string{"sleep", "infinity"}, + Labels: map[string]string{ + testContainerLabel: "true", + }, + }, nil, nil, nil, "") + require.NoError(t, err) + t.Cleanup(func() { + _ = cli.ContainerRemove(ctx, ctr.ID, container.RemoveOptions{ + RemoveVolumes: true, + Force: true, + }) + }) + err = cli.ContainerStart(ctx, ctr.ID, container.StartOptions{}) + require.NoError(t, err) + + // Then: the envbuilder binary exists in the image! + out := execContainer(t, ctr.ID, "/.envbuilder/bin/envbuilder --help") + require.Regexp(t, `(?s)^USAGE:\s+envbuilder`, strings.TrimSpace(out)) + require.NotEmpty(t, strings.TrimSpace(out)) + }) + t.Run("CacheAndPushAuth", func(t *testing.T) { t.Parallel() - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ ".devcontainer/Dockerfile": fmt.Sprintf(`FROM %s USER root ARG WORKDIR=/ @@ -1323,8 +1428,8 @@ RUN date --utc > /root/date.txt`, testImageAlpine), t.Run("CacheAndPushAuthFail", func(t *testing.T) { t.Parallel() - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ ".devcontainer/Dockerfile": fmt.Sprintf(`FROM %s USER root ARG WORKDIR=/ @@ -1390,8 +1495,8 @@ RUN date --utc > /root/date.txt`, testImageAlpine), t.Skip("TODO: https://github.com/coder/envbuilder/issues/230") t.Parallel() - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ "Dockerfile": fmt.Sprintf(`FROM %s AS a RUN date --utc > /root/date.txt FROM %s as b @@ -1448,8 +1553,8 @@ COPY --from=a /root/date.txt /date.txt`, testImageAlpine, testImageAlpine), t.Run("PushImageRequiresCache", func(t *testing.T) { t.Parallel() - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ ".devcontainer/Dockerfile": fmt.Sprintf(`FROM %s USER root ARG WORKDIR=/ @@ -1480,8 +1585,8 @@ RUN date --utc > /root/date.txt`, testImageAlpine), t.Run("PushErr", func(t *testing.T) { t.Parallel() - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ ".devcontainer/Dockerfile": fmt.Sprintf(`FROM %s USER root ARG WORKDIR=/ @@ -1518,8 +1623,8 @@ func TestChownHomedir(t *testing.T) { t.Parallel() // Ensures that a Git repository with a devcontainer.json is cloned and built. - srv := createGitServer(t, gitServerOptions{ - files: map[string]string{ + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ ".devcontainer/devcontainer.json": `{ "name": "Test", "build": { @@ -1591,33 +1696,6 @@ func TestMain(m *testing.M) { m.Run() } -type gitServerOptions struct { - files map[string]string - username string - password string - authMW func(http.Handler) http.Handler - tls bool -} - -// createGitServer creates a git repository with an in-memory filesystem -// and serves it over HTTP using a httptest.Server. -func createGitServer(t *testing.T, opts gitServerOptions) *httptest.Server { - t.Helper() - if opts.authMW == nil { - opts.authMW = mwtest.BasicAuthMW(opts.username, opts.password) - } - commits := make([]gittest.CommitFunc, 0) - for path, content := range opts.files { - commits = append(commits, gittest.Commit(t, path, content, "my test commit")) - } - fs := memfs.New() - _ = gittest.NewRepo(t, fs, commits...) - if opts.tls { - return httptest.NewTLSServer(opts.authMW(gittest.NewServer(fs))) - } - return httptest.NewServer(opts.authMW(gittest.NewServer(fs))) -} - func checkTestRegistry() { resp, err := http.Get("http://localhost:5000/v2/_catalog") if err != nil { diff --git a/testutil/gittest/gittest.go b/testutil/gittest/gittest.go index ffa9bd01..f3d5f1d3 100644 --- a/testutil/gittest/gittest.go +++ b/testutil/gittest/gittest.go @@ -6,6 +6,7 @@ import ( "log" "net" "net/http" + "net/http/httptest" "os" "os/exec" "sync" @@ -14,8 +15,10 @@ import ( gossh "golang.org/x/crypto/ssh" + "github.com/coder/envbuilder/testutil/mwtest" "github.com/gliderlabs/ssh" "github.com/go-git/go-billy/v5" + "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/cache" @@ -28,6 +31,33 @@ import ( "github.com/stretchr/testify/require" ) +type Options struct { + Files map[string]string + Username string + Password string + AuthMW func(http.Handler) http.Handler + TLS bool +} + +// CreateGitServer creates a git repository with an in-memory filesystem +// and serves it over HTTP using a httptest.Server. +func CreateGitServer(t *testing.T, opts Options) *httptest.Server { + t.Helper() + if opts.AuthMW == nil { + opts.AuthMW = mwtest.BasicAuthMW(opts.Username, opts.Password) + } + commits := make([]CommitFunc, 0) + for path, content := range opts.Files { + commits = append(commits, Commit(t, path, content, "my test commit")) + } + fs := memfs.New() + _ = NewRepo(t, fs, commits...) + if opts.TLS { + return httptest.NewTLSServer(opts.AuthMW(NewServer(fs))) + } + return httptest.NewServer(opts.AuthMW(NewServer(fs))) +} + // NewServer returns a http.Handler that serves a git repository. // It's expected that the repository is already initialized by the caller. func NewServer(fs billy.Filesystem) http.Handler { From cd63d0b71a40603d4061f65babd1951d890e05f3 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 15 Aug 2024 15:24:12 +0100 Subject: [PATCH 35/81] chore(deps): update kaniko to address Docker CVE (#318) (cherry picked from commit 12940b5dfe79275c4e3b00928999428febfedf6a) --- envbuilder.go | 2 +- go.mod | 21 ++++++++++++--------- go.sum | 42 ++++++++++++++++++++++++------------------ 3 files changed, 37 insertions(+), 28 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 35268bae..9360fe22 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -37,7 +37,7 @@ import ( "github.com/coder/envbuilder/devcontainer" "github.com/coder/envbuilder/internal/ebutil" "github.com/coder/envbuilder/log" - "github.com/containerd/containerd/platforms" + "github.com/containerd/platforms" "github.com/distribution/distribution/v3/configuration" "github.com/distribution/distribution/v3/registry/handlers" _ "github.com/distribution/distribution/v3/registry/storage/driver/filesystem" diff --git a/go.mod b/go.mod index 972a1f29..d9e320d1 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22.4 // There are a few options we need added to Kaniko! // See: https://github.com/GoogleContainerTools/kaniko/compare/main...coder:kaniko:main -replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240807142221-ffc5e60fca41 +replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240815135021-647365bde8a7 // Required to import codersdk due to gvisor dependency. replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20240702054557-aa558fbe5374 @@ -17,10 +17,10 @@ require ( github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352 github.com/coder/retry v1.5.1 github.com/coder/serpent v0.7.0 - github.com/containerd/containerd v1.7.15 + github.com/containerd/platforms v0.2.1 github.com/distribution/distribution/v3 v3.0.0-alpha.1 github.com/docker/cli v27.1.1+incompatible - github.com/docker/docker v26.1.0+incompatible + github.com/docker/docker v26.1.5+incompatible github.com/fatih/color v1.17.0 github.com/gliderlabs/ssh v0.3.7 github.com/go-git/go-billy/v5 v5.5.0 @@ -32,7 +32,7 @@ require ( github.com/mattn/go-isatty v0.0.20 github.com/moby/buildkit v0.13.1 github.com/otiai10/copy v1.14.0 - github.com/prometheus/procfs v0.15.0 + github.com/prometheus/procfs v0.15.1 github.com/sirupsen/logrus v1.9.3 github.com/skeema/knownhosts v1.3.0 github.com/stretchr/testify v1.9.0 @@ -68,8 +68,8 @@ require ( github.com/DataDog/go-tuf v1.0.2-0.5.2 // indirect github.com/DataDog/gostackparse v0.7.0 // indirect github.com/DataDog/sketches-go v1.4.2 // indirect - github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/Microsoft/hcsshim v0.11.4 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/Microsoft/hcsshim v0.11.7 // indirect github.com/ProtonMail/go-crypto v1.1.0-alpha.2 // indirect github.com/agext/levenshtein v1.2.3 // indirect github.com/akutz/memconn v0.1.0 // indirect @@ -106,11 +106,14 @@ require ( github.com/coder/terraform-provider-coder v0.23.0 // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/containerd/cgroups/v3 v3.0.2 // indirect + github.com/containerd/containerd v1.7.19 // indirect + github.com/containerd/containerd/api v1.7.19 // indirect github.com/containerd/continuity v0.4.3 // indirect + github.com/containerd/errdefs v0.1.0 // indirect github.com/containerd/fifo v1.1.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect - github.com/containerd/ttrpc v1.2.3 // indirect + github.com/containerd/ttrpc v1.2.5 // indirect github.com/containerd/typeurl/v2 v2.1.1 // indirect github.com/coreos/go-iptables v0.6.0 // indirect github.com/coreos/go-oidc/v3 v3.10.0 // indirect @@ -122,7 +125,7 @@ require ( github.com/dimchansky/utfbom v1.1.1 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/docker/docker-credential-helpers v0.8.1 // indirect + github.com/docker/docker-credential-helpers v0.8.2 // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect github.com/docker/go-metrics v0.0.1 // indirect @@ -211,7 +214,7 @@ require ( github.com/muesli/termenv v0.15.2 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect - github.com/opencontainers/runtime-spec v1.1.0 // indirect + github.com/opencontainers/runtime-spec v1.2.0 // indirect github.com/opencontainers/selinux v1.11.0 // indirect github.com/outcaste-io/ristretto v0.2.3 // indirect github.com/philhofer/fwd v1.1.2 // indirect diff --git a/go.sum b/go.sum index cc9e8546..39d04748 100644 --- a/go.sum +++ b/go.sum @@ -66,10 +66,10 @@ github.com/DataDog/sketches-go v1.4.2 h1:gppNudE9d19cQ98RYABOetxIhpTCl4m7CnbRZjv github.com/DataDog/sketches-go v1.4.2/go.mod h1:xJIXldczJyyjnbDop7ZZcLxJdV3+7Kra7H1KMgpgkLk= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= -github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Microsoft/hcsshim v0.11.7 h1:vl/nj3Bar/CvJSYo7gIQPyRWc9f3c6IeSNavBTSZNZQ= +github.com/Microsoft/hcsshim v0.11.7/go.mod h1:MV8xMfmECjl5HdO7U/3/hFVnkmSBjAjmA09d4bExKcU= github.com/ProtonMail/go-crypto v1.1.0-alpha.2 h1:bkyFVUP+ROOARdgCiJzNQo2V2kiB97LyUpzH9P6Hrlg= github.com/ProtonMail/go-crypto v1.1.0-alpha.2/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= @@ -171,8 +171,8 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352 h1:L/EjCuZxs5tOcqqCaASj/nu65TRYEFcTt8qRQfHZXX0= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352/go.mod h1:P1KoQSgnKEAG6Mnd3YlGzAophty+yKA9VV48LpfNRvo= -github.com/coder/kaniko v0.0.0-20240807142221-ffc5e60fca41 h1:1Ye7AcLnuT5IDv6il7Fxo+aqpzlWfedkpraCCwx8Lyo= -github.com/coder/kaniko v0.0.0-20240807142221-ffc5e60fca41/go.mod h1:YMK7BlxerzLlMwihGxNWUaFoN9LXCij4P+w/8/fNlcM= +github.com/coder/kaniko v0.0.0-20240815135021-647365bde8a7 h1:i5CTDhAUlZMXr4PdMz5RxTBiG0xltxj1npbEi1Ggzek= +github.com/coder/kaniko v0.0.0-20240815135021-647365bde8a7/go.mod h1:xlfIeo8SYBw3zwKb73wzz4Q5Q1wtnJy8ofYqGDAl/NA= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 h1:3A0ES21Ke+FxEM8CXx9n47SZOKOpgSE1bbJzlE4qPVs= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0/go.mod h1:5UuS2Ts+nTToAMeOjNlnHFkPahrtDkmpydBen/3wgZc= github.com/coder/quartz v0.1.0 h1:cLL+0g5l7xTf6ordRnUMMiZtRE8Sq5LxpghS63vEXrQ= @@ -189,18 +189,24 @@ github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaD github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/cgroups/v3 v3.0.2 h1:f5WFqIVSgo5IZmtTT3qVBo6TzI1ON6sycSBKkymb9L0= github.com/containerd/cgroups/v3 v3.0.2/go.mod h1:JUgITrzdFqp42uI2ryGA+ge0ap/nxzYgkGmIcetmErE= -github.com/containerd/containerd v1.7.15 h1:afEHXdil9iAm03BmhjzKyXnnEBtjaLJefdU7DV0IFes= -github.com/containerd/containerd v1.7.15/go.mod h1:ISzRRTMF8EXNpJlTzyr2XMhN+j9K302C21/+cr3kUnY= +github.com/containerd/containerd v1.7.19 h1:/xQ4XRJ0tamDkdzrrBAUy/LE5nCcxFKdBm4EcPrSMEE= +github.com/containerd/containerd v1.7.19/go.mod h1:h4FtNYUUMB4Phr6v+xG89RYKj9XccvbNSCKjdufCrkc= +github.com/containerd/containerd/api v1.7.19 h1:VWbJL+8Ap4Ju2mx9c9qS1uFSB1OVYr5JJrW2yT5vFoA= +github.com/containerd/containerd/api v1.7.19/go.mod h1:fwGavl3LNwAV5ilJ0sbrABL44AQxmNjDRcwheXDb6Ig= github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8= github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= +github.com/containerd/errdefs v0.1.0 h1:m0wCRBiu1WJT/Fr+iOoQHMQS/eP5myQ8lCv4Dz5ZURM= +github.com/containerd/errdefs v0.1.0/go.mod h1:YgWiiHtLmSeBrvpw+UfPijzbLaB77mEG1WwJTDETIV0= github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY= github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= github.com/containerd/stargz-snapshotter/estargz v0.15.1 h1:eXJjw9RbkLFgioVaTG+G/ZW/0kEe2oEKCdS/ZxIyoCU= github.com/containerd/stargz-snapshotter/estargz v0.15.1/go.mod h1:gr2RNwukQ/S9Nv33Lt6UC7xEx58C+LHRdoqbEKjz1Kk= -github.com/containerd/ttrpc v1.2.3 h1:4jlhbXIGvijRtNC8F/5CpuJZ7yKOBFGFOOXg1bkISz0= -github.com/containerd/ttrpc v1.2.3/go.mod h1:ieWsXucbb8Mj9PH0rXCw1i8IunRbbAiDkpXkbfflWBM= +github.com/containerd/ttrpc v1.2.5 h1:IFckT1EFQoFBMG4c3sMdT8EP3/aKfumK1msY+Ze4oLU= +github.com/containerd/ttrpc v1.2.5/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= github.com/containerd/typeurl/v2 v2.1.1 h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9FqcttATPO/4= github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0= github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk= @@ -234,10 +240,10 @@ github.com/docker/cli v27.1.1+incompatible h1:goaZxOqs4QKxznZjjBWKONQci/MywhtRv2 github.com/docker/cli v27.1.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v26.1.0+incompatible h1:W1G9MPNbskA6VZWL7b3ZljTh0pXI68FpINx0GKaOdaM= -github.com/docker/docker v26.1.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker-credential-helpers v0.8.1 h1:j/eKUktUltBtMzKqmfLB0PAgqYyMHOp5vfsD1807oKo= -github.com/docker/docker-credential-helpers v0.8.1/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M= +github.com/docker/docker v26.1.5+incompatible h1:NEAxTwEjxV6VbBMBoGG3zPqbiJosIApZjxlbrG9q3/g= +github.com/docker/docker v26.1.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo= +github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= @@ -601,8 +607,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= -github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= +github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= @@ -652,8 +658,8 @@ github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5E github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.15.0 h1:A82kmvXJq2jTu5YUhSGNlYoxh85zLnKgPz4bMZgI5Ek= -github.com/prometheus/procfs v0.15.0/go.mod h1:Y0RJ/Y5g5wJpkTisOtqwDSo4HwhGmLB4VQSw2sQJLHk= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5 h1:EaDatTxkdHG+U3Bk4EUr+DZ7fOGwTfezUiUJMaIcaho= github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5/go.mod h1:fyalQWdtzDBECAQFBJuQe5bzQ02jGd5Qcbgb97Flm7U= github.com/redis/go-redis/extra/redisotel/v9 v9.0.5 h1:EfpWLLCyXw8PSM2/XNJLjI3Pb27yVE+gIAfeqp8LUCc= From f09201910e208a6f24dcbf9e8f3d9b0828b26368 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 Aug 2024 11:26:59 +0000 Subject: [PATCH 36/81] chore: bump golang.org/x/sync from 0.7.0 to 0.8.0 (#310) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Cian Johnston (cherry picked from commit 768208484316d9d36c1881cae5793663dce31693) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d9e320d1..3c053573 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( go.uber.org/mock v0.4.0 golang.org/x/crypto v0.25.0 golang.org/x/mod v0.18.0 - golang.org/x/sync v0.7.0 + golang.org/x/sync v0.8.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 ) diff --git a/go.sum b/go.sum index 39d04748..4eab8d72 100644 --- a/go.sum +++ b/go.sum @@ -884,8 +884,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= From f62389ccdca06a01494f52acc7f7f55cb8e50ac5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 Aug 2024 12:55:11 +0100 Subject: [PATCH 37/81] chore: bump github.com/docker/docker from 26.1.0+incompatible to 26.1.5+incompatible (#316) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Cian Johnston (cherry picked from commit df860f6e66086b6ba6b46d920c7882443e77d1e3) --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 3c053573..562f3e26 100644 --- a/go.mod +++ b/go.mod @@ -19,13 +19,13 @@ require ( github.com/coder/serpent v0.7.0 github.com/containerd/platforms v0.2.1 github.com/distribution/distribution/v3 v3.0.0-alpha.1 - github.com/docker/cli v27.1.1+incompatible + github.com/docker/cli v27.0.3+incompatible github.com/docker/docker v26.1.5+incompatible github.com/fatih/color v1.17.0 github.com/gliderlabs/ssh v0.3.7 github.com/go-git/go-billy/v5 v5.5.0 github.com/go-git/go-git/v5 v5.12.0 - github.com/google/go-containerregistry v0.20.2 + github.com/google/go-containerregistry v0.20.1 github.com/google/uuid v1.6.0 github.com/hashicorp/go-multierror v1.1.1 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 diff --git a/go.sum b/go.sum index 4eab8d72..571d2454 100644 --- a/go.sum +++ b/go.sum @@ -236,8 +236,8 @@ github.com/distribution/distribution/v3 v3.0.0-alpha.1 h1:jn7I1gvjOvmLztH1+1cLiU github.com/distribution/distribution/v3 v3.0.0-alpha.1/go.mod h1:LCp4JZp1ZalYg0W/TN05jarCQu+h4w7xc7ZfQF4Y/cY= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/cli v27.1.1+incompatible h1:goaZxOqs4QKxznZjjBWKONQci/MywhtRv2oNn0GkeZE= -github.com/docker/cli v27.1.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v27.0.3+incompatible h1:usGs0/BoBW8MWxGeEtqPMkzOY56jZ6kYlSN5BLDioCQ= +github.com/docker/cli v27.0.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v26.1.5+incompatible h1:NEAxTwEjxV6VbBMBoGG3zPqbiJosIApZjxlbrG9q3/g= @@ -386,8 +386,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-containerregistry v0.20.2 h1:B1wPJ1SN/S7pB+ZAimcciVD+r+yV/l/DSArMxlbwseo= -github.com/google/go-containerregistry v0.20.2/go.mod h1:z38EKdKh4h7IP2gSfUUqEvalZBqs6AoLeWfUy34nQC8= +github.com/google/go-containerregistry v0.20.1 h1:eTgx9QNYugV4DN5mz4U8hiAGTi1ybXn0TPi4Smd8du0= +github.com/google/go-containerregistry v0.20.1/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= From 23d086e675ed2b969b9b9b47b54e8869fd82ea3c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 Aug 2024 15:02:23 +0300 Subject: [PATCH 38/81] chore: bump golang.org/x/crypto from 0.25.0 to 0.26.0 (#311) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Cian Johnston (cherry picked from commit b2aaa3e9a44c816cadcd3f3be578921d2702cead) --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 562f3e26..1c2e25f3 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a go.uber.org/mock v0.4.0 - golang.org/x/crypto v0.25.0 + golang.org/x/crypto v0.26.0 golang.org/x/mod v0.18.0 golang.org/x/sync v0.8.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 @@ -275,9 +275,9 @@ require ( golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect golang.org/x/net v0.26.0 // indirect golang.org/x/oauth2 v0.20.0 // indirect - golang.org/x/sys v0.22.0 // indirect - golang.org/x/term v0.22.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/sys v0.23.0 // indirect + golang.org/x/term v0.23.0 // indirect + golang.org/x/text v0.17.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.22.0 // indirect golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect diff --git a/go.sum b/go.sum index 571d2454..2b463e4f 100644 --- a/go.sum +++ b/go.sum @@ -835,8 +835,8 @@ golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= -golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= @@ -924,15 +924,15 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.1-0.20230131160137-e7d7f63158de/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= -golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -942,8 +942,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= From 2966db0ecd80b79d5eb94ff661e57c0a93000ec0 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 27 Aug 2024 15:18:29 +0100 Subject: [PATCH 39/81] fix(log): properly set logrus level (#327) (cherry picked from commit 8b9ec59b09ad0b6d64ecb70a3ba69bf5b124b426) --- envbuilder.go | 16 +++++-- log.go | 28 ------------ log/logrus.go | 61 +++++++++++++++++++++++++ log/logrus_test.go | 110 +++++++++++++++++++++++++++++++++++++++++++++ log_test.go | 19 -------- 5 files changed, 183 insertions(+), 51 deletions(-) delete mode 100644 log.go create mode 100644 log/logrus.go create mode 100644 log/logrus_test.go delete mode 100644 log_test.go diff --git a/envbuilder.go b/envbuilder.go index 9360fe22..7f3c983a 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -278,9 +278,13 @@ func Run(ctx context.Context, opts options.Options) error { } } - HijackLogrus(func(entry *logrus.Entry) { + lvl := log.LevelInfo + if opts.Verbose { + lvl = log.LevelDebug + } + log.HijackLogrus(lvl, func(entry *logrus.Entry) { for _, line := range strings.Split(entry.Message, "\r") { - opts.Logger(log.LevelInfo, "#%d: %s", stageNumber, color.HiBlackString(line)) + opts.Logger(log.FromLogrus(entry.Level), "#%d: %s", stageNumber, color.HiBlackString(line)) } }) @@ -1050,9 +1054,13 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) return nil, fmt.Errorf("no Dockerfile or devcontainer.json found") } - HijackLogrus(func(entry *logrus.Entry) { + lvl := log.LevelInfo + if opts.Verbose { + lvl = log.LevelDebug + } + log.HijackLogrus(lvl, func(entry *logrus.Entry) { for _, line := range strings.Split(entry.Message, "\r") { - opts.Logger(log.LevelInfo, "#%d: %s", stageNumber, color.HiBlackString(line)) + opts.Logger(log.FromLogrus(entry.Level), "#%d: %s", stageNumber, color.HiBlackString(line)) } }) diff --git a/log.go b/log.go deleted file mode 100644 index ad476c1d..00000000 --- a/log.go +++ /dev/null @@ -1,28 +0,0 @@ -package envbuilder - -import ( - "io" - - "github.com/sirupsen/logrus" -) - -// HijackLogrus hijacks the logrus logger and calls the callback for each log entry. -// This is an abuse of logrus, the package that Kaniko uses, but it exposes -// no other way to obtain the log entries. -func HijackLogrus(callback func(entry *logrus.Entry)) { - logrus.StandardLogger().SetOutput(io.Discard) - logrus.StandardLogger().SetFormatter(&logrusFormatter{ - callback: callback, - empty: []byte{}, - }) -} - -type logrusFormatter struct { - callback func(entry *logrus.Entry) - empty []byte -} - -func (f *logrusFormatter) Format(entry *logrus.Entry) ([]byte, error) { - f.callback(entry) - return f.empty, nil -} diff --git a/log/logrus.go b/log/logrus.go new file mode 100644 index 00000000..3d70b114 --- /dev/null +++ b/log/logrus.go @@ -0,0 +1,61 @@ +package log + +import ( + "io" + + "github.com/sirupsen/logrus" +) + +// HijackLogrus hijacks the logrus logger and calls the callback for each log entry. +// This is an abuse of logrus, the package that Kaniko uses, but it exposes +// no other way to obtain the log entries. +func HijackLogrus(lvl Level, callback func(entry *logrus.Entry)) { + logrus.StandardLogger().SetOutput(io.Discard) + logrus.StandardLogger().SetLevel(ToLogrus(lvl)) + logrus.StandardLogger().SetFormatter(&logrusFormatter{ + callback: callback, + empty: []byte{}, + }) +} + +type logrusFormatter struct { + callback func(entry *logrus.Entry) + empty []byte +} + +func (f *logrusFormatter) Format(entry *logrus.Entry) ([]byte, error) { + f.callback(entry) + return f.empty, nil +} + +func ToLogrus(lvl Level) logrus.Level { + switch lvl { + case LevelTrace: + return logrus.TraceLevel + case LevelDebug: + return logrus.DebugLevel + case LevelInfo: + return logrus.InfoLevel + case LevelWarn: + return logrus.WarnLevel + case LevelError: + return logrus.ErrorLevel + default: + return logrus.InfoLevel + } +} + +func FromLogrus(lvl logrus.Level) Level { + switch lvl { + case logrus.TraceLevel: + return LevelTrace + case logrus.DebugLevel: + return LevelDebug + case logrus.InfoLevel: + return LevelInfo + case logrus.WarnLevel: + return LevelWarn + default: // Error, Fatal, Panic + return LevelError + } +} diff --git a/log/logrus_test.go b/log/logrus_test.go new file mode 100644 index 00000000..7b606696 --- /dev/null +++ b/log/logrus_test.go @@ -0,0 +1,110 @@ +package log_test + +import ( + "context" + "testing" + "time" + + "github.com/coder/envbuilder/log" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +func TestHijackLogrus_Info(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + t.Cleanup(cancel) + messages := make(chan *logrus.Entry) + + logf := func(entry *logrus.Entry) { + t.Logf("got msg level: %s msg: %q", entry.Level, entry.Message) + messages <- entry + } + + log.HijackLogrus(log.LevelInfo, logf) + + done := make(chan struct{}) + go func() { + defer close(done) + // The following should be filtered out. + logrus.Trace("Tracing!") + logrus.Debug("Debugging!") + // We should receive the below. + logrus.Info("Testing!") + logrus.Warn("Warning!") + logrus.Error("Error!") + }() + + require.Equal(t, "Testing!", rcvCtx(ctx, t, messages).Message) + require.Equal(t, "Warning!", rcvCtx(ctx, t, messages).Message) + require.Equal(t, "Error!", rcvCtx(ctx, t, messages).Message) + <-done +} + +func TestHijackLogrus_Debug(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + t.Cleanup(cancel) + messages := make(chan *logrus.Entry) + + logf := func(entry *logrus.Entry) { + t.Logf("got msg level: %s msg: %q", entry.Level, entry.Message) + messages <- entry + } + + log.HijackLogrus(log.LevelDebug, logf) + + done := make(chan struct{}) + go func() { + defer close(done) + // The following should be filtered out. + logrus.Trace("Tracing!") + // We should receive the below. + logrus.Debug("Debugging!") + logrus.Info("Testing!") + logrus.Warn("Warning!") + logrus.Error("Error!") + }() + + require.Equal(t, "Debugging!", rcvCtx(ctx, t, messages).Message) + require.Equal(t, "Testing!", rcvCtx(ctx, t, messages).Message) + require.Equal(t, "Warning!", rcvCtx(ctx, t, messages).Message) + require.Equal(t, "Error!", rcvCtx(ctx, t, messages).Message) + <-done +} + +func TestHijackLogrus_Error(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + t.Cleanup(cancel) + messages := make(chan *logrus.Entry) + + logf := func(entry *logrus.Entry) { + t.Logf("got msg level: %s msg: %q", entry.Level, entry.Message) + messages <- entry + } + + log.HijackLogrus(log.LevelError, logf) + + done := make(chan struct{}) + go func() { + defer close(done) + // The following should be filtered out. + logrus.Trace("Tracing!") + logrus.Debug("Debugging!") + logrus.Info("Testing!") + logrus.Warn("Warning!") + // We should receive the below. + logrus.Error("Error!") + }() + + require.Equal(t, "Error!", rcvCtx(ctx, t, messages).Message) + <-done +} + +func rcvCtx[T any](ctx context.Context, t *testing.T, ch <-chan T) (v T) { + t.Helper() + select { + case <-ctx.Done(): + t.Fatal("timeout") + case v = <-ch: + } + return v +} diff --git a/log_test.go b/log_test.go deleted file mode 100644 index 63d5e6cd..00000000 --- a/log_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package envbuilder_test - -import ( - "testing" - - "github.com/coder/envbuilder" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/require" -) - -func TestHijackLogrus(t *testing.T) { - messages := make(chan *logrus.Entry, 1) - envbuilder.HijackLogrus(func(entry *logrus.Entry) { - messages <- entry - }) - logrus.Infof("Testing!") - message := <-messages - require.Equal(t, "Testing!", message.Message) -} From 4ef99c099093eb6b61345b085c89821146125d02 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Fri, 30 Aug 2024 14:17:48 +0300 Subject: [PATCH 40/81] chore: update kaniko fork to fix multi-stage cache probing (#325) (cherry picked from commit c5efab518ea98b30ee8a005ce5d1cf157d160cf4) --- go.mod | 4 +- go.sum | 8 +- integration/integration_test.go | 167 +++++++++++++++++++++++++++++--- 3 files changed, 158 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 1c2e25f3..e1d01738 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22.4 // There are a few options we need added to Kaniko! // See: https://github.com/GoogleContainerTools/kaniko/compare/main...coder:kaniko:main -replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240815135021-647365bde8a7 +replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240830092517-0668f96c8816 // Required to import codersdk due to gvisor dependency. replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20240702054557-aa558fbe5374 @@ -178,7 +178,6 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/native v1.1.1-0.20230202152459-5c7d0dd6ab86 // indirect github.com/jsimonetti/rtnetlink v1.3.5 // indirect - github.com/karrick/godirwalk v1.16.1 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/kortschak/wol v0.0.0-20200729010619-da482cc4850a // indirect @@ -246,6 +245,7 @@ require ( github.com/tailscale/wireguard-go v0.0.0-20231121184858-cc193a0b3272 // indirect github.com/tcnksm/go-httpstat v0.2.0 // indirect github.com/tinylib/msgp v1.1.8 // indirect + github.com/twpayne/go-vfs/v5 v5.0.4 // indirect github.com/u-root/uio v0.0.0-20240209044354-b3d14b93376a // indirect github.com/valyala/fasthttp v1.55.0 // indirect github.com/vbatts/tar-split v0.11.5 // indirect diff --git a/go.sum b/go.sum index 2b463e4f..6101dcf6 100644 --- a/go.sum +++ b/go.sum @@ -171,8 +171,8 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352 h1:L/EjCuZxs5tOcqqCaASj/nu65TRYEFcTt8qRQfHZXX0= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352/go.mod h1:P1KoQSgnKEAG6Mnd3YlGzAophty+yKA9VV48LpfNRvo= -github.com/coder/kaniko v0.0.0-20240815135021-647365bde8a7 h1:i5CTDhAUlZMXr4PdMz5RxTBiG0xltxj1npbEi1Ggzek= -github.com/coder/kaniko v0.0.0-20240815135021-647365bde8a7/go.mod h1:xlfIeo8SYBw3zwKb73wzz4Q5Q1wtnJy8ofYqGDAl/NA= +github.com/coder/kaniko v0.0.0-20240830092517-0668f96c8816 h1:idB8jAnkYWkHYddbJ+WnGtM2wrhh3+JpjPwHcQ2lYhU= +github.com/coder/kaniko v0.0.0-20240830092517-0668f96c8816/go.mod h1:XoTDIhNF0Ll4tLmRYdOn31udU9w5zFrY2PME/crSRCA= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 h1:3A0ES21Ke+FxEM8CXx9n47SZOKOpgSE1bbJzlE4qPVs= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0/go.mod h1:5UuS2Ts+nTToAMeOjNlnHFkPahrtDkmpydBen/3wgZc= github.com/coder/quartz v0.1.0 h1:cLL+0g5l7xTf6ordRnUMMiZtRE8Sq5LxpghS63vEXrQ= @@ -489,8 +489,6 @@ github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= -github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= @@ -733,6 +731,8 @@ github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/twpayne/go-vfs/v5 v5.0.4 h1:/ne3h+rW7f5YOyOFguz+3ztfUwzOLR0Vts3y0mMAitg= +github.com/twpayne/go-vfs/v5 v5.0.4/go.mod h1:zTPFJUbgsEMFNSWnWQlLq9wh4AN83edZzx3VXbxrS1w= github.com/u-root/uio v0.0.0-20240209044354-b3d14b93376a h1:BH1SOPEvehD2kVrndDnGJiUF0TrBpNs+iyYocu6h0og= github.com/u-root/uio v0.0.0-20240209044354-b3d14b93376a/go.mod h1:P3a5rG4X7tI17Nn3aOIAYr5HbIMukwXG0urG0WuL8OA= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= diff --git a/integration/integration_test.go b/integration/integration_test.go index af051473..43d728c2 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -1485,22 +1485,26 @@ RUN date --utc > /root/date.txt`, testImageAlpine), }) t.Run("CacheAndPushMultistage", func(t *testing.T) { - // Currently fails with: - // /home/coder/src/coder/envbuilder/integration/integration_test.go:1417: "error: unable to get cached image: error fake building stage: failed to optimize instructions: failed to get files used from context: failed to get fileinfo for /.envbuilder/0/root/date.txt: lstat /.envbuilder/0/root/date.txt: no such file or directory" - // /home/coder/src/coder/envbuilder/integration/integration_test.go:1156: - // Error Trace: /home/coder/src/coder/envbuilder/integration/integration_test.go:1156 - // Error: Received unexpected error: - // error: unable to get cached image: error fake building stage: failed to optimize instructions: failed to get files used from context: failed to get fileinfo for /.envbuilder/0/root/date.txt: lstat /.envbuilder/0/root/date.txt: no such file or directory - // Test: TestPushImage/CacheAndPushMultistage - t.Skip("TODO: https://github.com/coder/envbuilder/issues/230") t.Parallel() srv := gittest.CreateGitServer(t, gittest.Options{ Files: map[string]string{ - "Dockerfile": fmt.Sprintf(`FROM %s AS a -RUN date --utc > /root/date.txt -FROM %s as b -COPY --from=a /root/date.txt /date.txt`, testImageAlpine, testImageAlpine), + "Dockerfile": fmt.Sprintf(` +FROM %[1]s AS prebuild +RUN mkdir /the-past /the-future \ + && echo "hello from the past" > /the-past/hello.txt \ + && cd /the-past \ + && ln -s hello.txt hello.link \ + && echo "hello from the future" > /the-future/hello.txt + +FROM %[1]s +USER root +ARG WORKDIR=/ +WORKDIR $WORKDIR +ENV FOO=bar +COPY --from=prebuild /the-past /the-past +COPY --from=prebuild /the-future/hello.txt /the-future/hello.txt +`, testImageAlpine), }, }) @@ -1525,16 +1529,122 @@ COPY --from=a /root/date.txt /date.txt`, testImageAlpine, testImageAlpine), require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") // When: we run envbuilder with PUSH_IMAGE set + _, err = runEnvbuilder(t, runOpts{env: []string{ + envbuilderEnv("GIT_URL", srv.URL), + envbuilderEnv("CACHE_REPO", testRepo), + envbuilderEnv("PUSH_IMAGE", "1"), + envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), + }}) + require.NoError(t, err) + + // Then: the image should be pushed + _, err = remote.Image(ref) + require.NoError(t, err, "expected image to be present after build + push") + + // Then: re-running envbuilder with GET_CACHED_IMAGE should succeed ctrID, err := runEnvbuilder(t, runOpts{env: []string{ + envbuilderEnv("GIT_URL", srv.URL), + envbuilderEnv("CACHE_REPO", testRepo), + envbuilderEnv("GET_CACHED_IMAGE", "1"), + envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), + }}) + require.NoError(t, err) + + // Then: the cached image ref should be emitted in the container logs + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + require.NoError(t, err) + defer cli.Close() + logs, err := cli.ContainerLogs(ctx, ctrID, container.LogsOptions{ + ShowStdout: true, + ShowStderr: true, + }) + require.NoError(t, err) + defer logs.Close() + logBytes, err := io.ReadAll(logs) + require.NoError(t, err) + require.Regexp(t, `ENVBUILDER_CACHED_IMAGE=(\S+)`, string(logBytes)) + + // When: we pull the image we just built + rc, err := cli.ImagePull(ctx, ref.String(), image.PullOptions{}) + require.NoError(t, err) + t.Cleanup(func() { _ = rc.Close() }) + _, err = io.ReadAll(rc) + require.NoError(t, err) + + // When: we run the image we just built + ctr, err := cli.ContainerCreate(ctx, &container.Config{ + Image: ref.String(), + Entrypoint: []string{"sleep", "infinity"}, + Labels: map[string]string{ + testContainerLabel: "true", + }, + }, nil, nil, nil, "") + require.NoError(t, err) + t.Cleanup(func() { + _ = cli.ContainerRemove(ctx, ctr.ID, container.RemoveOptions{ + RemoveVolumes: true, + Force: true, + }) + }) + err = cli.ContainerStart(ctx, ctr.ID, container.StartOptions{}) + require.NoError(t, err) + + // Then: The files from the prebuild stage are present. + out := execContainer(t, ctr.ID, "/bin/sh -c 'cat /the-past/hello.txt /the-future/hello.txt; readlink -f /the-past/hello.link'") + require.Equal(t, "hello from the past\nhello from the future\n/the-past/hello.txt", strings.TrimSpace(out)) + }) + + t.Run("MultistgeCacheMissAfterChange", func(t *testing.T) { + t.Parallel() + dockerfilePrebuildContents := fmt.Sprintf(` +FROM %[1]s AS prebuild +RUN mkdir /the-past /the-future \ + && echo "hello from the past" > /the-past/hello.txt \ + && cd /the-past \ + && ln -s hello.txt hello.link \ + && echo "hello from the future" > /the-future/hello.txt + +# Workaround for https://github.com/coder/envbuilder/issues/231 +FROM %[1]s +`, testImageAlpine) + + dockerfileContents := fmt.Sprintf(` +FROM %s +USER root +ARG WORKDIR=/ +WORKDIR $WORKDIR +ENV FOO=bar +COPY --from=prebuild /the-past /the-past +COPY --from=prebuild /the-future/hello.txt /the-future/hello.txt +RUN echo $FOO > /root/foo.txt +RUN date --utc > /root/date.txt +`, testImageAlpine) + + newServer := func(dockerfile string) *httptest.Server { + return gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{"Dockerfile": dockerfile}, + }) + } + srv := newServer(dockerfilePrebuildContents + dockerfileContents) + + // Given: an empty registry + testReg := setupInMemoryRegistry(t, setupInMemoryRegistryOpts{}) + testRepo := testReg + "/test" + ref, err := name.ParseReference(testRepo + ":latest") + require.NoError(t, err) + _, err = remote.Image(ref) + require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") + + // When: we run envbuilder with PUSH_IMAGE set + _, err = runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), envbuilderEnv("PUSH_IMAGE", "1"), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), }}) require.NoError(t, err) - // Then: The file copied from stage a should be present - out := execContainer(t, ctrID, "cat /date.txt") - require.NotEmpty(t, out) // Then: the image should be pushed _, err = remote.Image(ref) @@ -1548,6 +1658,33 @@ COPY --from=a /root/date.txt /date.txt`, testImageAlpine, testImageAlpine), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), }}) require.NoError(t, err) + + // When: we change the Dockerfile + srv.Close() + dockerfilePrebuildContents = strings.Replace(dockerfilePrebuildContents, "hello from the future", "hello from the future, but different", 1) + srv = newServer(dockerfilePrebuildContents) + + // When: we rebuild the prebuild stage so that the cache is created + _, err = runEnvbuilder(t, runOpts{env: []string{ + envbuilderEnv("GIT_URL", srv.URL), + envbuilderEnv("CACHE_REPO", testRepo), + envbuilderEnv("PUSH_IMAGE", "1"), + envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), + }}) + require.NoError(t, err) + + // Then: re-running envbuilder with GET_CACHED_IMAGE should still fail + // on the second stage because the first stage file has changed. + srv.Close() + srv = newServer(dockerfilePrebuildContents + dockerfileContents) + _, err = runEnvbuilder(t, runOpts{env: []string{ + envbuilderEnv("GIT_URL", srv.URL), + envbuilderEnv("CACHE_REPO", testRepo), + envbuilderEnv("GET_CACHED_IMAGE", "1"), + envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), + envbuilderEnv("VERBOSE", "1"), + }}) + require.ErrorContains(t, err, "error probing build cache: uncached COPY command") }) t.Run("PushImageRequiresCache", func(t *testing.T) { From eacf6c9492eca84c74dff7b15eea88513ddffc16 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 30 Aug 2024 14:30:05 +0100 Subject: [PATCH 41/81] chore: bump github.com/docker/cli from 27.0.3+incompatible to 27.2.0+incompatible (#329) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> (cherry picked from commit bb53f2fa1e26a1e992bfae87ce5b0cb7efb32e9c) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e1d01738..da66a779 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/coder/serpent v0.7.0 github.com/containerd/platforms v0.2.1 github.com/distribution/distribution/v3 v3.0.0-alpha.1 - github.com/docker/cli v27.0.3+incompatible + github.com/docker/cli v27.2.0+incompatible github.com/docker/docker v26.1.5+incompatible github.com/fatih/color v1.17.0 github.com/gliderlabs/ssh v0.3.7 diff --git a/go.sum b/go.sum index 6101dcf6..675a00e3 100644 --- a/go.sum +++ b/go.sum @@ -236,8 +236,8 @@ github.com/distribution/distribution/v3 v3.0.0-alpha.1 h1:jn7I1gvjOvmLztH1+1cLiU github.com/distribution/distribution/v3 v3.0.0-alpha.1/go.mod h1:LCp4JZp1ZalYg0W/TN05jarCQu+h4w7xc7ZfQF4Y/cY= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/cli v27.0.3+incompatible h1:usGs0/BoBW8MWxGeEtqPMkzOY56jZ6kYlSN5BLDioCQ= -github.com/docker/cli v27.0.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v27.2.0+incompatible h1:yHD1QEB1/0vr5eBNpu8tncu8gWxg8EydFPOSKHzXSMM= +github.com/docker/cli v27.2.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v26.1.5+incompatible h1:NEAxTwEjxV6VbBMBoGG3zPqbiJosIApZjxlbrG9q3/g= From 3f7bbf6aabfea125965ef226d9881a646cddcb54 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 30 Aug 2024 15:50:58 +0100 Subject: [PATCH 42/81] chore: update kaniko fork to fix 32-bit ARM build failure (#330) (cherry picked from commit fb7e689f39edd84d93f3ed0b99fc494c8f0a9f0e) --- .github/workflows/ci.yaml | 9 ++++++--- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ecb23ed1..457b3117 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -99,11 +99,14 @@ jobs: - name: Build if: github.event_name == 'pull_request' run: | - BASE=ghcr.io/coder/envbuilder-preview + ./scripts/build.sh \ + --arch=amd64 ./scripts/build.sh \ - --arch=amd64 \ - --base=$BASE + --arch=arm64 + + ./scripts/build.sh \ + --arch=arm - name: Build and Push if: github.ref == 'refs/heads/main' diff --git a/go.mod b/go.mod index da66a779..1abca545 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22.4 // There are a few options we need added to Kaniko! // See: https://github.com/GoogleContainerTools/kaniko/compare/main...coder:kaniko:main -replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240830092517-0668f96c8816 +replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240830141327-f307586e3dca // Required to import codersdk due to gvisor dependency. replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20240702054557-aa558fbe5374 diff --git a/go.sum b/go.sum index 675a00e3..e07e3750 100644 --- a/go.sum +++ b/go.sum @@ -171,8 +171,8 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352 h1:L/EjCuZxs5tOcqqCaASj/nu65TRYEFcTt8qRQfHZXX0= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352/go.mod h1:P1KoQSgnKEAG6Mnd3YlGzAophty+yKA9VV48LpfNRvo= -github.com/coder/kaniko v0.0.0-20240830092517-0668f96c8816 h1:idB8jAnkYWkHYddbJ+WnGtM2wrhh3+JpjPwHcQ2lYhU= -github.com/coder/kaniko v0.0.0-20240830092517-0668f96c8816/go.mod h1:XoTDIhNF0Ll4tLmRYdOn31udU9w5zFrY2PME/crSRCA= +github.com/coder/kaniko v0.0.0-20240830141327-f307586e3dca h1:PrcSWrllqipTrtet50a3VyAJEQmjziIZyhpy0bsC6o0= +github.com/coder/kaniko v0.0.0-20240830141327-f307586e3dca/go.mod h1:XoTDIhNF0Ll4tLmRYdOn31udU9w5zFrY2PME/crSRCA= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 h1:3A0ES21Ke+FxEM8CXx9n47SZOKOpgSE1bbJzlE4qPVs= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0/go.mod h1:5UuS2Ts+nTToAMeOjNlnHFkPahrtDkmpydBen/3wgZc= github.com/coder/quartz v0.1.0 h1:cLL+0g5l7xTf6ordRnUMMiZtRE8Sq5LxpghS63vEXrQ= From 8c364f59879fdc9ec0a78045885ab67836f51a3c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 14:19:41 +0100 Subject: [PATCH 43/81] chore: bump golang.org/x/mod from 0.18.0 to 0.21.0 (#338) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> (cherry picked from commit 754b9ddb4cd94e92f7d4ec95012ca4debe59c632) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1abca545..9b806fb3 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a go.uber.org/mock v0.4.0 golang.org/x/crypto v0.26.0 - golang.org/x/mod v0.18.0 + golang.org/x/mod v0.21.0 golang.org/x/sync v0.8.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 ) diff --git a/go.sum b/go.sum index e07e3750..c1f1fcb5 100644 --- a/go.sum +++ b/go.sum @@ -848,8 +848,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= -golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= From fb50f8375ed4bc48abbb44326984c3c63a97bd8a Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 9 Sep 2024 23:03:09 +0100 Subject: [PATCH 44/81] fix: allow setting MagicDir in Options (#337) - Unexports constants.MagicDir and moves it to an internal package. - Moves other const/var declarations in `constants` package to where they are used - Removes constants package - Adds MagicDirBase to options (intentionally not configurable via CLI) - Removes usages of default MagicDir to instead use path provided by options Co-authored-by: Danny Kopping Co-authored-by: Mathias Fredriksson (cherry picked from commit e6c8c66068fd58f0b873d681c390f8c7756dd04c) --- constants/constants.go | 64 ---------- envbuilder.go | 122 ++++++++++---------- integration/integration_test.go | 17 +-- internal/magicdir/magicdir.go | 78 +++++++++++++ internal/magicdir/magicdir_internal_test.go | 38 ++++++ options/defaults.go | 18 ++- options/defaults_test.go | 9 +- options/options.go | 14 ++- 8 files changed, 217 insertions(+), 143 deletions(-) delete mode 100644 constants/constants.go create mode 100644 internal/magicdir/magicdir.go create mode 100644 internal/magicdir/magicdir_internal_test.go diff --git a/constants/constants.go b/constants/constants.go deleted file mode 100644 index fefa1394..00000000 --- a/constants/constants.go +++ /dev/null @@ -1,64 +0,0 @@ -package constants - -import ( - "errors" - "fmt" - "path/filepath" -) - -const ( - // WorkspacesDir is the path to the directory where - // all workspaces are stored by default. - WorkspacesDir = "/workspaces" - - // EmptyWorkspaceDir is the path to a workspace that has - // nothing going on... it's empty! - EmptyWorkspaceDir = WorkspacesDir + "/empty" - - // MagicDir is where all envbuilder related files are stored. - // This is a special directory that must not be modified - // by the user or images. - MagicDir = "/.envbuilder" -) - -var ( - ErrNoFallbackImage = errors.New("no fallback image has been specified") - - // MagicFile is a file that is created in the workspace - // when envbuilder has already been run. This is used - // to skip building when a container is restarting. - // e.g. docker stop -> docker start - MagicFile = filepath.Join(MagicDir, "built") - - // MagicFile is the location of the build context when - // using remote build mode. - MagicRemoteRepoDir = filepath.Join(MagicDir, "repo") - - // MagicBinaryLocation is the expected location of the envbuilder binary - // inside a builder image. - MagicBinaryLocation = filepath.Join(MagicDir, "bin", "envbuilder") - - // MagicImage is a file that is created in the image when - // envbuilder has already been run. This is used to skip - // the destructive initial build step when 'resuming' envbuilder - // from a previously built image. - MagicImage = filepath.Join(MagicDir, "image") - - // MagicTempDir is a directory inside the build context inside which - // we place files referenced by MagicDirectives. - MagicTempDir = ".envbuilder.tmp" - - // MagicDirectives are directives automatically appended to Dockerfiles - // when pushing the image. These directives allow the built image to be - // 're-used'. - MagicDirectives = fmt.Sprintf(` -COPY --chmod=0755 %[1]s %[2]s -COPY --chmod=0644 %[3]s %[4]s -USER root -WORKDIR / -ENTRYPOINT [%[2]q] -`, - ".envbuilder.tmp/envbuilder", MagicBinaryLocation, - ".envbuilder.tmp/image", MagicImage, - ) -) diff --git a/envbuilder.go b/envbuilder.go index 7f3c983a..ac64beea 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -25,7 +25,6 @@ import ( "time" "github.com/coder/envbuilder/buildinfo" - "github.com/coder/envbuilder/constants" "github.com/coder/envbuilder/git" "github.com/coder/envbuilder/options" "github.com/go-git/go-billy/v5" @@ -36,6 +35,7 @@ import ( "github.com/GoogleContainerTools/kaniko/pkg/util" "github.com/coder/envbuilder/devcontainer" "github.com/coder/envbuilder/internal/ebutil" + "github.com/coder/envbuilder/internal/magicdir" "github.com/coder/envbuilder/log" "github.com/containerd/platforms" "github.com/distribution/distribution/v3/configuration" @@ -52,6 +52,9 @@ import ( "golang.org/x/xerrors" ) +// ErrNoFallbackImage is returned when no fallback image has been specified. +var ErrNoFallbackImage = errors.New("no fallback image has been specified") + // DockerConfig represents the Docker configuration file. type DockerConfig configfile.ConfigFile @@ -68,6 +71,9 @@ func Run(ctx context.Context, opts options.Options) error { if opts.CacheRepo == "" && opts.PushImage { return fmt.Errorf("--cache-repo must be set when using --push-image") } + + magicDir := magicdir.At(opts.MagicDirBase) + // Default to the shell! initArgs := []string{"-c", opts.InitScript} if opts.InitArgs != "" { @@ -92,7 +98,7 @@ func Run(ctx context.Context, opts options.Options) error { opts.Logger(log.LevelInfo, "%s %s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder"), buildinfo.Version()) - cleanupDockerConfigJSON, err := initDockerConfigJSON(opts.DockerConfigBase64) + cleanupDockerConfigJSON, err := initDockerConfigJSON(opts.Logger, magicDir, opts.DockerConfigBase64) if err != nil { return err } @@ -168,7 +174,7 @@ func Run(ctx context.Context, opts options.Options) error { } defaultBuildParams := func() (*devcontainer.Compiled, error) { - dockerfile := filepath.Join(constants.MagicDir, "Dockerfile") + dockerfile := magicDir.Join("Dockerfile") file, err := opts.Filesystem.OpenFile(dockerfile, os.O_CREATE|os.O_WRONLY, 0o644) if err != nil { return nil, err @@ -176,11 +182,11 @@ func Run(ctx context.Context, opts options.Options) error { defer file.Close() if opts.FallbackImage == "" { if fallbackErr != nil { - return nil, xerrors.Errorf("%s: %w", fallbackErr.Error(), constants.ErrNoFallbackImage) + return nil, xerrors.Errorf("%s: %w", fallbackErr.Error(), ErrNoFallbackImage) } // We can't use errors.Join here because our tests // don't support parsing a multiline error. - return nil, constants.ErrNoFallbackImage + return nil, ErrNoFallbackImage } content := "FROM " + opts.FallbackImage _, err = file.Write([]byte(content)) @@ -190,7 +196,7 @@ func Run(ctx context.Context, opts options.Options) error { return &devcontainer.Compiled{ DockerfilePath: dockerfile, DockerfileContent: content, - BuildContext: constants.MagicDir, + BuildContext: magicDir.Path(), }, nil } @@ -232,7 +238,7 @@ func Run(ctx context.Context, opts options.Options) error { opts.Logger(log.LevelInfo, "No Dockerfile or image specified; falling back to the default image...") fallbackDockerfile = defaultParams.DockerfilePath } - buildParams, err = devContainer.Compile(opts.Filesystem, devcontainerDir, constants.MagicDir, fallbackDockerfile, opts.WorkspaceFolder, false, os.LookupEnv) + buildParams, err = devContainer.Compile(opts.Filesystem, devcontainerDir, magicDir.Path(), fallbackDockerfile, opts.WorkspaceFolder, false, os.LookupEnv) if err != nil { return fmt.Errorf("compile devcontainer.json: %w", err) } @@ -304,7 +310,7 @@ func Run(ctx context.Context, opts options.Options) error { // So we add them to the default ignore list. See: // https://github.com/GoogleContainerTools/kaniko/blob/63be4990ca5a60bdf06ddc4d10aa4eca0c0bc714/cmd/executor/cmd/root.go#L136 ignorePaths := append([]string{ - constants.MagicDir, + magicDir.Path(), opts.WorkspaceFolder, // See: https://github.com/coder/envbuilder/issues/37 "/etc/resolv.conf", @@ -332,31 +338,25 @@ func Run(ctx context.Context, opts options.Options) error { if err := util.AddAllowedPathToDefaultIgnoreList(opts.BinaryPath); err != nil { return fmt.Errorf("add envbuilder binary to ignore list: %w", err) } - if err := util.AddAllowedPathToDefaultIgnoreList(constants.MagicImage); err != nil { + if err := util.AddAllowedPathToDefaultIgnoreList(magicDir.Image()); err != nil { return fmt.Errorf("add magic image file to ignore list: %w", err) } - magicTempDir := filepath.Join(buildParams.BuildContext, constants.MagicTempDir) - if err := opts.Filesystem.MkdirAll(magicTempDir, 0o755); err != nil { + magicTempDir := magicdir.At(buildParams.BuildContext, magicdir.TempDir) + if err := opts.Filesystem.MkdirAll(magicTempDir.Path(), 0o755); err != nil { return fmt.Errorf("create magic temp dir in build context: %w", err) } // Add the magic directives that embed the binary into the built image. - buildParams.DockerfileContent += constants.MagicDirectives + buildParams.DockerfileContent += magicdir.Directives // Copy the envbuilder binary into the build context. // External callers will need to specify the path to the desired envbuilder binary. - envbuilderBinDest := filepath.Join( - magicTempDir, - filepath.Base(constants.MagicBinaryLocation), - ) + envbuilderBinDest := filepath.Join(magicTempDir.Path(), "envbuilder") // Also touch the magic file that signifies the image has been built! - magicImageDest := filepath.Join( - magicTempDir, - filepath.Base(constants.MagicImage), - ) + magicImageDest := magicTempDir.Image() // Clean up after build! var cleanupOnce sync.Once cleanupBuildContext = func() { cleanupOnce.Do(func() { - for _, path := range []string{magicImageDest, envbuilderBinDest, magicTempDir} { + for _, path := range []string{magicImageDest, envbuilderBinDest, magicTempDir.Path()} { if err := opts.Filesystem.Remove(path); err != nil { opts.Logger(log.LevelWarn, "failed to clean up magic temp dir from build context: %w", err) } @@ -370,15 +370,14 @@ func Run(ctx context.Context, opts options.Options) error { return fmt.Errorf("copy envbuilder binary to build context: %w", err) } - opts.Logger(log.LevelDebug, "touching magic image file at %q in build context %q", magicImageDest, buildParams.BuildContext) + opts.Logger(log.LevelDebug, "touching magic image file at %q in build context %q", magicImageDest, magicTempDir) if err := touchFile(opts.Filesystem, magicImageDest, 0o755); err != nil { return fmt.Errorf("touch magic image file in build context: %w", err) } - } // temp move of all ro mounts - tempRemountDest := filepath.Join("/", constants.MagicDir, "mnt") + tempRemountDest := magicDir.Join("mnt") // ignorePrefixes is a superset of ignorePaths that we pass to kaniko's // IgnoreList. ignorePrefixes := append([]string{"/dev", "/proc", "/sys"}, ignorePaths...) @@ -399,8 +398,8 @@ func Run(ctx context.Context, opts options.Options) error { defer closeStderr() build := func() (v1.Image, error) { defer cleanupBuildContext() - _, alreadyBuiltErr := opts.Filesystem.Stat(constants.MagicFile) - _, isImageErr := opts.Filesystem.Stat(constants.MagicImage) + _, alreadyBuiltErr := opts.Filesystem.Stat(magicDir.Built()) + _, isImageErr := opts.Filesystem.Stat(magicDir.Image()) if (alreadyBuiltErr == nil && opts.SkipRebuild) || isImageErr == nil { endStage := startStage("🏗️ Skipping build because of cache...") imageRef, err := devcontainer.ImageFromDockerfile(buildParams.DockerfileContent) @@ -545,7 +544,7 @@ func Run(ctx context.Context, opts options.Options) error { // Create the magic file to indicate that this build // has already been ran before! - file, err := opts.Filesystem.Create(constants.MagicFile) + file, err := opts.Filesystem.Create(magicDir.Built()) if err != nil { return fmt.Errorf("create magic file: %w", err) } @@ -752,7 +751,7 @@ func Run(ctx context.Context, opts options.Options) error { opts.Logger(log.LevelInfo, "=== Running the setup command %q as the root user...", opts.SetupScript) envKey := "ENVBUILDER_ENV" - envFile := filepath.Join("/", constants.MagicDir, "environ") + envFile := magicDir.Join("environ") file, err := os.Create(envFile) if err != nil { return fmt.Errorf("create environ file: %w", err) @@ -862,6 +861,8 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) return nil, fmt.Errorf("--cache-repo must be set when using --get-cached-image") } + magicDir := magicdir.At(opts.MagicDirBase) + stageNumber := 0 startStage := func(format string, args ...any) func(format string, args ...any) { now := time.Now() @@ -876,7 +877,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) opts.Logger(log.LevelInfo, "%s %s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder"), buildinfo.Version()) - cleanupDockerConfigJSON, err := initDockerConfigJSON(opts.DockerConfigBase64) + cleanupDockerConfigJSON, err := initDockerConfigJSON(opts.Logger, magicDir, opts.DockerConfigBase64) if err != nil { return nil, err } @@ -960,11 +961,11 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) defer file.Close() if opts.FallbackImage == "" { if fallbackErr != nil { - return nil, fmt.Errorf("%s: %w", fallbackErr.Error(), constants.ErrNoFallbackImage) + return nil, fmt.Errorf("%s: %w", fallbackErr.Error(), ErrNoFallbackImage) } // We can't use errors.Join here because our tests // don't support parsing a multiline error. - return nil, constants.ErrNoFallbackImage + return nil, ErrNoFallbackImage } content := "FROM " + opts.FallbackImage _, err = file.Write([]byte(content)) @@ -1080,7 +1081,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) // So we add them to the default ignore list. See: // https://github.com/GoogleContainerTools/kaniko/blob/63be4990ca5a60bdf06ddc4d10aa4eca0c0bc714/cmd/executor/cmd/root.go#L136 ignorePaths := append([]string{ - constants.MagicDir, + magicDir.Path(), opts.WorkspaceFolder, // See: https://github.com/coder/envbuilder/issues/37 "/etc/resolv.conf", @@ -1103,29 +1104,25 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) // build via executor.RunCacheProbe we need to have the *exact* copy of the // envbuilder binary available used to build the image and we also need to // add the magic directives to the Dockerfile content. - buildParams.DockerfileContent += constants.MagicDirectives - magicTempDir := filepath.Join(buildParams.BuildContext, constants.MagicTempDir) + // MAGICDIR + buildParams.DockerfileContent += magicdir.Directives + magicTempDir := filepath.Join(buildParams.BuildContext, magicdir.TempDir) if err := opts.Filesystem.MkdirAll(magicTempDir, 0o755); err != nil { return nil, fmt.Errorf("create magic temp dir in build context: %w", err) } - envbuilderBinDest := filepath.Join( - magicTempDir, - filepath.Base(constants.MagicBinaryLocation), - ) + envbuilderBinDest := filepath.Join(magicTempDir, "envbuilder") // Copy the envbuilder binary into the build context. - opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, buildParams.BuildContext) + opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, envbuilderBinDest) if err := copyFile(opts.Filesystem, opts.BinaryPath, envbuilderBinDest, 0o755); err != nil { return nil, xerrors.Errorf("copy envbuilder binary to build context: %w", err) } - // Also touch the magic file that signifies the image has been built! - magicImageDest := filepath.Join( - magicTempDir, - filepath.Base(constants.MagicImage), - ) + // Also touch the magic file that signifies the image has been built!A + magicImageDest := filepath.Join(magicTempDir, "image") + opts.Logger(log.LevelDebug, "touching magic image file at %q in build context %q", magicImageDest, magicTempDir) if err := touchFile(opts.Filesystem, magicImageDest, 0o755); err != nil { - return nil, fmt.Errorf("touch magic image file in build context: %w", err) + return nil, fmt.Errorf("touch magic image file at %q: %w", magicImageDest, err) } defer func() { // Clean up after we're done! @@ -1417,21 +1414,24 @@ func findDevcontainerJSON(workspaceFolder string, options options.Options) (stri // maybeDeleteFilesystem wraps util.DeleteFilesystem with a guard to hopefully stop // folks from unwittingly deleting their entire root directory. func maybeDeleteFilesystem(logger log.Func, force bool) error { + // We always expect the magic directory to be set to the default, signifying that + // the user is running envbuilder in a container. + // If this is set to anything else we should bail out to prevent accidental data loss. + // defaultMagicDir := magicdir.MagicDir("") kanikoDir, ok := os.LookupEnv("KANIKO_DIR") - if !ok || strings.TrimSpace(kanikoDir) != constants.MagicDir { - if force { - bailoutSecs := 10 - logger(log.LevelWarn, "WARNING! BYPASSING SAFETY CHECK! THIS WILL DELETE YOUR ROOT FILESYSTEM!") - logger(log.LevelWarn, "You have %d seconds to bail out!", bailoutSecs) - for i := bailoutSecs; i > 0; i-- { - logger(log.LevelWarn, "%d...", i) - <-time.After(time.Second) - } - } else { - logger(log.LevelError, "KANIKO_DIR is not set to %s. Bailing!\n", constants.MagicDir) + if !ok || strings.TrimSpace(kanikoDir) != magicdir.Default.Path() { + if !force { + logger(log.LevelError, "KANIKO_DIR is not set to %s. Bailing!\n", magicdir.Default.Path()) logger(log.LevelError, "To bypass this check, set FORCE_SAFE=true.") return errors.New("safety check failed") } + bailoutSecs := 10 + logger(log.LevelWarn, "WARNING! BYPASSING SAFETY CHECK! THIS WILL DELETE YOUR ROOT FILESYSTEM!") + logger(log.LevelWarn, "You have %d seconds to bail out!", bailoutSecs) + for i := bailoutSecs; i > 0; i-- { + logger(log.LevelWarn, "%d...", i) + <-time.After(time.Second) + } } return util.DeleteFilesystem() @@ -1469,13 +1469,13 @@ func touchFile(fs billy.Filesystem, dst string, mode fs.FileMode) error { return f.Close() } -func initDockerConfigJSON(dockerConfigBase64 string) (func() error, error) { +func initDockerConfigJSON(logf log.Func, magicDir magicdir.MagicDir, dockerConfigBase64 string) (func() error, error) { var cleanupOnce sync.Once noop := func() error { return nil } if dockerConfigBase64 == "" { return noop, nil } - cfgPath := filepath.Join(constants.MagicDir, "config.json") + cfgPath := filepath.Join(magicDir.Path(), "config.json") decoded, err := base64.StdEncoding.DecodeString(dockerConfigBase64) if err != nil { return noop, fmt.Errorf("decode docker config: %w", err) @@ -1489,10 +1489,14 @@ func initDockerConfigJSON(dockerConfigBase64 string) (func() error, error) { if err != nil { return noop, fmt.Errorf("parse docker config: %w", err) } + for k := range configFile.AuthConfigs { + logf(log.LevelInfo, "Docker config contains auth for registry %q", k) + } err = os.WriteFile(cfgPath, decoded, 0o644) if err != nil { return noop, fmt.Errorf("write docker config: %w", err) } + logf(log.LevelInfo, "Wrote Docker config JSON to %s", cfgPath) cleanup := func() error { var cleanupErr error cleanupOnce.Do(func() { @@ -1501,7 +1505,7 @@ func initDockerConfigJSON(dockerConfigBase64 string) (func() error, error) { if !errors.Is(err, fs.ErrNotExist) { cleanupErr = fmt.Errorf("remove docker config: %w", cleanupErr) } - _, _ = fmt.Fprintf(os.Stderr, "failed to remove the Docker config secret file: %s\n", cleanupErr) + logf(log.LevelError, "Failed to remove the Docker config secret file: %s", cleanupErr) } }) return cleanupErr diff --git a/integration/integration_test.go b/integration/integration_test.go index 43d728c2..e0e012ba 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -22,8 +22,8 @@ import ( "time" "github.com/coder/envbuilder" - "github.com/coder/envbuilder/constants" "github.com/coder/envbuilder/devcontainer/features" + "github.com/coder/envbuilder/internal/magicdir" "github.com/coder/envbuilder/options" "github.com/coder/envbuilder/testutil/gittest" "github.com/coder/envbuilder/testutil/mwtest" @@ -365,7 +365,8 @@ func TestBuildFromDockerfile(t *testing.T) { require.Equal(t, "hello", strings.TrimSpace(output)) // Verify that the Docker configuration secret file is removed - output = execContainer(t, ctr, "stat "+filepath.Join(constants.MagicDir, "config.json")) + configJSONContainerPath := magicdir.Default.Join("config.json") + output = execContainer(t, ctr, "stat "+configJSONContainerPath) require.Contains(t, output, "No such file or directory") } @@ -591,7 +592,7 @@ func TestCloneFailsFallback(t *testing.T) { _, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", "bad-value"), }}) - require.ErrorContains(t, err, constants.ErrNoFallbackImage.Error()) + require.ErrorContains(t, err, envbuilder.ErrNoFallbackImage.Error()) }) } @@ -609,7 +610,7 @@ func TestBuildFailsFallback(t *testing.T) { envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), }}) - require.ErrorContains(t, err, constants.ErrNoFallbackImage.Error()) + require.ErrorContains(t, err, envbuilder.ErrNoFallbackImage.Error()) require.ErrorContains(t, err, "dockerfile parse error") }) t.Run("FailsBuild", func(t *testing.T) { @@ -625,7 +626,7 @@ RUN exit 1`, envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), }}) - require.ErrorContains(t, err, constants.ErrNoFallbackImage.Error()) + require.ErrorContains(t, err, envbuilder.ErrNoFallbackImage.Error()) }) t.Run("BadDevcontainer", func(t *testing.T) { t.Parallel() @@ -638,7 +639,7 @@ RUN exit 1`, _, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), }}) - require.ErrorContains(t, err, constants.ErrNoFallbackImage.Error()) + require.ErrorContains(t, err, envbuilder.ErrNoFallbackImage.Error()) }) t.Run("NoImageOrDockerfile", func(t *testing.T) { t.Parallel() @@ -971,7 +972,7 @@ func setupPassthroughRegistry(t *testing.T, image string, opts *setupPassthrough func TestNoMethodFails(t *testing.T) { _, err := runEnvbuilder(t, runOpts{env: []string{}}) - require.ErrorContains(t, err, constants.ErrNoFallbackImage.Error()) + require.ErrorContains(t, err, envbuilder.ErrNoFallbackImage.Error()) } func TestDockerfileBuildContext(t *testing.T) { @@ -1157,6 +1158,7 @@ RUN date --utc > /root/date.txt`, testImageAlpine), envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), envbuilderEnv("GET_CACHED_IMAGE", "1"), + envbuilderEnv("VERBOSE", "1"), }}) require.ErrorContains(t, err, "error probing build cache: uncached RUN command") // Then: it should fail to build the image and nothing should be pushed @@ -1168,6 +1170,7 @@ RUN date --utc > /root/date.txt`, testImageAlpine), envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), envbuilderEnv("PUSH_IMAGE", "1"), + envbuilderEnv("VERBOSE", "1"), }}) require.NoError(t, err) diff --git a/internal/magicdir/magicdir.go b/internal/magicdir/magicdir.go new file mode 100644 index 00000000..31bcd7c9 --- /dev/null +++ b/internal/magicdir/magicdir.go @@ -0,0 +1,78 @@ +package magicdir + +import ( + "fmt" + "path/filepath" +) + +const ( + // defaultMagicDirBase is the default working location for envbuilder. + // This is a special directory that must not be modified by the user + // or images. This is intentionally unexported. + defaultMagicDirBase = "/.envbuilder" + + // TempDir is a directory inside the build context inside which + // we place files referenced by MagicDirectives. + TempDir = ".envbuilder.tmp" +) + +var ( + // Default is the default working directory for Envbuilder. + // This defaults to /.envbuilder. It should only be used when Envbuilder + // is known to be running as root inside a container. + Default MagicDir + // Directives are directives automatically appended to Dockerfiles + // when pushing the image. These directives allow the built image to be + // 're-used'. + Directives = fmt.Sprintf(` +COPY --chmod=0755 %[1]s/envbuilder %[2]s/bin/envbuilder +COPY --chmod=0644 %[1]s/image %[2]s/image +USER root +WORKDIR / +ENTRYPOINT ["%[2]s/bin/envbuilder"] +`, TempDir, defaultMagicDirBase) +) + +// MagicDir is a working directory for envbuilder. It +// will also be present in images built by envbuilder. +type MagicDir struct { + base string +} + +// At returns a MagicDir rooted at filepath.Join(paths...) +func At(paths ...string) MagicDir { + if len(paths) == 0 { + return MagicDir{} + } + return MagicDir{base: filepath.Join(paths...)} +} + +// Join returns the result of filepath.Join([m.Path, paths...]). +func (m MagicDir) Join(paths ...string) string { + return filepath.Join(append([]string{m.Path()}, paths...)...) +} + +// String returns the string representation of the MagicDir. +func (m MagicDir) Path() string { + // Instead of the zero value, use defaultMagicDir. + if m.base == "" { + return defaultMagicDirBase + } + return m.base +} + +// Built is a file that is created in the workspace +// when envbuilder has already been run. This is used +// to skip building when a container is restarting. +// e.g. docker stop -> docker start +func (m MagicDir) Built() string { + return m.Join("built") +} + +// Image is a file that is created in the image when +// envbuilder has already been run. This is used to skip +// the destructive initial build step when 'resuming' envbuilder +// from a previously built image. +func (m MagicDir) Image() string { + return m.Join("image") +} diff --git a/internal/magicdir/magicdir_internal_test.go b/internal/magicdir/magicdir_internal_test.go new file mode 100644 index 00000000..43b66ba0 --- /dev/null +++ b/internal/magicdir/magicdir_internal_test.go @@ -0,0 +1,38 @@ +package magicdir + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_MagicDir(t *testing.T) { + t.Parallel() + + t.Run("Default", func(t *testing.T) { + t.Parallel() + require.Equal(t, defaultMagicDirBase+"/foo", Default.Join("foo")) + require.Equal(t, defaultMagicDirBase, Default.Path()) + require.Equal(t, defaultMagicDirBase+"/built", Default.Built()) + require.Equal(t, defaultMagicDirBase+"/image", Default.Image()) + }) + + t.Run("ZeroValue", func(t *testing.T) { + t.Parallel() + var md MagicDir + require.Equal(t, defaultMagicDirBase+"/foo", md.Join("foo")) + require.Equal(t, defaultMagicDirBase, md.Path()) + require.Equal(t, defaultMagicDirBase+"/built", md.Built()) + require.Equal(t, defaultMagicDirBase+"/image", md.Image()) + }) + + t.Run("At", func(t *testing.T) { + t.Parallel() + tmpDir := t.TempDir() + md := At(tmpDir) + require.Equal(t, tmpDir+"/foo", md.Join("foo")) + require.Equal(t, tmpDir, md.Path()) + require.Equal(t, tmpDir+"/built", md.Built()) + require.Equal(t, tmpDir+"/image", md.Image()) + }) +} diff --git a/options/defaults.go b/options/defaults.go index 42e48063..df3d436c 100644 --- a/options/defaults.go +++ b/options/defaults.go @@ -7,24 +7,28 @@ import ( "github.com/go-git/go-billy/v5/osfs" giturls "github.com/chainguard-dev/git-urls" - "github.com/coder/envbuilder/constants" "github.com/coder/envbuilder/internal/chmodfs" + "github.com/coder/envbuilder/internal/magicdir" ) +// EmptyWorkspaceDir is the path to a workspace that has +// nothing going on... it's empty! +var EmptyWorkspaceDir = "/workspaces/empty" + // DefaultWorkspaceFolder returns the default workspace folder // for a given repository URL. func DefaultWorkspaceFolder(repoURL string) string { if repoURL == "" { - return constants.EmptyWorkspaceDir + return EmptyWorkspaceDir } parsed, err := giturls.Parse(repoURL) if err != nil { - return constants.EmptyWorkspaceDir + return EmptyWorkspaceDir } name := strings.Split(parsed.Path, "/") hasOwnerAndRepo := len(name) >= 2 if !hasOwnerAndRepo { - return constants.EmptyWorkspaceDir + return EmptyWorkspaceDir } repo := strings.TrimSuffix(name[len(name)-1], ".git") return fmt.Sprintf("/workspaces/%s", repo) @@ -55,7 +59,13 @@ func (o *Options) SetDefaults() { if o.WorkspaceFolder == "" { o.WorkspaceFolder = DefaultWorkspaceFolder(o.GitURL) } + if o.RemoteRepoDir == "" { + o.RemoteRepoDir = magicdir.Default.Join("repo") + } if o.BinaryPath == "" { o.BinaryPath = "/.envbuilder/bin/envbuilder" } + if o.MagicDirBase == "" { + o.MagicDirBase = magicdir.Default.Path() + } } diff --git a/options/defaults_test.go b/options/defaults_test.go index 48783585..8c9946f6 100644 --- a/options/defaults_test.go +++ b/options/defaults_test.go @@ -8,7 +8,6 @@ import ( "github.com/stretchr/testify/assert" - "github.com/coder/envbuilder/constants" "github.com/coder/envbuilder/options" "github.com/stretchr/testify/require" ) @@ -44,7 +43,7 @@ func TestDefaultWorkspaceFolder(t *testing.T) { { name: "empty", gitURL: "", - expected: constants.EmptyWorkspaceDir, + expected: options.EmptyWorkspaceDir, }, } for _, tt := range successTests { @@ -70,7 +69,7 @@ func TestDefaultWorkspaceFolder(t *testing.T) { for _, tt := range invalidTests { t.Run(tt.name, func(t *testing.T) { dir := options.DefaultWorkspaceFolder(tt.invalidURL) - require.Equal(t, constants.EmptyWorkspaceDir, dir) + require.Equal(t, options.EmptyWorkspaceDir, dir) }) } } @@ -84,7 +83,9 @@ func TestOptions_SetDefaults(t *testing.T) { IgnorePaths: []string{"/var/run", "/product_uuid", "/product_name"}, Filesystem: chmodfs.New(osfs.New("/")), GitURL: "", - WorkspaceFolder: constants.EmptyWorkspaceDir, + WorkspaceFolder: options.EmptyWorkspaceDir, + MagicDirBase: "/.envbuilder", + RemoteRepoDir: "/.envbuilder/repo", BinaryPath: "/.envbuilder/bin/envbuilder", } diff --git a/options/options.go b/options/options.go index d7bd66b3..a0058cd3 100644 --- a/options/options.go +++ b/options/options.go @@ -7,7 +7,6 @@ import ( "os" "strings" - "github.com/coder/envbuilder/constants" "github.com/coder/envbuilder/log" "github.com/coder/serpent" "github.com/go-git/go-billy/v5" @@ -166,6 +165,11 @@ type Options struct { // attempting to probe the build cache. This is only relevant when // GetCachedImage is true. BinaryPath string + + // MagicDirBase is the path to the directory where all envbuilder files should be + // stored. By default, this is set to `/.envbuilder`. This is intentionally + // excluded from the CLI options. + MagicDirBase string } const envPrefix = "ENVBUILDER_" @@ -456,10 +460,10 @@ func (o *Options) CLI() serpent.OptionSet { "working on the same repository.", }, { - Flag: "remote-repo-dir", - Env: WithEnvPrefix("REMOTE_REPO_DIR"), - Value: serpent.StringOf(&o.RemoteRepoDir), - Default: constants.MagicRemoteRepoDir, + Flag: "remote-repo-dir", + Env: WithEnvPrefix("REMOTE_REPO_DIR"), + Value: serpent.StringOf(&o.RemoteRepoDir), + // Default: magicdir.Default.Join("repo"), // TODO: reinstate once legacy opts are removed. Hidden: true, Description: "Specify the destination directory for the cloned repo when using remote repo build mode.", }, From 3d120082eaad5df855bfa9b830f269333011803f Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 10 Sep 2024 09:01:52 +0100 Subject: [PATCH 45/81] fix: set DOCKER_CONFIG dynamically (#336) (cherry picked from commit 4547249277ac7e27083e9a855fad4603c5c2769e) --- envbuilder.go | 9 ++++++++- scripts/Dockerfile | 2 -- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index ac64beea..934c51e3 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -1475,7 +1475,7 @@ func initDockerConfigJSON(logf log.Func, magicDir magicdir.MagicDir, dockerConfi if dockerConfigBase64 == "" { return noop, nil } - cfgPath := filepath.Join(magicDir.Path(), "config.json") + cfgPath := magicDir.Join("config.json") decoded, err := base64.StdEncoding.DecodeString(dockerConfigBase64) if err != nil { return noop, fmt.Errorf("decode docker config: %w", err) @@ -1497,9 +1497,16 @@ func initDockerConfigJSON(logf log.Func, magicDir magicdir.MagicDir, dockerConfi return noop, fmt.Errorf("write docker config: %w", err) } logf(log.LevelInfo, "Wrote Docker config JSON to %s", cfgPath) + oldDockerConfig := os.Getenv("DOCKER_CONFIG") + _ = os.Setenv("DOCKER_CONFIG", magicDir.Path()) + newDockerConfig := os.Getenv("DOCKER_CONFIG") + logf(log.LevelInfo, "Set DOCKER_CONFIG to %s", newDockerConfig) cleanup := func() error { var cleanupErr error cleanupOnce.Do(func() { + // Restore the old DOCKER_CONFIG value. + os.Setenv("DOCKER_CONFIG", oldDockerConfig) + logf(log.LevelInfo, "Restored DOCKER_CONFIG to %s", oldDockerConfig) // Remove the Docker config secret file! if cleanupErr = os.Remove(cfgPath); err != nil { if !errors.Is(err, fs.ErrNotExist) { diff --git a/scripts/Dockerfile b/scripts/Dockerfile index b8198a1d..6259407b 100644 --- a/scripts/Dockerfile +++ b/scripts/Dockerfile @@ -4,7 +4,5 @@ ARG TARGETARCH COPY envbuilder-${TARGETARCH} /.envbuilder/bin/envbuilder ENV KANIKO_DIR /.envbuilder -# Kaniko looks for the Docker config at $DOCKER_CONFIG/config.json -ENV DOCKER_CONFIG /.envbuilder ENTRYPOINT ["/.envbuilder/bin/envbuilder"] From c31fd00a9d3092f686464334e8150b7607283632 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Tue, 10 Sep 2024 11:28:23 +0300 Subject: [PATCH 46/81] chore: remove remote-repo-dir option (#340) (cherry picked from commit b7781d802f885f87f9bc42adce2ba53b586f94d1) --- envbuilder.go | 4 ++-- options/defaults.go | 3 --- options/defaults_test.go | 1 - options/options.go | 12 ------------ 4 files changed, 2 insertions(+), 18 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 934c51e3..07b34807 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -148,7 +148,7 @@ func Run(ctx context.Context, opts options.Options) error { if err != nil { return fmt.Errorf("git clone options: %w", err) } - cloneOpts.Path = opts.RemoteRepoDir + cloneOpts.Path = magicDir.Join("repo") endStage := startStage("📦 Remote repo build mode enabled, cloning %s to %s for build context...", newColor(color.FgCyan).Sprintf(opts.GitURL), @@ -927,7 +927,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) if err != nil { return nil, fmt.Errorf("git clone options: %w", err) } - cloneOpts.Path = opts.RemoteRepoDir + cloneOpts.Path = magicDir.Join("repo") endStage := startStage("📦 Remote repo build mode enabled, cloning %s to %s for build context...", newColor(color.FgCyan).Sprintf(opts.GitURL), diff --git a/options/defaults.go b/options/defaults.go index df3d436c..220480d8 100644 --- a/options/defaults.go +++ b/options/defaults.go @@ -59,9 +59,6 @@ func (o *Options) SetDefaults() { if o.WorkspaceFolder == "" { o.WorkspaceFolder = DefaultWorkspaceFolder(o.GitURL) } - if o.RemoteRepoDir == "" { - o.RemoteRepoDir = magicdir.Default.Join("repo") - } if o.BinaryPath == "" { o.BinaryPath = "/.envbuilder/bin/envbuilder" } diff --git a/options/defaults_test.go b/options/defaults_test.go index 8c9946f6..4387c084 100644 --- a/options/defaults_test.go +++ b/options/defaults_test.go @@ -85,7 +85,6 @@ func TestOptions_SetDefaults(t *testing.T) { GitURL: "", WorkspaceFolder: options.EmptyWorkspaceDir, MagicDirBase: "/.envbuilder", - RemoteRepoDir: "/.envbuilder/repo", BinaryPath: "/.envbuilder/bin/envbuilder", } diff --git a/options/options.go b/options/options.go index a0058cd3..5b2586c7 100644 --- a/options/options.go +++ b/options/options.go @@ -157,10 +157,6 @@ type Options struct { // working on the same repository. RemoteRepoBuildMode bool - // RemoteRepoDir is the destination directory for the cloned repo when using - // remote repo build mode. - RemoteRepoDir string - // BinaryPath is the path to the local envbuilder binary when // attempting to probe the build cache. This is only relevant when // GetCachedImage is true. @@ -459,14 +455,6 @@ func (o *Options) CLI() serpent.OptionSet { "be used to improving cache utilization when multiple users are building " + "working on the same repository.", }, - { - Flag: "remote-repo-dir", - Env: WithEnvPrefix("REMOTE_REPO_DIR"), - Value: serpent.StringOf(&o.RemoteRepoDir), - // Default: magicdir.Default.Join("repo"), // TODO: reinstate once legacy opts are removed. - Hidden: true, - Description: "Specify the destination directory for the cloned repo when using remote repo build mode.", - }, { Flag: "verbose", Env: WithEnvPrefix("VERBOSE"), From 4d0cc7885bd22da787df7c2e00e12f093a5c7f15 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 12 Sep 2024 13:24:59 +0300 Subject: [PATCH 47/81] fix(devcontainer): parse user in multi-stage builds (#343) Co-authored-by: Cian Johnston (cherry picked from commit df8ea67455aa3011d08a28f0f372ecad078a0cad) --- devcontainer/devcontainer.go | 94 +++++++++++++--- devcontainer/devcontainer_test.go | 171 +++++++++++++++++++++++++----- integration/integration_test.go | 57 ++++++++++ 3 files changed, 281 insertions(+), 41 deletions(-) diff --git a/devcontainer/devcontainer.go b/devcontainer/devcontainer.go index 7ac8d26d..6135c0ef 100644 --- a/devcontainer/devcontainer.go +++ b/devcontainer/devcontainer.go @@ -15,6 +15,8 @@ import ( "github.com/go-git/go-billy/v5" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/moby/buildkit/frontend/dockerfile/instructions" + "github.com/moby/buildkit/frontend/dockerfile/parser" "github.com/moby/buildkit/frontend/dockerfile/shell" "github.com/tailscale/hujson" ) @@ -202,16 +204,9 @@ func (s *Spec) Compile(fs billy.Filesystem, devcontainerDir, scratchDir string, // We should make a best-effort attempt to find the user. // Features must be executed as root, so we need to swap back // to the running user afterwards. - params.User = UserFromDockerfile(params.DockerfileContent) - } - if params.User == "" { - imageRef, err := ImageFromDockerfile(params.DockerfileContent) + params.User, err = UserFromDockerfile(params.DockerfileContent) if err != nil { - return nil, fmt.Errorf("parse image from dockerfile: %w", err) - } - params.User, err = UserFromImage(imageRef) - if err != nil { - return nil, fmt.Errorf("get user from image: %w", err) + return nil, fmt.Errorf("user from dockerfile: %w", err) } } remoteUser := s.RemoteUser @@ -313,17 +308,82 @@ func (s *Spec) compileFeatures(fs billy.Filesystem, devcontainerDir, scratchDir // UserFromDockerfile inspects the contents of a provided Dockerfile // and returns the user that will be used to run the container. -func UserFromDockerfile(dockerfileContent string) string { - lines := strings.Split(dockerfileContent, "\n") - // Iterate over lines in reverse - for i := len(lines) - 1; i >= 0; i-- { - line := lines[i] - if !strings.HasPrefix(line, "USER ") { +func UserFromDockerfile(dockerfileContent string) (user string, err error) { + res, err := parser.Parse(strings.NewReader(dockerfileContent)) + if err != nil { + return "", fmt.Errorf("parse dockerfile: %w", err) + } + + // Parse stages and user commands to determine the relevant user + // from the final stage. + var ( + stages []*instructions.Stage + stageNames = make(map[string]*instructions.Stage) + stageUser = make(map[*instructions.Stage]*instructions.UserCommand) + currentStage *instructions.Stage + ) + for _, child := range res.AST.Children { + inst, err := instructions.ParseInstruction(child) + if err != nil { + return "", fmt.Errorf("parse instruction: %w", err) + } + + switch i := inst.(type) { + case *instructions.Stage: + stages = append(stages, i) + if i.Name != "" { + stageNames[i.Name] = i + } + currentStage = i + case *instructions.UserCommand: + if currentStage == nil { + continue + } + stageUser[currentStage] = i + } + } + + // Iterate over stages in bottom-up order to find the user, + // skipping any stages not referenced by the final stage. + lookupStage := stages[len(stages)-1] + for i := len(stages) - 1; i >= 0; i-- { + stage := stages[i] + if stage != lookupStage { continue } - return strings.TrimSpace(strings.TrimPrefix(line, "USER ")) + + if user, ok := stageUser[stage]; ok { + return user.User, nil + } + + // If we reach the scratch stage, we can't determine the user. + if stage.BaseName == "scratch" { + return "", nil + } + + // Check if this FROM references another stage. + if stage.BaseName != "" { + var ok bool + lookupStage, ok = stageNames[stage.BaseName] + if ok { + continue + } + } + + // If we can't find a user command, try to find the user from + // the image. + ref, err := name.ParseReference(strings.TrimSpace(stage.BaseName)) + if err != nil { + return "", fmt.Errorf("parse image ref %q: %w", stage.BaseName, err) + } + user, err := UserFromImage(ref) + if err != nil { + return "", fmt.Errorf("user from image %s: %w", ref.Name(), err) + } + return user, nil } - return "" + + return "", nil } // ImageFromDockerfile inspects the contents of a provided Dockerfile diff --git a/devcontainer/devcontainer_test.go b/devcontainer/devcontainer_test.go index c18c6b73..923680b9 100644 --- a/devcontainer/devcontainer_test.go +++ b/devcontainer/devcontainer_test.go @@ -190,12 +190,6 @@ func TestCompileDevContainer(t *testing.T) { }) } -func TestUserFromDockerfile(t *testing.T) { - t.Parallel() - user := devcontainer.UserFromDockerfile("FROM ubuntu\nUSER kyle") - require.Equal(t, "kyle", user) -} - func TestImageFromDockerfile(t *testing.T) { t.Parallel() for _, tc := range []struct { @@ -224,27 +218,156 @@ func TestImageFromDockerfile(t *testing.T) { } } -func TestUserFromImage(t *testing.T) { +func TestUserFrom(t *testing.T) { t.Parallel() - registry := registrytest.New(t) - image, err := partial.UncompressedToImage(emptyImage{configFile: &v1.ConfigFile{ - Config: v1.Config{ - User: "example", - }, - }}) - require.NoError(t, err) - parsed, err := url.Parse(registry) - require.NoError(t, err) - parsed.Path = "coder/test:latest" - ref, err := name.ParseReference(strings.TrimPrefix(parsed.String(), "http://")) - require.NoError(t, err) - err = remote.Write(ref, image) - require.NoError(t, err) + t.Run("Image", func(t *testing.T) { + t.Parallel() + registry := registrytest.New(t) + image, err := partial.UncompressedToImage(emptyImage{configFile: &v1.ConfigFile{ + Config: v1.Config{ + User: "example", + }, + }}) + require.NoError(t, err) - user, err := devcontainer.UserFromImage(ref) - require.NoError(t, err) - require.Equal(t, "example", user) + parsed, err := url.Parse(registry) + require.NoError(t, err) + parsed.Path = "coder/test:latest" + ref, err := name.ParseReference(strings.TrimPrefix(parsed.String(), "http://")) + require.NoError(t, err) + err = remote.Write(ref, image) + require.NoError(t, err) + + user, err := devcontainer.UserFromImage(ref) + require.NoError(t, err) + require.Equal(t, "example", user) + }) + + t.Run("Dockerfile", func(t *testing.T) { + t.Parallel() + tests := []struct { + name string + content string + user string + }{ + { + name: "Empty", + content: "FROM scratch", + user: "", + }, + { + name: "User", + content: "FROM scratch\nUSER kyle", + user: "kyle", + }, + { + name: "Env with default", + content: "FROM scratch\nENV MYUSER=maf\nUSER ${MYUSER}", + user: "${MYUSER}", // This should be "maf" but the current implementation doesn't support this. + }, + { + name: "Env var with default", + content: "FROM scratch\nUSER ${MYUSER:-maf}", + user: "${MYUSER:-maf}", // This should be "maf" but the current implementation doesn't support this. + }, + { + name: "Arg", + content: "FROM scratch\nARG MYUSER\nUSER ${MYUSER}", + user: "${MYUSER}", // This should be "" or populated but the current implementation doesn't support this. + }, + { + name: "Arg with default", + content: "FROM scratch\nARG MYUSER=maf\nUSER ${MYUSER}", + user: "${MYUSER}", // This should be "maf" but the current implementation doesn't support this. + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + user, err := devcontainer.UserFromDockerfile(tt.content) + require.NoError(t, err) + require.Equal(t, tt.user, user) + }) + } + }) + + t.Run("Multi-stage", func(t *testing.T) { + t.Parallel() + + registry := registrytest.New(t) + for tag, user := range map[string]string{ + "one": "maf", + "two": "fam", + } { + image, err := partial.UncompressedToImage(emptyImage{configFile: &v1.ConfigFile{ + Config: v1.Config{ + User: user, + }, + }}) + require.NoError(t, err) + parsed, err := url.Parse(registry) + require.NoError(t, err) + parsed.Path = "coder/test:" + tag + ref, err := name.ParseReference(strings.TrimPrefix(parsed.String(), "http://")) + fmt.Println(ref) + require.NoError(t, err) + err = remote.Write(ref, image) + require.NoError(t, err) + } + + tests := []struct { + name string + images map[string]string + content string + user string + }{ + { + name: "Single", + content: "FROM coder/test:one", + user: "maf", + }, + { + name: "Multi", + content: "FROM ubuntu AS u\nFROM coder/test:two", + user: "fam", + }, + { + name: "Multi-2", + content: "FROM coder/test:two AS two\nUSER maffam\nFROM coder/test:one AS one", + user: "maf", + }, + { + name: "Multi-3", + content: "FROM coder/test:two AS two\nFROM coder/test:one AS one\nUSER fammaf", + user: "fammaf", + }, + { + name: "Multi-4", + content: `FROM ubuntu AS a +USER root +RUN useradd --create-home pickme +USER pickme +FROM a AS other +USER root +RUN useradd --create-home notme +USER notme +FROM a`, + user: "pickme", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + content := strings.ReplaceAll(tt.content, "coder/test", strings.TrimPrefix(registry, "http://")+"/coder/test") + + user, err := devcontainer.UserFromDockerfile(content) + require.NoError(t, err) + require.Equal(t, tt.user, user) + }) + } + }) } type emptyImage struct { diff --git a/integration/integration_test.go b/integration/integration_test.go index e0e012ba..29b5e8a7 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -102,6 +102,63 @@ func TestInitScriptInitCommand(t *testing.T) { require.NoError(t, ctx.Err(), "init script did not execute for legacy env vars") } +func TestDanglingBuildStage(t *testing.T) { + t.Parallel() + + // Ensures that a Git repository with a devcontainer.json is cloned and built. + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ + "devcontainer.json": `{ + "name": "Test", + "build": { + "dockerfile": "Dockerfile" + }, + }`, + "Dockerfile": fmt.Sprintf(`FROM %s as a +RUN date > /root/date.txt`, testImageUbuntu), + }, + }) + ctr, err := runEnvbuilder(t, runOpts{env: []string{ + envbuilderEnv("GIT_URL", srv.URL), + }}) + require.NoError(t, err) + + output := execContainer(t, ctr, "cat /date.txt") + require.NotEmpty(t, strings.TrimSpace(output)) +} + +func TestUserFromMultistage(t *testing.T) { + t.Parallel() + + // Ensures that a Git repository with a devcontainer.json is cloned and built. + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ + "devcontainer.json": `{ + "name": "Test", + "build": { + "dockerfile": "Dockerfile" + }, + }`, + "Dockerfile": fmt.Sprintf(`FROM %s AS a +USER root +RUN useradd --create-home pickme +USER pickme +FROM a AS other +USER root +RUN useradd --create-home notme +USER notme +FROM a AS b`, testImageUbuntu), + }, + }) + ctr, err := runEnvbuilder(t, runOpts{env: []string{ + envbuilderEnv("GIT_URL", srv.URL), + }}) + require.NoError(t, err) + + output := execContainer(t, ctr, "ps aux | awk '/^pickme / {print $1}' | sort -u") + require.Equal(t, "pickme", strings.TrimSpace(output)) +} + func TestUidGid(t *testing.T) { t.Parallel() t.Run("MultiStage", func(t *testing.T) { From 7d9abfc654ec203bf55656fdaa50d20f0aa74d5d Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 12 Sep 2024 14:43:35 +0100 Subject: [PATCH 48/81] feat(git): log parsed gitURL and warn if local (#345) (cherry picked from commit e14b95257b64c396daa40f2c5a8ad40b49efb902) --- envbuilder.go | 56 +++++++++++++++++---------------- git/git.go | 68 ++++++++++++++++++++++++++-------------- git/git_test.go | 83 ++++++++++++++++++++++++++----------------------- 3 files changed, 118 insertions(+), 89 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 07b34807..3df35622 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -112,22 +112,25 @@ func Run(ctx context.Context, opts options.Options) error { var fallbackErr error var cloned bool if opts.GitURL != "" { - cloneOpts, err := git.CloneOptionsFromOptions(opts) - if err != nil { - return fmt.Errorf("git clone options: %w", err) - } - endStage := startStage("📦 Cloning %s to %s...", newColor(color.FgCyan).Sprintf(opts.GitURL), - newColor(color.FgCyan).Sprintf(cloneOpts.Path), + newColor(color.FgCyan).Sprintf(opts.WorkspaceFolder), ) - stageNum := stageNumber - w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNum, line) }) + logStage := func(format string, args ...any) { + opts.Logger(log.LevelInfo, "#%d: %s", stageNum, fmt.Sprintf(format, args...)) + } + + cloneOpts, err := git.CloneOptionsFromOptions(logStage, opts) + if err != nil { + return fmt.Errorf("git clone options: %w", err) + } + + w := git.ProgressWriter(logStage) defer w.Close() cloneOpts.Progress = w - cloned, fallbackErr = git.CloneRepo(ctx, cloneOpts) + cloned, fallbackErr = git.CloneRepo(ctx, logStage, cloneOpts) if fallbackErr == nil { if cloned { endStage("📦 Cloned repository!") @@ -144,7 +147,7 @@ func Run(ctx context.Context, opts options.Options) error { // Always clone the repo in remote repo build mode into a location that // we control that isn't affected by the users changes. if opts.RemoteRepoBuildMode { - cloneOpts, err := git.CloneOptionsFromOptions(opts) + cloneOpts, err := git.CloneOptionsFromOptions(logStage, opts) if err != nil { return fmt.Errorf("git clone options: %w", err) } @@ -155,12 +158,11 @@ func Run(ctx context.Context, opts options.Options) error { newColor(color.FgCyan).Sprintf(cloneOpts.Path), ) - stageNum := stageNumber - w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNum, line) }) + w := git.ProgressWriter(logStage) defer w.Close() cloneOpts.Progress = w - fallbackErr = git.ShallowCloneRepo(ctx, cloneOpts) + fallbackErr = git.ShallowCloneRepo(ctx, logStage, cloneOpts) if fallbackErr == nil { endStage("📦 Cloned repository!") buildTimeWorkspaceFolder = cloneOpts.Path @@ -891,25 +893,28 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) var fallbackErr error var cloned bool if opts.GitURL != "" { + endStage := startStage("📦 Cloning %s to %s...", + newColor(color.FgCyan).Sprintf(opts.GitURL), + newColor(color.FgCyan).Sprintf(opts.WorkspaceFolder), + ) + stageNum := stageNumber + logStage := func(format string, args ...any) { + opts.Logger(log.LevelInfo, "#%d: %s", stageNum, fmt.Sprintf(format, args...)) + } + // In cache probe mode we should only attempt to clone the full // repository if remote repo build mode isn't enabled. if !opts.RemoteRepoBuildMode { - cloneOpts, err := git.CloneOptionsFromOptions(opts) + cloneOpts, err := git.CloneOptionsFromOptions(logStage, opts) if err != nil { return nil, fmt.Errorf("git clone options: %w", err) } - endStage := startStage("📦 Cloning %s to %s...", - newColor(color.FgCyan).Sprintf(opts.GitURL), - newColor(color.FgCyan).Sprintf(cloneOpts.Path), - ) - - stageNum := stageNumber - w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNum, line) }) + w := git.ProgressWriter(logStage) defer w.Close() cloneOpts.Progress = w - cloned, fallbackErr = git.CloneRepo(ctx, cloneOpts) + cloned, fallbackErr = git.CloneRepo(ctx, logStage, cloneOpts) if fallbackErr == nil { if cloned { endStage("📦 Cloned repository!") @@ -923,7 +928,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) _ = w.Close() } else { - cloneOpts, err := git.CloneOptionsFromOptions(opts) + cloneOpts, err := git.CloneOptionsFromOptions(logStage, opts) if err != nil { return nil, fmt.Errorf("git clone options: %w", err) } @@ -934,12 +939,11 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) newColor(color.FgCyan).Sprintf(cloneOpts.Path), ) - stageNum := stageNumber - w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNum, line) }) + w := git.ProgressWriter(logStage) defer w.Close() cloneOpts.Progress = w - fallbackErr = git.ShallowCloneRepo(ctx, cloneOpts) + fallbackErr = git.ShallowCloneRepo(ctx, logStage, cloneOpts) if fallbackErr == nil { endStage("📦 Cloned repository!") buildTimeWorkspaceFolder = cloneOpts.Path diff --git a/git/git.go b/git/git.go index 1404f089..7d132c3a 100644 --- a/git/git.go +++ b/git/git.go @@ -12,7 +12,6 @@ import ( "github.com/coder/envbuilder/options" giturls "github.com/chainguard-dev/git-urls" - "github.com/coder/envbuilder/log" "github.com/go-git/go-billy/v5" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" @@ -47,11 +46,12 @@ type CloneRepoOptions struct { // be cloned again. // // The bool returned states whether the repository was cloned or not. -func CloneRepo(ctx context.Context, opts CloneRepoOptions) (bool, error) { +func CloneRepo(ctx context.Context, logf func(string, ...any), opts CloneRepoOptions) (bool, error) { parsed, err := giturls.Parse(opts.RepoURL) if err != nil { return false, fmt.Errorf("parse url %q: %w", opts.RepoURL, err) } + logf("Parsed Git URL as %q", parsed.Redacted()) if parsed.Hostname() == "dev.azure.com" { // Azure DevOps requires capabilities multi_ack / multi_ack_detailed, // which are not fully implemented and by default are included in @@ -73,6 +73,7 @@ func CloneRepo(ctx context.Context, opts CloneRepoOptions) (bool, error) { transport.UnsupportedCapabilities = []capability.Capability{ capability.ThinPack, } + logf("Workaround for Azure DevOps: marking thin-pack as unsupported") } err = opts.Storage.MkdirAll(opts.Path, 0o755) @@ -131,7 +132,7 @@ func CloneRepo(ctx context.Context, opts CloneRepoOptions) (bool, error) { // clone will not be performed. // // The bool returned states whether the repository was cloned or not. -func ShallowCloneRepo(ctx context.Context, opts CloneRepoOptions) error { +func ShallowCloneRepo(ctx context.Context, logf func(string, ...any), opts CloneRepoOptions) error { opts.Depth = 1 opts.SingleBranch = true @@ -150,7 +151,7 @@ func ShallowCloneRepo(ctx context.Context, opts CloneRepoOptions) error { } } - cloned, err := CloneRepo(ctx, opts) + cloned, err := CloneRepo(ctx, logf, opts) if err != nil { return err } @@ -182,14 +183,14 @@ func ReadPrivateKey(path string) (gossh.Signer, error) { // LogHostKeyCallback is a HostKeyCallback that just logs host keys // and does nothing else. -func LogHostKeyCallback(logger log.Func) gossh.HostKeyCallback { +func LogHostKeyCallback(logger func(string, ...any)) gossh.HostKeyCallback { return func(hostname string, remote net.Addr, key gossh.PublicKey) error { var sb strings.Builder _ = knownhosts.WriteKnownHost(&sb, hostname, remote, key) // skeema/knownhosts uses a fake public key to determine the host key // algorithms. Ignore this one. if s := sb.String(); !strings.Contains(s, "fake-public-key ZmFrZSBwdWJsaWMga2V5") { - logger(log.LevelInfo, "#1: 🔑 Got host key: %s", strings.TrimSpace(s)) + logger("🔑 Got host key: %s", strings.TrimSpace(s)) } return nil } @@ -203,6 +204,8 @@ func LogHostKeyCallback(logger log.Func) gossh.HostKeyCallback { // | https?://host.tld/repo | Not Set | Set | HTTP Basic | // | https?://host.tld/repo | Set | Not Set | HTTP Basic | // | https?://host.tld/repo | Set | Set | HTTP Basic | +// | file://path/to/repo | - | - | None | +// | path/to/repo | - | - | None | // | All other formats | - | - | SSH | // // For SSH authentication, the default username is "git" but will honour @@ -214,27 +217,42 @@ func LogHostKeyCallback(logger log.Func) gossh.HostKeyCallback { // If SSH_KNOWN_HOSTS is not set, the SSH auth method will be configured // to accept and log all host keys. Otherwise, host key checking will be // performed as usual. -func SetupRepoAuth(options *options.Options) transport.AuthMethod { +func SetupRepoAuth(logf func(string, ...any), options *options.Options) transport.AuthMethod { if options.GitURL == "" { - options.Logger(log.LevelInfo, "#1: ❔ No Git URL supplied!") + logf("❔ No Git URL supplied!") return nil } - if strings.HasPrefix(options.GitURL, "http://") || strings.HasPrefix(options.GitURL, "https://") { + parsedURL, err := giturls.Parse(options.GitURL) + if err != nil { + logf("❌ Failed to parse Git URL: %s", err.Error()) + return nil + } + + if parsedURL.Scheme == "http" || parsedURL.Scheme == "https" { // Special case: no auth if options.GitUsername == "" && options.GitPassword == "" { - options.Logger(log.LevelInfo, "#1: 👤 Using no authentication!") + logf("👤 Using no authentication!") return nil } // Basic Auth // NOTE: we previously inserted the credentials into the repo URL. // This was removed in https://github.com/coder/envbuilder/pull/141 - options.Logger(log.LevelInfo, "#1: 🔒 Using HTTP basic authentication!") + logf("🔒 Using HTTP basic authentication!") return &githttp.BasicAuth{ Username: options.GitUsername, Password: options.GitPassword, } } + if parsedURL.Scheme == "file" { + // go-git will try to fallback to using the `git` command for local + // filesystem clones. However, it's more likely than not that the + // `git` command is not present in the container image. Log a warning + // but continue. Also, no auth. + logf("🚧 Using local filesystem clone! This requires the git executable to be present!") + return nil + } + // Generally git clones over SSH use the 'git' user, but respect // GIT_USERNAME if set. if options.GitUsername == "" { @@ -242,30 +260,30 @@ func SetupRepoAuth(options *options.Options) transport.AuthMethod { } // Assume SSH auth for all other formats. - options.Logger(log.LevelInfo, "#1: 🔑 Using SSH authentication!") + logf("🔑 Using SSH authentication!") var signer ssh.Signer if options.GitSSHPrivateKeyPath != "" { s, err := ReadPrivateKey(options.GitSSHPrivateKeyPath) if err != nil { - options.Logger(log.LevelError, "#1: ❌ Failed to read private key from %s: %s", options.GitSSHPrivateKeyPath, err.Error()) + logf("❌ Failed to read private key from %s: %s", options.GitSSHPrivateKeyPath, err.Error()) } else { - options.Logger(log.LevelInfo, "#1: 🔑 Using %s key!", s.PublicKey().Type()) + logf("🔑 Using %s key!", s.PublicKey().Type()) signer = s } } // If no SSH key set, fall back to agent auth. if signer == nil { - options.Logger(log.LevelError, "#1: 🔑 No SSH key found, falling back to agent!") + logf("🔑 No SSH key found, falling back to agent!") auth, err := gitssh.NewSSHAgentAuth(options.GitUsername) if err != nil { - options.Logger(log.LevelError, "#1: ❌ Failed to connect to SSH agent: %s", err.Error()) + logf("❌ Failed to connect to SSH agent: " + err.Error()) return nil // nothing else we can do } if os.Getenv("SSH_KNOWN_HOSTS") == "" { - options.Logger(log.LevelWarn, "#1: 🔓 SSH_KNOWN_HOSTS not set, accepting all host keys!") - auth.HostKeyCallback = LogHostKeyCallback(options.Logger) + logf("🔓 SSH_KNOWN_HOSTS not set, accepting all host keys!") + auth.HostKeyCallback = LogHostKeyCallback(logf) } return auth } @@ -283,19 +301,20 @@ func SetupRepoAuth(options *options.Options) transport.AuthMethod { // Duplicated code due to Go's type system. if os.Getenv("SSH_KNOWN_HOSTS") == "" { - options.Logger(log.LevelWarn, "#1: 🔓 SSH_KNOWN_HOSTS not set, accepting all host keys!") - auth.HostKeyCallback = LogHostKeyCallback(options.Logger) + logf("🔓 SSH_KNOWN_HOSTS not set, accepting all host keys!") + auth.HostKeyCallback = LogHostKeyCallback(logf) } return auth } -func CloneOptionsFromOptions(options options.Options) (CloneRepoOptions, error) { +func CloneOptionsFromOptions(logf func(string, ...any), options options.Options) (CloneRepoOptions, error) { caBundle, err := options.CABundle() if err != nil { return CloneRepoOptions{}, err } cloneOpts := CloneRepoOptions{ + RepoURL: options.GitURL, Path: options.WorkspaceFolder, Storage: options.Filesystem, Insecure: options.Insecure, @@ -304,13 +323,12 @@ func CloneOptionsFromOptions(options options.Options) (CloneRepoOptions, error) CABundle: caBundle, } - cloneOpts.RepoAuth = SetupRepoAuth(&options) + cloneOpts.RepoAuth = SetupRepoAuth(logf, &options) if options.GitHTTPProxyURL != "" { cloneOpts.ProxyOptions = transport.ProxyOptions{ URL: options.GitHTTPProxyURL, } } - cloneOpts.RepoURL = options.GitURL return cloneOpts, nil } @@ -331,7 +349,7 @@ func (w *progressWriter) Close() error { return err2 } -func ProgressWriter(write func(line string)) io.WriteCloser { +func ProgressWriter(write func(line string, args ...any)) io.WriteCloser { reader, writer := io.Pipe() done := make(chan struct{}) go func() { @@ -347,6 +365,8 @@ func ProgressWriter(write func(line string)) io.WriteCloser { if line == "" { continue } + // Escape % signs so that they don't get interpreted as format specifiers + line = strings.Replace(line, "%", "%%", -1) write(strings.TrimSpace(line)) } } diff --git a/git/git_test.go b/git/git_test.go index 14656886..e7a58f90 100644 --- a/git/git_test.go +++ b/git/git_test.go @@ -13,12 +13,10 @@ import ( "testing" "github.com/coder/envbuilder/git" - "github.com/coder/envbuilder/options" - - "github.com/coder/envbuilder/log" "github.com/coder/envbuilder/testutil/gittest" "github.com/coder/envbuilder/testutil/mwtest" + "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-billy/v5/osfs" @@ -91,7 +89,7 @@ func TestCloneRepo(t *testing.T) { clientFS := memfs.New() // A repo already exists! _ = gittest.NewRepo(t, clientFS) - cloned, err := git.CloneRepo(context.Background(), git.CloneRepoOptions{ + cloned, err := git.CloneRepo(context.Background(), t.Logf, git.CloneRepoOptions{ Path: "/", RepoURL: srv.URL, Storage: clientFS, @@ -109,7 +107,7 @@ func TestCloneRepo(t *testing.T) { srv := httptest.NewServer(authMW(gittest.NewServer(srvFS))) clientFS := memfs.New() - cloned, err := git.CloneRepo(context.Background(), git.CloneRepoOptions{ + cloned, err := git.CloneRepo(context.Background(), t.Logf, git.CloneRepoOptions{ Path: "/workspace", RepoURL: srv.URL, Storage: clientFS, @@ -146,7 +144,7 @@ func TestCloneRepo(t *testing.T) { authURL.User = url.UserPassword(tc.username, tc.password) clientFS := memfs.New() - cloned, err := git.CloneRepo(context.Background(), git.CloneRepoOptions{ + cloned, err := git.CloneRepo(context.Background(), t.Logf, git.CloneRepoOptions{ Path: "/workspace", RepoURL: authURL.String(), Storage: clientFS, @@ -191,7 +189,7 @@ func TestShallowCloneRepo(t *testing.T) { require.NoError(t, err) require.NoError(t, f.Close()) - err = git.ShallowCloneRepo(context.Background(), git.CloneRepoOptions{ + err = git.ShallowCloneRepo(context.Background(), t.Logf, git.CloneRepoOptions{ Path: "/repo", RepoURL: srv.URL, Storage: clientFS, @@ -219,7 +217,7 @@ func TestShallowCloneRepo(t *testing.T) { clientFS := memfs.New() - err := git.ShallowCloneRepo(context.Background(), git.CloneRepoOptions{ + err := git.ShallowCloneRepo(context.Background(), t.Logf, git.CloneRepoOptions{ Path: "/repo", RepoURL: srv.URL, Storage: clientFS, @@ -252,7 +250,7 @@ func TestCloneRepoSSH(t *testing.T) { gitURL := tr.String() clientFS := memfs.New() - cloned, err := git.CloneRepo(context.Background(), git.CloneRepoOptions{ + cloned, err := git.CloneRepo(context.Background(), t.Logf, git.CloneRepoOptions{ Path: "/workspace", RepoURL: gitURL, Storage: clientFS, @@ -284,7 +282,7 @@ func TestCloneRepoSSH(t *testing.T) { clientFS := memfs.New() anotherKey := randKeygen(t) - cloned, err := git.CloneRepo(context.Background(), git.CloneRepoOptions{ + cloned, err := git.CloneRepo(context.Background(), t.Logf, git.CloneRepoOptions{ Path: "/workspace", RepoURL: gitURL, Storage: clientFS, @@ -314,7 +312,7 @@ func TestCloneRepoSSH(t *testing.T) { gitURL := tr.String() clientFS := memfs.New() - cloned, err := git.CloneRepo(context.Background(), git.CloneRepoOptions{ + cloned, err := git.CloneRepo(context.Background(), t.Logf, git.CloneRepoOptions{ Path: "/workspace", RepoURL: gitURL, Storage: clientFS, @@ -335,19 +333,16 @@ func TestCloneRepoSSH(t *testing.T) { func TestSetupRepoAuth(t *testing.T) { t.Setenv("SSH_AUTH_SOCK", "") t.Run("Empty", func(t *testing.T) { - opts := &options.Options{ - Logger: testLog(t), - } - auth := git.SetupRepoAuth(opts) + opts := &options.Options{} + auth := git.SetupRepoAuth(t.Logf, opts) require.Nil(t, auth) }) t.Run("HTTP/NoAuth", func(t *testing.T) { opts := &options.Options{ GitURL: "http://host.tld/repo", - Logger: testLog(t), } - auth := git.SetupRepoAuth(opts) + auth := git.SetupRepoAuth(t.Logf, opts) require.Nil(t, auth) }) @@ -356,9 +351,8 @@ func TestSetupRepoAuth(t *testing.T) { GitURL: "http://host.tld/repo", GitUsername: "user", GitPassword: "pass", - Logger: testLog(t), } - auth := git.SetupRepoAuth(opts) + auth := git.SetupRepoAuth(t.Logf, opts) ba, ok := auth.(*githttp.BasicAuth) require.True(t, ok) require.Equal(t, opts.GitUsername, ba.Username) @@ -370,9 +364,8 @@ func TestSetupRepoAuth(t *testing.T) { GitURL: "https://host.tld/repo", GitUsername: "user", GitPassword: "pass", - Logger: testLog(t), } - auth := git.SetupRepoAuth(opts) + auth := git.SetupRepoAuth(t.Logf, opts) ba, ok := auth.(*githttp.BasicAuth) require.True(t, ok) require.Equal(t, opts.GitUsername, ba.Username) @@ -384,9 +377,8 @@ func TestSetupRepoAuth(t *testing.T) { opts := &options.Options{ GitURL: "ssh://host.tld/repo", GitSSHPrivateKeyPath: kPath, - Logger: testLog(t), } - auth := git.SetupRepoAuth(opts) + auth := git.SetupRepoAuth(t.Logf, opts) _, ok := auth.(*gitssh.PublicKeys) require.True(t, ok) }) @@ -396,9 +388,8 @@ func TestSetupRepoAuth(t *testing.T) { opts := &options.Options{ GitURL: "git@host.tld:repo/path", GitSSHPrivateKeyPath: kPath, - Logger: testLog(t), } - auth := git.SetupRepoAuth(opts) + auth := git.SetupRepoAuth(t.Logf, opts) _, ok := auth.(*gitssh.PublicKeys) require.True(t, ok) }) @@ -409,9 +400,8 @@ func TestSetupRepoAuth(t *testing.T) { opts := &options.Options{ GitURL: "git://git@host.tld:repo/path", GitSSHPrivateKeyPath: kPath, - Logger: testLog(t), } - auth := git.SetupRepoAuth(opts) + auth := git.SetupRepoAuth(t.Logf, opts) _, ok := auth.(*gitssh.PublicKeys) require.True(t, ok) }) @@ -422,9 +412,8 @@ func TestSetupRepoAuth(t *testing.T) { GitURL: "host.tld:12345/repo/path", GitSSHPrivateKeyPath: kPath, GitUsername: "user", - Logger: testLog(t), } - auth := git.SetupRepoAuth(opts) + auth := git.SetupRepoAuth(t.Logf, opts) _, ok := auth.(*gitssh.PublicKeys) require.True(t, ok) }) @@ -434,9 +423,8 @@ func TestSetupRepoAuth(t *testing.T) { opts := &options.Options{ GitURL: "ssh://git@host.tld:repo/path", GitSSHPrivateKeyPath: kPath, - Logger: testLog(t), } - auth := git.SetupRepoAuth(opts) + auth := git.SetupRepoAuth(t.Logf, opts) pk, ok := auth.(*gitssh.PublicKeys) require.True(t, ok) require.NotNil(t, pk.Signer) @@ -448,11 +436,34 @@ func TestSetupRepoAuth(t *testing.T) { t.Run("SSH/NoAuthMethods", func(t *testing.T) { opts := &options.Options{ GitURL: "ssh://git@host.tld:repo/path", - Logger: testLog(t), } - auth := git.SetupRepoAuth(opts) + auth := git.SetupRepoAuth(t.Logf, opts) require.Nil(t, auth) // TODO: actually test SSH_AUTH_SOCK }) + + t.Run("NoHostname/RepoOnly", func(t *testing.T) { + opts := &options.Options{ + GitURL: "repo", + } + auth := git.SetupRepoAuth(t.Logf, opts) + require.Nil(t, auth) + }) + + t.Run("NoHostname/Org/Repo", func(t *testing.T) { + opts := &options.Options{ + GitURL: "org/repo", + } + auth := git.SetupRepoAuth(t.Logf, opts) + require.Nil(t, auth) + }) + + t.Run("NoHostname/AbsolutePathish", func(t *testing.T) { + opts := &options.Options{ + GitURL: "/org/repo", + } + auth := git.SetupRepoAuth(t.Logf, opts) + require.Nil(t, auth) + }) } func mustRead(t *testing.T, fs billy.Filesystem, path string) string { @@ -474,12 +485,6 @@ func randKeygen(t *testing.T) gossh.Signer { return signer } -func testLog(t *testing.T) log.Func { - return func(_ log.Level, format string, args ...interface{}) { - t.Logf(format, args...) - } -} - // nolint:gosec // Throw-away key for testing. DO NOT REUSE. var testKey = `-----BEGIN OPENSSH PRIVATE KEY----- b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW From 67fbe78f4b34cfc8b8bf5f257edbcb8d87fad7c4 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Tue, 24 Sep 2024 14:14:02 +0300 Subject: [PATCH 49/81] fix: improve cached image startup and cache features (#353) (cherry picked from commit 08bd99bc5ff919251408486f7b256824e3667891) --- envbuilder.go | 776 +++++++++++++++++--------------- integration/integration_test.go | 521 +++++++++++---------- internal/magicdir/magicdir.go | 5 + 3 files changed, 704 insertions(+), 598 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 3df35622..bc94d89c 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -276,407 +276,425 @@ func Run(ctx context.Context, opts options.Options) error { } } - if buildParams == nil { - // If there isn't a devcontainer.json file in the repository, - // we fallback to whatever the `DefaultImage` is. - var err error - buildParams, err = defaultBuildParams() - if err != nil { - return fmt.Errorf("no Dockerfile or devcontainer.json found: %w", err) + var ( + username string + skippedRebuild bool + ) + if _, err := os.Stat(magicDir.Image()); errors.Is(err, fs.ErrNotExist) { + if buildParams == nil { + // If there isn't a devcontainer.json file in the repository, + // we fallback to whatever the `DefaultImage` is. + var err error + buildParams, err = defaultBuildParams() + if err != nil { + return fmt.Errorf("no Dockerfile or devcontainer.json found: %w", err) + } } - } - lvl := log.LevelInfo - if opts.Verbose { - lvl = log.LevelDebug - } - log.HijackLogrus(lvl, func(entry *logrus.Entry) { - for _, line := range strings.Split(entry.Message, "\r") { - opts.Logger(log.FromLogrus(entry.Level), "#%d: %s", stageNumber, color.HiBlackString(line)) + lvl := log.LevelInfo + if opts.Verbose { + lvl = log.LevelDebug } - }) + log.HijackLogrus(lvl, func(entry *logrus.Entry) { + for _, line := range strings.Split(entry.Message, "\r") { + opts.Logger(log.FromLogrus(entry.Level), "#%d: %s", stageNumber, color.HiBlackString(line)) + } + }) - if opts.LayerCacheDir != "" { - if opts.CacheRepo != "" { - opts.Logger(log.LevelWarn, "Overriding cache repo with local registry...") - } - localRegistry, closeLocalRegistry, err := serveLocalRegistry(ctx, opts.Logger, opts.LayerCacheDir) - if err != nil { - return err + if opts.LayerCacheDir != "" { + if opts.CacheRepo != "" { + opts.Logger(log.LevelWarn, "Overriding cache repo with local registry...") + } + localRegistry, closeLocalRegistry, err := serveLocalRegistry(ctx, opts.Logger, opts.LayerCacheDir) + if err != nil { + return err + } + defer closeLocalRegistry() + opts.CacheRepo = localRegistry } - defer closeLocalRegistry() - opts.CacheRepo = localRegistry - } - // IgnorePaths in the Kaniko opts doesn't properly ignore paths. - // So we add them to the default ignore list. See: - // https://github.com/GoogleContainerTools/kaniko/blob/63be4990ca5a60bdf06ddc4d10aa4eca0c0bc714/cmd/executor/cmd/root.go#L136 - ignorePaths := append([]string{ - magicDir.Path(), - opts.WorkspaceFolder, - // See: https://github.com/coder/envbuilder/issues/37 - "/etc/resolv.conf", - }, opts.IgnorePaths...) + // IgnorePaths in the Kaniko opts doesn't properly ignore paths. + // So we add them to the default ignore list. See: + // https://github.com/GoogleContainerTools/kaniko/blob/63be4990ca5a60bdf06ddc4d10aa4eca0c0bc714/cmd/executor/cmd/root.go#L136 + ignorePaths := append([]string{ + magicDir.Path(), + opts.WorkspaceFolder, + // See: https://github.com/coder/envbuilder/issues/37 + "/etc/resolv.conf", + }, opts.IgnorePaths...) - if opts.LayerCacheDir != "" { - ignorePaths = append(ignorePaths, opts.LayerCacheDir) - } - - for _, ignorePath := range ignorePaths { - util.AddToDefaultIgnoreList(util.IgnoreListEntry{ - Path: ignorePath, - PrefixMatchOnly: false, - AllowedPaths: nil, - }) - } + if opts.LayerCacheDir != "" { + ignorePaths = append(ignorePaths, opts.LayerCacheDir) + } - // In order to allow 'resuming' envbuilder, embed the binary into the image - // if it is being pushed. - // As these files will be owned by root, it is considerate to clean up - // after we're done! - cleanupBuildContext := func() {} - if opts.PushImage { - // Add exceptions in Kaniko's ignorelist for these magic files we add. - if err := util.AddAllowedPathToDefaultIgnoreList(opts.BinaryPath); err != nil { - return fmt.Errorf("add envbuilder binary to ignore list: %w", err) - } - if err := util.AddAllowedPathToDefaultIgnoreList(magicDir.Image()); err != nil { - return fmt.Errorf("add magic image file to ignore list: %w", err) - } - magicTempDir := magicdir.At(buildParams.BuildContext, magicdir.TempDir) - if err := opts.Filesystem.MkdirAll(magicTempDir.Path(), 0o755); err != nil { - return fmt.Errorf("create magic temp dir in build context: %w", err) - } - // Add the magic directives that embed the binary into the built image. - buildParams.DockerfileContent += magicdir.Directives - // Copy the envbuilder binary into the build context. - // External callers will need to specify the path to the desired envbuilder binary. - envbuilderBinDest := filepath.Join(magicTempDir.Path(), "envbuilder") - // Also touch the magic file that signifies the image has been built! - magicImageDest := magicTempDir.Image() - // Clean up after build! - var cleanupOnce sync.Once - cleanupBuildContext = func() { - cleanupOnce.Do(func() { - for _, path := range []string{magicImageDest, envbuilderBinDest, magicTempDir.Path()} { - if err := opts.Filesystem.Remove(path); err != nil { - opts.Logger(log.LevelWarn, "failed to clean up magic temp dir from build context: %w", err) - } - } + for _, ignorePath := range ignorePaths { + util.AddToDefaultIgnoreList(util.IgnoreListEntry{ + Path: ignorePath, + PrefixMatchOnly: false, + AllowedPaths: nil, }) } - defer cleanupBuildContext() - opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, envbuilderBinDest) - if err := copyFile(opts.Filesystem, opts.BinaryPath, envbuilderBinDest, 0o755); err != nil { - return fmt.Errorf("copy envbuilder binary to build context: %w", err) - } + // In order to allow 'resuming' envbuilder, embed the binary into the image + // if it is being pushed. + // As these files will be owned by root, it is considerate to clean up + // after we're done! + cleanupBuildContext := func() {} + if opts.PushImage { + // Add exceptions in Kaniko's ignorelist for these magic files we add. + if err := util.AddAllowedPathToDefaultIgnoreList(opts.BinaryPath); err != nil { + return fmt.Errorf("add envbuilder binary to ignore list: %w", err) + } + if err := util.AddAllowedPathToDefaultIgnoreList(magicDir.Image()); err != nil { + return fmt.Errorf("add magic image file to ignore list: %w", err) + } + if err := util.AddAllowedPathToDefaultIgnoreList(magicDir.Features()); err != nil { + return fmt.Errorf("add features to ignore list: %w", err) + } + magicTempDir := magicdir.At(buildParams.BuildContext, magicdir.TempDir) + if err := opts.Filesystem.MkdirAll(magicTempDir.Path(), 0o755); err != nil { + return fmt.Errorf("create magic temp dir in build context: %w", err) + } + // Add the magic directives that embed the binary into the built image. + buildParams.DockerfileContent += magicdir.Directives + + envbuilderBinDest := filepath.Join(magicTempDir.Path(), "envbuilder") + magicImageDest := magicTempDir.Image() + + // Clean up after build! + var cleanupOnce sync.Once + cleanupBuildContext = func() { + cleanupOnce.Do(func() { + for _, path := range []string{magicImageDest, envbuilderBinDest, magicTempDir.Path()} { + if err := opts.Filesystem.Remove(path); err != nil { + opts.Logger(log.LevelWarn, "failed to clean up magic temp dir from build context: %w", err) + } + } + }) + } + defer cleanupBuildContext() - opts.Logger(log.LevelDebug, "touching magic image file at %q in build context %q", magicImageDest, magicTempDir) - if err := touchFile(opts.Filesystem, magicImageDest, 0o755); err != nil { - return fmt.Errorf("touch magic image file in build context: %w", err) - } - } + // Copy the envbuilder binary into the build context. External callers + // will need to specify the path to the desired envbuilder binary. + opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, envbuilderBinDest) + if err := copyFile(opts.Filesystem, opts.BinaryPath, envbuilderBinDest, 0o755); err != nil { + return fmt.Errorf("copy envbuilder binary to build context: %w", err) + } - // temp move of all ro mounts - tempRemountDest := magicDir.Join("mnt") - // ignorePrefixes is a superset of ignorePaths that we pass to kaniko's - // IgnoreList. - ignorePrefixes := append([]string{"/dev", "/proc", "/sys"}, ignorePaths...) - restoreMounts, err := ebutil.TempRemount(opts.Logger, tempRemountDest, ignorePrefixes...) - defer func() { // restoreMounts should never be nil - if err := restoreMounts(); err != nil { - opts.Logger(log.LevelError, "restore mounts: %s", err.Error()) + // Also write the magic file that signifies the image has been built. + // Since the user in the image is set to root, we also store the user + // in the magic file to be used by envbuilder when the image is run. + opts.Logger(log.LevelDebug, "writing magic image file at %q in build context %q", magicImageDest, magicTempDir) + if err := writeFile(opts.Filesystem, magicImageDest, 0o755, fmt.Sprintf("USER=%s\n", buildParams.User)); err != nil { + return fmt.Errorf("write magic image file in build context: %w", err) + } } - }() - if err != nil { - return fmt.Errorf("temp remount: %w", err) - } - skippedRebuild := false - stdoutWriter, closeStdout := log.Writer(opts.Logger) - defer closeStdout() - stderrWriter, closeStderr := log.Writer(opts.Logger) - defer closeStderr() - build := func() (v1.Image, error) { - defer cleanupBuildContext() - _, alreadyBuiltErr := opts.Filesystem.Stat(magicDir.Built()) - _, isImageErr := opts.Filesystem.Stat(magicDir.Image()) - if (alreadyBuiltErr == nil && opts.SkipRebuild) || isImageErr == nil { - endStage := startStage("🏗️ Skipping build because of cache...") - imageRef, err := devcontainer.ImageFromDockerfile(buildParams.DockerfileContent) - if err != nil { - return nil, fmt.Errorf("image from dockerfile: %w", err) + // temp move of all ro mounts + tempRemountDest := magicDir.Join("mnt") + // ignorePrefixes is a superset of ignorePaths that we pass to kaniko's + // IgnoreList. + ignorePrefixes := append([]string{"/dev", "/proc", "/sys"}, ignorePaths...) + restoreMounts, err := ebutil.TempRemount(opts.Logger, tempRemountDest, ignorePrefixes...) + defer func() { // restoreMounts should never be nil + if err := restoreMounts(); err != nil { + opts.Logger(log.LevelError, "restore mounts: %s", err.Error()) } - image, err := remote.Image(imageRef, remote.WithAuthFromKeychain(creds.GetKeychain())) + }() + if err != nil { + return fmt.Errorf("temp remount: %w", err) + } + + stdoutWriter, closeStdout := log.Writer(opts.Logger) + defer closeStdout() + stderrWriter, closeStderr := log.Writer(opts.Logger) + defer closeStderr() + build := func() (v1.Image, error) { + defer cleanupBuildContext() + _, alreadyBuiltErr := opts.Filesystem.Stat(magicDir.Built()) + _, isImageErr := opts.Filesystem.Stat(magicDir.Image()) + if (alreadyBuiltErr == nil && opts.SkipRebuild) || isImageErr == nil { + endStage := startStage("🏗️ Skipping build because of cache...") + imageRef, err := devcontainer.ImageFromDockerfile(buildParams.DockerfileContent) + if err != nil { + return nil, fmt.Errorf("image from dockerfile: %w", err) + } + image, err := remote.Image(imageRef, remote.WithAuthFromKeychain(creds.GetKeychain())) + if err != nil { + return nil, fmt.Errorf("image from remote: %w", err) + } + endStage("🏗️ Found image from remote!") + skippedRebuild = true + return image, nil + } + + // This is required for deleting the filesystem prior to build! + err = util.InitIgnoreList() if err != nil { - return nil, fmt.Errorf("image from remote: %w", err) + return nil, fmt.Errorf("init ignore list: %w", err) } - endStage("🏗️ Found image from remote!") - skippedRebuild = true - return image, nil - } - // This is required for deleting the filesystem prior to build! - err = util.InitIgnoreList() - if err != nil { - return nil, fmt.Errorf("init ignore list: %w", err) - } + // It's possible that the container will already have files in it, and + // we don't want to merge a new container with the old one. + if err := maybeDeleteFilesystem(opts.Logger, opts.ForceSafe); err != nil { + return nil, fmt.Errorf("delete filesystem: %w", err) + } - // It's possible that the container will already have files in it, and - // we don't want to merge a new container with the old one. - if err := maybeDeleteFilesystem(opts.Logger, opts.ForceSafe); err != nil { - return nil, fmt.Errorf("delete filesystem: %w", err) - } + cacheTTL := time.Hour * 24 * 7 + if opts.CacheTTLDays != 0 { + cacheTTL = time.Hour * 24 * time.Duration(opts.CacheTTLDays) + } - cacheTTL := time.Hour * 24 * 7 - if opts.CacheTTLDays != 0 { - cacheTTL = time.Hour * 24 * time.Duration(opts.CacheTTLDays) - } + // At this point we have all the context, we can now build! + registryMirror := []string{} + if val, ok := os.LookupEnv("KANIKO_REGISTRY_MIRROR"); ok { + registryMirror = strings.Split(val, ";") + } + var destinations []string + if opts.CacheRepo != "" { + destinations = append(destinations, opts.CacheRepo) + } + kOpts := &config.KanikoOptions{ + // Boilerplate! + CustomPlatform: platforms.Format(platforms.Normalize(platforms.DefaultSpec())), + SnapshotMode: "redo", + RunV2: true, + RunStdout: stdoutWriter, + RunStderr: stderrWriter, + Destinations: destinations, + NoPush: !opts.PushImage || len(destinations) == 0, + CacheRunLayers: true, + CacheCopyLayers: true, + CompressedCaching: true, + Compression: config.ZStd, + // Maps to "default" level, ~100-300 MB/sec according to + // benchmarks in klauspost/compress README + // https://github.com/klauspost/compress/blob/67a538e2b4df11f8ec7139388838a13bce84b5d5/zstd/encoder_options.go#L188 + CompressionLevel: 3, + CacheOptions: config.CacheOptions{ + CacheTTL: cacheTTL, + CacheDir: opts.BaseImageCacheDir, + }, + ForceUnpack: true, + BuildArgs: buildParams.BuildArgs, + CacheRepo: opts.CacheRepo, + Cache: opts.CacheRepo != "" || opts.BaseImageCacheDir != "", + DockerfilePath: buildParams.DockerfilePath, + DockerfileContent: buildParams.DockerfileContent, + RegistryOptions: config.RegistryOptions{ + Insecure: opts.Insecure, + InsecurePull: opts.Insecure, + SkipTLSVerify: opts.Insecure, + // Enables registry mirror features in Kaniko, see more in link below + // https://github.com/GoogleContainerTools/kaniko?tab=readme-ov-file#flag---registry-mirror + // Related to PR #114 + // https://github.com/coder/envbuilder/pull/114 + RegistryMirrors: registryMirror, + }, + SrcContext: buildParams.BuildContext, + + // For cached image utilization, produce reproducible builds. + Reproducible: opts.PushImage, + } - // At this point we have all the context, we can now build! - registryMirror := []string{} - if val, ok := os.LookupEnv("KANIKO_REGISTRY_MIRROR"); ok { - registryMirror = strings.Split(val, ";") - } - var destinations []string - if opts.CacheRepo != "" { - destinations = append(destinations, opts.CacheRepo) - } - kOpts := &config.KanikoOptions{ - // Boilerplate! - CustomPlatform: platforms.Format(platforms.Normalize(platforms.DefaultSpec())), - SnapshotMode: "redo", - RunV2: true, - RunStdout: stdoutWriter, - RunStderr: stderrWriter, - Destinations: destinations, - NoPush: !opts.PushImage || len(destinations) == 0, - CacheRunLayers: true, - CacheCopyLayers: true, - CompressedCaching: true, - Compression: config.ZStd, - // Maps to "default" level, ~100-300 MB/sec according to - // benchmarks in klauspost/compress README - // https://github.com/klauspost/compress/blob/67a538e2b4df11f8ec7139388838a13bce84b5d5/zstd/encoder_options.go#L188 - CompressionLevel: 3, - CacheOptions: config.CacheOptions{ - CacheTTL: cacheTTL, - CacheDir: opts.BaseImageCacheDir, - }, - ForceUnpack: true, - BuildArgs: buildParams.BuildArgs, - CacheRepo: opts.CacheRepo, - Cache: opts.CacheRepo != "" || opts.BaseImageCacheDir != "", - DockerfilePath: buildParams.DockerfilePath, - DockerfileContent: buildParams.DockerfileContent, - RegistryOptions: config.RegistryOptions{ - Insecure: opts.Insecure, - InsecurePull: opts.Insecure, - SkipTLSVerify: opts.Insecure, - // Enables registry mirror features in Kaniko, see more in link below - // https://github.com/GoogleContainerTools/kaniko?tab=readme-ov-file#flag---registry-mirror - // Related to PR #114 - // https://github.com/coder/envbuilder/pull/114 - RegistryMirrors: registryMirror, - }, - SrcContext: buildParams.BuildContext, + endStage := startStage("🏗️ Building image...") + image, err := executor.DoBuild(kOpts) + if err != nil { + return nil, xerrors.Errorf("do build: %w", err) + } + endStage("🏗️ Built image!") + if opts.PushImage { + endStage = startStage("🏗️ Pushing image...") + if err := executor.DoPush(image, kOpts); err != nil { + return nil, xerrors.Errorf("do push: %w", err) + } + endStage("🏗️ Pushed image!") + } - // For cached image utilization, produce reproducible builds. - Reproducible: opts.PushImage, + return image, err } - endStage := startStage("🏗️ Building image...") - image, err := executor.DoBuild(kOpts) + // At this point we have all the context, we can now build! + image, err := build() if err != nil { - return nil, xerrors.Errorf("do build: %w", err) - } - endStage("🏗️ Built image!") - if opts.PushImage { - endStage = startStage("🏗️ Pushing image...") - if err := executor.DoPush(image, kOpts); err != nil { - return nil, xerrors.Errorf("do push: %w", err) + fallback := false + switch { + case strings.Contains(err.Error(), "parsing dockerfile"): + fallback = true + fallbackErr = err + case strings.Contains(err.Error(), "error building stage"): + fallback = true + fallbackErr = err + // This occurs when the image cannot be found! + case strings.Contains(err.Error(), "authentication required"): + fallback = true + fallbackErr = err + // This occurs from Docker Hub when the image cannot be found! + case strings.Contains(err.Error(), "manifest unknown"): + fallback = true + fallbackErr = err + case strings.Contains(err.Error(), "unexpected status code 401 Unauthorized"): + opts.Logger(log.LevelError, "Unable to pull the provided image. Ensure your registry credentials are correct!") } - endStage("🏗️ Pushed image!") - } - - return image, err - } - - // At this point we have all the context, we can now build! - image, err := build() - if err != nil { - fallback := false - switch { - case strings.Contains(err.Error(), "parsing dockerfile"): - fallback = true - fallbackErr = err - case strings.Contains(err.Error(), "error building stage"): - fallback = true - fallbackErr = err - // This occurs when the image cannot be found! - case strings.Contains(err.Error(), "authentication required"): - fallback = true - fallbackErr = err - // This occurs from Docker Hub when the image cannot be found! - case strings.Contains(err.Error(), "manifest unknown"): - fallback = true - fallbackErr = err - case strings.Contains(err.Error(), "unexpected status code 401 Unauthorized"): - opts.Logger(log.LevelError, "Unable to pull the provided image. Ensure your registry credentials are correct!") - } - if !fallback || opts.ExitOnBuildFailure { - return err + if !fallback || opts.ExitOnBuildFailure { + return err + } + opts.Logger(log.LevelError, "Failed to build: %s", err) + opts.Logger(log.LevelError, "Falling back to the default image...") + buildParams, err = defaultBuildParams() + if err != nil { + return err + } + image, err = build() } - opts.Logger(log.LevelError, "Failed to build: %s", err) - opts.Logger(log.LevelError, "Falling back to the default image...") - buildParams, err = defaultBuildParams() if err != nil { - return err + return fmt.Errorf("build with kaniko: %w", err) } - image, err = build() - } - if err != nil { - return fmt.Errorf("build with kaniko: %w", err) - } - - if err := restoreMounts(); err != nil { - return fmt.Errorf("restore mounts: %w", err) - } - // Create the magic file to indicate that this build - // has already been ran before! - file, err := opts.Filesystem.Create(magicDir.Built()) - if err != nil { - return fmt.Errorf("create magic file: %w", err) - } - _ = file.Close() - - configFile, err := image.ConfigFile() - if err != nil { - return fmt.Errorf("get image config: %w", err) - } - - containerEnv := make(map[string]string) - remoteEnv := make(map[string]string) + if err := restoreMounts(); err != nil { + return fmt.Errorf("restore mounts: %w", err) + } - // devcontainer metadata can be persisted through a standard label - devContainerMetadata, exists := configFile.Config.Labels["devcontainer.metadata"] - if exists { - var devContainer []*devcontainer.Spec - devContainerMetadataBytes, err := hujson.Standardize([]byte(devContainerMetadata)) + // Create the magic file to indicate that this build + // has already been ran before! + file, err := opts.Filesystem.Create(magicDir.Built()) if err != nil { - return fmt.Errorf("humanize json for dev container metadata: %w", err) + return fmt.Errorf("create magic file: %w", err) } - err = json.Unmarshal(devContainerMetadataBytes, &devContainer) + _ = file.Close() + + configFile, err := image.ConfigFile() if err != nil { - return fmt.Errorf("unmarshal metadata: %w", err) + return fmt.Errorf("get image config: %w", err) } - opts.Logger(log.LevelInfo, "#%d: 👀 Found devcontainer.json label metadata in image...", stageNumber) - for _, container := range devContainer { - if container.RemoteUser != "" { - opts.Logger(log.LevelInfo, "#%d: 🧑 Updating the user to %q!", stageNumber, container.RemoteUser) - configFile.Config.User = container.RemoteUser - } - maps.Copy(containerEnv, container.ContainerEnv) - maps.Copy(remoteEnv, container.RemoteEnv) - if !container.OnCreateCommand.IsEmpty() { - scripts.OnCreateCommand = container.OnCreateCommand - } - if !container.UpdateContentCommand.IsEmpty() { - scripts.UpdateContentCommand = container.UpdateContentCommand + containerEnv := make(map[string]string) + remoteEnv := make(map[string]string) + + // devcontainer metadata can be persisted through a standard label + devContainerMetadata, exists := configFile.Config.Labels["devcontainer.metadata"] + if exists { + var devContainer []*devcontainer.Spec + devContainerMetadataBytes, err := hujson.Standardize([]byte(devContainerMetadata)) + if err != nil { + return fmt.Errorf("humanize json for dev container metadata: %w", err) } - if !container.PostCreateCommand.IsEmpty() { - scripts.PostCreateCommand = container.PostCreateCommand + err = json.Unmarshal(devContainerMetadataBytes, &devContainer) + if err != nil { + return fmt.Errorf("unmarshal metadata: %w", err) } - if !container.PostStartCommand.IsEmpty() { - scripts.PostStartCommand = container.PostStartCommand + opts.Logger(log.LevelInfo, "#%d: 👀 Found devcontainer.json label metadata in image...", stageNumber) + for _, container := range devContainer { + if container.RemoteUser != "" { + opts.Logger(log.LevelInfo, "#%d: 🧑 Updating the user to %q!", stageNumber, container.RemoteUser) + + configFile.Config.User = container.RemoteUser + } + maps.Copy(containerEnv, container.ContainerEnv) + maps.Copy(remoteEnv, container.RemoteEnv) + if !container.OnCreateCommand.IsEmpty() { + scripts.OnCreateCommand = container.OnCreateCommand + } + if !container.UpdateContentCommand.IsEmpty() { + scripts.UpdateContentCommand = container.UpdateContentCommand + } + if !container.PostCreateCommand.IsEmpty() { + scripts.PostCreateCommand = container.PostCreateCommand + } + if !container.PostStartCommand.IsEmpty() { + scripts.PostStartCommand = container.PostStartCommand + } } } - } - // Sanitize the environment of any opts! - options.UnsetEnv() + // Sanitize the environment of any opts! + options.UnsetEnv() - // Remove the Docker config secret file! - if err := cleanupDockerConfigJSON(); err != nil { - return err - } + // Remove the Docker config secret file! + if err := cleanupDockerConfigJSON(); err != nil { + return err + } - environ, err := os.ReadFile("/etc/environment") - if err == nil { - for _, env := range strings.Split(string(environ), "\n") { - pair := strings.SplitN(env, "=", 2) - if len(pair) != 2 { - continue + environ, err := os.ReadFile("/etc/environment") + if err == nil { + for _, env := range strings.Split(string(environ), "\n") { + pair := strings.SplitN(env, "=", 2) + if len(pair) != 2 { + continue + } + os.Setenv(pair[0], pair[1]) } - os.Setenv(pair[0], pair[1]) } - } - allEnvKeys := make(map[string]struct{}) + allEnvKeys := make(map[string]struct{}) - // It must be set in this parent process otherwise nothing will be found! - for _, env := range configFile.Config.Env { - pair := strings.SplitN(env, "=", 2) - os.Setenv(pair[0], pair[1]) - allEnvKeys[pair[0]] = struct{}{} - } - maps.Copy(containerEnv, buildParams.ContainerEnv) - maps.Copy(remoteEnv, buildParams.RemoteEnv) - - // Set Envbuilder runtime markers - containerEnv["ENVBUILDER"] = "true" - if devcontainerPath != "" { - containerEnv["DEVCONTAINER"] = "true" - containerEnv["DEVCONTAINER_CONFIG"] = devcontainerPath - } + // It must be set in this parent process otherwise nothing will be found! + for _, env := range configFile.Config.Env { + pair := strings.SplitN(env, "=", 2) + os.Setenv(pair[0], pair[1]) + allEnvKeys[pair[0]] = struct{}{} + } + maps.Copy(containerEnv, buildParams.ContainerEnv) + maps.Copy(remoteEnv, buildParams.RemoteEnv) - for _, env := range []map[string]string{containerEnv, remoteEnv} { - envKeys := make([]string, 0, len(env)) - for key := range env { - envKeys = append(envKeys, key) - allEnvKeys[key] = struct{}{} + // Set Envbuilder runtime markers + containerEnv["ENVBUILDER"] = "true" + if devcontainerPath != "" { + containerEnv["DEVCONTAINER"] = "true" + containerEnv["DEVCONTAINER_CONFIG"] = devcontainerPath } - sort.Strings(envKeys) - for _, envVar := range envKeys { - value := devcontainer.SubstituteVars(env[envVar], opts.WorkspaceFolder, os.LookupEnv) - os.Setenv(envVar, value) + + for _, env := range []map[string]string{containerEnv, remoteEnv} { + envKeys := make([]string, 0, len(env)) + for key := range env { + envKeys = append(envKeys, key) + allEnvKeys[key] = struct{}{} + } + sort.Strings(envKeys) + for _, envVar := range envKeys { + value := devcontainer.SubstituteVars(env[envVar], opts.WorkspaceFolder, os.LookupEnv) + os.Setenv(envVar, value) + } } - } - // Do not export env if we skipped a rebuild, because ENV directives - // from the Dockerfile would not have been processed and we'd miss these - // in the export. We should have generated a complete set of environment - // on the intial build, so exporting environment variables a second time - // isn't useful anyway. - if opts.ExportEnvFile != "" && !skippedRebuild { - exportEnvFile, err := os.Create(opts.ExportEnvFile) - if err != nil { - return fmt.Errorf("failed to open EXPORT_ENV_FILE %q: %w", opts.ExportEnvFile, err) + // Do not export env if we skipped a rebuild, because ENV directives + // from the Dockerfile would not have been processed and we'd miss these + // in the export. We should have generated a complete set of environment + // on the intial build, so exporting environment variables a second time + // isn't useful anyway. + if opts.ExportEnvFile != "" && !skippedRebuild { + exportEnvFile, err := os.Create(opts.ExportEnvFile) + if err != nil { + return fmt.Errorf("failed to open EXPORT_ENV_FILE %q: %w", opts.ExportEnvFile, err) + } + + envKeys := make([]string, 0, len(allEnvKeys)) + for key := range allEnvKeys { + envKeys = append(envKeys, key) + } + sort.Strings(envKeys) + for _, key := range envKeys { + fmt.Fprintf(exportEnvFile, "%s=%s\n", key, os.Getenv(key)) + } + + exportEnvFile.Close() } - envKeys := make([]string, 0, len(allEnvKeys)) - for key := range allEnvKeys { - envKeys = append(envKeys, key) + username = configFile.Config.User + if buildParams.User != "" { + username = buildParams.User } - sort.Strings(envKeys) - for _, key := range envKeys { - fmt.Fprintf(exportEnvFile, "%s=%s\n", key, os.Getenv(key)) + } else { + skippedRebuild = true + magicEnv, err := parseMagicImageFile(opts.Filesystem, magicDir.Image()) + if err != nil { + return fmt.Errorf("parse magic env: %w", err) } - - exportEnvFile.Close() - } - - username := configFile.Config.User - if buildParams.User != "" { - username = buildParams.User + username = magicEnv["USER"] } if username == "" { opts.Logger(log.LevelWarn, "#%d: no user specified, using root", stageNumber) } - userInfo, err := getUser(username) if err != nil { return fmt.Errorf("update user: %w", err) @@ -957,7 +975,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) } defaultBuildParams := func() (*devcontainer.Compiled, error) { - dockerfile := filepath.Join(buildTimeWorkspaceFolder, "Dockerfile") + dockerfile := magicDir.Join("Dockerfile") file, err := opts.Filesystem.OpenFile(dockerfile, os.O_CREATE|os.O_WRONLY, 0o644) if err != nil { return nil, err @@ -979,7 +997,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) return &devcontainer.Compiled{ DockerfilePath: dockerfile, DockerfileContent: content, - BuildContext: buildTimeWorkspaceFolder, + BuildContext: magicDir.Path(), }, nil } @@ -1019,7 +1037,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) opts.Logger(log.LevelInfo, "No Dockerfile or image specified; falling back to the default image...") fallbackDockerfile = defaultParams.DockerfilePath } - buildParams, err = devContainer.Compile(opts.Filesystem, devcontainerDir, buildTimeWorkspaceFolder, fallbackDockerfile, opts.WorkspaceFolder, false, os.LookupEnv) + buildParams, err = devContainer.Compile(opts.Filesystem, devcontainerDir, magicDir.Path(), fallbackDockerfile, opts.WorkspaceFolder, false, os.LookupEnv) if err != nil { return nil, fmt.Errorf("compile devcontainer.json: %w", err) } @@ -1110,26 +1128,16 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) // add the magic directives to the Dockerfile content. // MAGICDIR buildParams.DockerfileContent += magicdir.Directives + magicTempDir := filepath.Join(buildParams.BuildContext, magicdir.TempDir) if err := opts.Filesystem.MkdirAll(magicTempDir, 0o755); err != nil { return nil, fmt.Errorf("create magic temp dir in build context: %w", err) } envbuilderBinDest := filepath.Join(magicTempDir, "envbuilder") - - // Copy the envbuilder binary into the build context. - opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, envbuilderBinDest) - if err := copyFile(opts.Filesystem, opts.BinaryPath, envbuilderBinDest, 0o755); err != nil { - return nil, xerrors.Errorf("copy envbuilder binary to build context: %w", err) - } - - // Also touch the magic file that signifies the image has been built!A magicImageDest := filepath.Join(magicTempDir, "image") - opts.Logger(log.LevelDebug, "touching magic image file at %q in build context %q", magicImageDest, magicTempDir) - if err := touchFile(opts.Filesystem, magicImageDest, 0o755); err != nil { - return nil, fmt.Errorf("touch magic image file at %q: %w", magicImageDest, err) - } + + // Clean up after probe! defer func() { - // Clean up after we're done! for _, path := range []string{magicImageDest, envbuilderBinDest, magicTempDir} { if err := opts.Filesystem.Remove(path); err != nil { opts.Logger(log.LevelWarn, "failed to clean up magic temp dir from build context: %w", err) @@ -1137,6 +1145,21 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) } }() + // Copy the envbuilder binary into the build context. External callers + // will need to specify the path to the desired envbuilder binary. + opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, envbuilderBinDest) + if err := copyFile(opts.Filesystem, opts.BinaryPath, envbuilderBinDest, 0o755); err != nil { + return nil, xerrors.Errorf("copy envbuilder binary to build context: %w", err) + } + + // Also write the magic file that signifies the image has been built. + // Since the user in the image is set to root, we also store the user + // in the magic file to be used by envbuilder when the image is run. + opts.Logger(log.LevelDebug, "writing magic image file at %q in build context %q", magicImageDest, magicTempDir) + if err := writeFile(opts.Filesystem, magicImageDest, 0o755, fmt.Sprintf("USER=%s\n", buildParams.User)); err != nil { + return nil, fmt.Errorf("write magic image file in build context: %w", err) + } + stdoutWriter, closeStdout := log.Writer(opts.Logger) defer closeStdout() stderrWriter, closeStderr := log.Writer(opts.Logger) @@ -1465,12 +1488,43 @@ func copyFile(fs billy.Filesystem, src, dst string, mode fs.FileMode) error { return nil } -func touchFile(fs billy.Filesystem, dst string, mode fs.FileMode) error { - f, err := fs.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) +func writeFile(fs billy.Filesystem, dst string, mode fs.FileMode, content string) error { + f, err := fs.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode) if err != nil { - return xerrors.Errorf("failed to touch file: %w", err) + return fmt.Errorf("open file: %w", err) + } + defer f.Close() + _, err = f.Write([]byte(content)) + if err != nil { + return fmt.Errorf("write file: %w", err) + } + return nil +} + +func parseMagicImageFile(fs billy.Filesystem, path string) (map[string]string, error) { + file, err := fs.Open(path) + if err != nil { + return nil, fmt.Errorf("open magic image file: %w", err) + } + defer file.Close() + + env := make(map[string]string) + s := bufio.NewScanner(file) + for s.Scan() { + line := strings.TrimSpace(s.Text()) + if line == "" { + continue + } + parts := strings.SplitN(line, "=", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("invalid magic image file format: %q", line) + } + env[parts[0]] = parts[1] + } + if err := s.Err(); err != nil { + return nil, fmt.Errorf("scan magic image file: %w", err) } - return f.Close() + return env, nil } func initDockerConfigJSON(logf log.Func, magicDir magicdir.MagicDir, dockerConfigBase64 string) (func() error, error) { diff --git a/integration/integration_test.go b/integration/integration_test.go index 29b5e8a7..f88829aa 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -17,6 +17,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strings" "testing" "time" @@ -41,6 +42,7 @@ import ( "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/uuid" @@ -155,8 +157,17 @@ FROM a AS b`, testImageUbuntu), }}) require.NoError(t, err) - output := execContainer(t, ctr, "ps aux | awk '/^pickme / {print $1}' | sort -u") - require.Equal(t, "pickme", strings.TrimSpace(output)) + // Check that envbuilder started command as user. + // Since envbuilder starts as root, probe for up to 10 seconds. + for i := 0; i < 10; i++ { + out := execContainer(t, ctr, "ps aux | awk '/^pickme * 1 / {print $1}' | sort -u") + got := strings.TrimSpace(out) + if got == "pickme" { + return + } + time.Sleep(time.Second) + } + require.Fail(t, "expected pid 1 to be running as pickme") } func TestUidGid(t *testing.T) { @@ -1160,7 +1171,7 @@ RUN date --utc > /root/date.txt`, testImageAlpine), _, err = remote.Image(ref) require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") - // When: we run envbuilder with PUSH_IMAGE set + // When: we run envbuilder with no PUSH_IMAGE set _, err = runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), @@ -1184,6 +1195,9 @@ RUN date --utc > /root/date.txt`, testImageAlpine), t.Run("CacheAndPush", func(t *testing.T) { t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + srv := gittest.CreateGitServer(t, gittest.Options{ Files: map[string]string{ ".devcontainer/Dockerfile": fmt.Sprintf(`FROM %s @@ -1210,90 +1224,33 @@ RUN date --utc > /root/date.txt`, testImageAlpine), _, err = remote.Image(ref) require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") - // When: we run envbuilder with GET_CACHED_IMAGE - _, err = runEnvbuilder(t, runOpts{env: []string{ + opts := []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), - envbuilderEnv("GET_CACHED_IMAGE", "1"), envbuilderEnv("VERBOSE", "1"), - }}) + } + + // When: we run envbuilder with GET_CACHED_IMAGE + _, err = runEnvbuilder(t, runOpts{env: append(opts, + envbuilderEnv("GET_CACHED_IMAGE", "1"), + )}) require.ErrorContains(t, err, "error probing build cache: uncached RUN command") // Then: it should fail to build the image and nothing should be pushed _, err = remote.Image(ref) require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") // When: we run envbuilder with PUSH_IMAGE set - _, err = runEnvbuilder(t, runOpts{env: []string{ - envbuilderEnv("GIT_URL", srv.URL), - envbuilderEnv("CACHE_REPO", testRepo), - envbuilderEnv("PUSH_IMAGE", "1"), - envbuilderEnv("VERBOSE", "1"), - }}) - require.NoError(t, err) - - // Then: the image should be pushed - img, err := remote.Image(ref) - require.NoError(t, err, "expected image to be present after build + push") - - // Then: the image should have its directives replaced with those required - // to run envbuilder automatically - configFile, err := img.ConfigFile() - require.NoError(t, err, "expected image to return a config file") - - assert.Equal(t, "root", configFile.Config.User, "user must be root") - assert.Equal(t, "/", configFile.Config.WorkingDir, "workdir must be /") - if assert.Len(t, configFile.Config.Entrypoint, 1) { - assert.Equal(t, "/.envbuilder/bin/envbuilder", configFile.Config.Entrypoint[0], "incorrect entrypoint") - } - - // Then: re-running envbuilder with GET_CACHED_IMAGE should succeed - ctrID, err := runEnvbuilder(t, runOpts{env: []string{ - envbuilderEnv("GIT_URL", srv.URL), - envbuilderEnv("CACHE_REPO", testRepo), - envbuilderEnv("GET_CACHED_IMAGE", "1"), - }}) - require.NoError(t, err) + _ = pushImage(t, ref, nil, opts...) - // Then: the cached image ref should be emitted in the container logs - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) require.NoError(t, err) defer cli.Close() - logs, err := cli.ContainerLogs(ctx, ctrID, container.LogsOptions{ - ShowStdout: true, - ShowStderr: true, - }) - require.NoError(t, err) - defer logs.Close() - logBytes, err := io.ReadAll(logs) - require.NoError(t, err) - require.Regexp(t, `ENVBUILDER_CACHED_IMAGE=(\S+)`, string(logBytes)) - // When: we pull the image we just built - rc, err := cli.ImagePull(ctx, ref.String(), image.PullOptions{}) - require.NoError(t, err) - t.Cleanup(func() { _ = rc.Close() }) - _, err = io.ReadAll(rc) - require.NoError(t, err) + // Then: re-running envbuilder with GET_CACHED_IMAGE should succeed + cachedRef := getCachedImage(ctx, t, cli, opts...) // When: we run the image we just built - ctr, err := cli.ContainerCreate(ctx, &container.Config{ - Image: ref.String(), - Entrypoint: []string{"sleep", "infinity"}, - Labels: map[string]string{ - testContainerLabel: "true", - }, - }, nil, nil, nil, "") - require.NoError(t, err) - t.Cleanup(func() { - _ = cli.ContainerRemove(ctx, ctr.ID, container.RemoveOptions{ - RemoveVolumes: true, - Force: true, - }) - }) - err = cli.ContainerStart(ctx, ctr.ID, container.StartOptions{}) - require.NoError(t, err) + ctr := startContainerFromRef(ctx, t, cli, cachedRef) // Then: the envbuilder binary exists in the image! out := execContainer(t, ctr.ID, "/.envbuilder/bin/envbuilder --help") @@ -1305,6 +1262,9 @@ RUN date --utc > /root/date.txt`, testImageAlpine), t.Run("CacheAndPushDevcontainerOnly", func(t *testing.T) { t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + srv := gittest.CreateGitServer(t, gittest.Options{ Files: map[string]string{ ".devcontainer/devcontainer.json": fmt.Sprintf(`{"image": %q}`, testImageAlpine), @@ -1319,88 +1279,32 @@ RUN date --utc > /root/date.txt`, testImageAlpine), _, err = remote.Image(ref) require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") - // When: we run envbuilder with GET_CACHED_IMAGE - _, err = runEnvbuilder(t, runOpts{env: []string{ + opts := []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), + } + + // When: we run envbuilder with GET_CACHED_IMAGE + _, err = runEnvbuilder(t, runOpts{env: append(opts, envbuilderEnv("GET_CACHED_IMAGE", "1"), - }}) + )}) require.ErrorContains(t, err, "error probing build cache: uncached COPY command") // Then: it should fail to build the image and nothing should be pushed _, err = remote.Image(ref) require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") // When: we run envbuilder with PUSH_IMAGE set - _, err = runEnvbuilder(t, runOpts{env: []string{ - envbuilderEnv("GIT_URL", srv.URL), - envbuilderEnv("CACHE_REPO", testRepo), - envbuilderEnv("PUSH_IMAGE", "1"), - }}) - require.NoError(t, err) - - // Then: the image should be pushed - img, err := remote.Image(ref) - require.NoError(t, err, "expected image to be present after build + push") - - // Then: the image should have its directives replaced with those required - // to run envbuilder automatically - configFile, err := img.ConfigFile() - require.NoError(t, err, "expected image to return a config file") - - assert.Equal(t, "root", configFile.Config.User, "user must be root") - assert.Equal(t, "/", configFile.Config.WorkingDir, "workdir must be /") - if assert.Len(t, configFile.Config.Entrypoint, 1) { - assert.Equal(t, "/.envbuilder/bin/envbuilder", configFile.Config.Entrypoint[0], "incorrect entrypoint") - } - - // Then: re-running envbuilder with GET_CACHED_IMAGE should succeed - ctrID, err := runEnvbuilder(t, runOpts{env: []string{ - envbuilderEnv("GIT_URL", srv.URL), - envbuilderEnv("CACHE_REPO", testRepo), - envbuilderEnv("GET_CACHED_IMAGE", "1"), - }}) - require.NoError(t, err) + _ = pushImage(t, ref, nil, opts...) - // Then: the cached image ref should be emitted in the container logs - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) require.NoError(t, err) defer cli.Close() - logs, err := cli.ContainerLogs(ctx, ctrID, container.LogsOptions{ - ShowStdout: true, - ShowStderr: true, - }) - require.NoError(t, err) - defer logs.Close() - logBytes, err := io.ReadAll(logs) - require.NoError(t, err) - require.Regexp(t, `ENVBUILDER_CACHED_IMAGE=(\S+)`, string(logBytes)) - // When: we pull the image we just built - rc, err := cli.ImagePull(ctx, ref.String(), image.PullOptions{}) - require.NoError(t, err) - t.Cleanup(func() { _ = rc.Close() }) - _, err = io.ReadAll(rc) - require.NoError(t, err) + // Then: re-running envbuilder with GET_CACHED_IMAGE should succeed + cachedRef := getCachedImage(ctx, t, cli, opts...) // When: we run the image we just built - ctr, err := cli.ContainerCreate(ctx, &container.Config{ - Image: ref.String(), - Entrypoint: []string{"sleep", "infinity"}, - Labels: map[string]string{ - testContainerLabel: "true", - }, - }, nil, nil, nil, "") - require.NoError(t, err) - t.Cleanup(func() { - _ = cli.ContainerRemove(ctx, ctr.ID, container.RemoveOptions{ - RemoveVolumes: true, - Force: true, - }) - }) - err = cli.ContainerStart(ctx, ctr.ID, container.StartOptions{}) - require.NoError(t, err) + ctr := startContainerFromRef(ctx, t, cli, cachedRef) // Then: the envbuilder binary exists in the image! out := execContainer(t, ctr.ID, "/.envbuilder/bin/envbuilder --help") @@ -1430,18 +1334,18 @@ RUN date --utc > /root/date.txt`, testImageAlpine), }) // Given: an empty registry - opts := setupInMemoryRegistryOpts{ + authOpts := setupInMemoryRegistryOpts{ Username: "testing", Password: "testing", } - remoteAuthOpt := remote.WithAuth(&authn.Basic{Username: opts.Username, Password: opts.Password}) - testReg := setupInMemoryRegistry(t, opts) + remoteAuthOpt := remote.WithAuth(&authn.Basic{Username: authOpts.Username, Password: authOpts.Password}) + testReg := setupInMemoryRegistry(t, authOpts) testRepo := testReg + "/test" regAuthJSON, err := json.Marshal(envbuilder.DockerConfig{ AuthConfigs: map[string]clitypes.AuthConfig{ testRepo: { - Username: opts.Username, - Password: opts.Password, + Username: authOpts.Username, + Password: authOpts.Password, }, }, }) @@ -1451,37 +1355,32 @@ RUN date --utc > /root/date.txt`, testImageAlpine), _, err = remote.Image(ref, remoteAuthOpt) require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") - // When: we run envbuilder with GET_CACHED_IMAGE - _, err = runEnvbuilder(t, runOpts{env: []string{ + opts := []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), + envbuilderEnv("DOCKER_CONFIG_BASE64", base64.StdEncoding.EncodeToString(regAuthJSON)), + } + + // When: we run envbuilder with GET_CACHED_IMAGE + _, err = runEnvbuilder(t, runOpts{env: append(opts, envbuilderEnv("GET_CACHED_IMAGE", "1"), - }}) + )}) require.ErrorContains(t, err, "error probing build cache: uncached RUN command") // Then: it should fail to build the image and nothing should be pushed _, err = remote.Image(ref, remoteAuthOpt) require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") // When: we run envbuilder with PUSH_IMAGE set - _, err = runEnvbuilder(t, runOpts{env: []string{ - envbuilderEnv("GIT_URL", srv.URL), - envbuilderEnv("CACHE_REPO", testRepo), - envbuilderEnv("PUSH_IMAGE", "1"), - envbuilderEnv("DOCKER_CONFIG_BASE64", base64.StdEncoding.EncodeToString(regAuthJSON)), - }}) - require.NoError(t, err) + _ = pushImage(t, ref, remoteAuthOpt, opts...) // Then: the image should be pushed _, err = remote.Image(ref, remoteAuthOpt) require.NoError(t, err, "expected image to be present after build + push") // Then: re-running envbuilder with GET_CACHED_IMAGE should succeed - _, err = runEnvbuilder(t, runOpts{env: []string{ - envbuilderEnv("GIT_URL", srv.URL), - envbuilderEnv("CACHE_REPO", testRepo), + _, err = runEnvbuilder(t, runOpts{env: append(opts, envbuilderEnv("GET_CACHED_IMAGE", "1"), - envbuilderEnv("DOCKER_CONFIG_BASE64", base64.StdEncoding.EncodeToString(regAuthJSON)), - }}) + )}) require.NoError(t, err) }) @@ -1507,35 +1406,36 @@ RUN date --utc > /root/date.txt`, testImageAlpine), }) // Given: an empty registry - opts := setupInMemoryRegistryOpts{ + authOpts := setupInMemoryRegistryOpts{ Username: "testing", Password: "testing", } - remoteAuthOpt := remote.WithAuth(&authn.Basic{Username: opts.Username, Password: opts.Password}) - testReg := setupInMemoryRegistry(t, opts) + remoteAuthOpt := remote.WithAuth(&authn.Basic{Username: authOpts.Username, Password: authOpts.Password}) + testReg := setupInMemoryRegistry(t, authOpts) testRepo := testReg + "/test" ref, err := name.ParseReference(testRepo + ":latest") require.NoError(t, err) _, err = remote.Image(ref, remoteAuthOpt) require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") - // When: we run envbuilder with GET_CACHED_IMAGE - _, err = runEnvbuilder(t, runOpts{env: []string{ + opts := []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), + } + + // When: we run envbuilder with GET_CACHED_IMAGE + _, err = runEnvbuilder(t, runOpts{env: append(opts, envbuilderEnv("GET_CACHED_IMAGE", "1"), - }}) + )}) require.ErrorContains(t, err, "error probing build cache: uncached RUN command") // Then: it should fail to build the image and nothing should be pushed _, err = remote.Image(ref, remoteAuthOpt) require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") // When: we run envbuilder with PUSH_IMAGE set - _, err = runEnvbuilder(t, runOpts{env: []string{ - envbuilderEnv("GIT_URL", srv.URL), - envbuilderEnv("CACHE_REPO", testRepo), + _, err = runEnvbuilder(t, runOpts{env: append(opts, envbuilderEnv("PUSH_IMAGE", "1"), - }}) + )}) // Then: it should fail with an Unauthorized error require.ErrorContains(t, err, "401 Unauthorized", "expected unauthorized error using no auth when cache repo requires it") @@ -1547,6 +1447,9 @@ RUN date --utc > /root/date.txt`, testImageAlpine), t.Run("CacheAndPushMultistage", func(t *testing.T) { t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + srv := gittest.CreateGitServer(t, gittest.Options{ Files: map[string]string{ "Dockerfile": fmt.Sprintf(` @@ -1576,80 +1479,33 @@ COPY --from=prebuild /the-future/hello.txt /the-future/hello.txt _, err = remote.Image(ref) require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") - // When: we run envbuilder with GET_CACHED_IMAGE - _, err = runEnvbuilder(t, runOpts{env: []string{ + opts := []string{ envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), - envbuilderEnv("GET_CACHED_IMAGE", "1"), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), - }}) + } + + // When: we run envbuilder with GET_CACHED_IMAGE + _, err = runEnvbuilder(t, runOpts{env: append(opts, + envbuilderEnv("GET_CACHED_IMAGE", "1"), + )}) require.ErrorContains(t, err, "error probing build cache: uncached RUN command") // Then: it should fail to build the image and nothing should be pushed _, err = remote.Image(ref) require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") // When: we run envbuilder with PUSH_IMAGE set - _, err = runEnvbuilder(t, runOpts{env: []string{ - envbuilderEnv("GIT_URL", srv.URL), - envbuilderEnv("CACHE_REPO", testRepo), - envbuilderEnv("PUSH_IMAGE", "1"), - envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), - }}) - require.NoError(t, err) - - // Then: the image should be pushed - _, err = remote.Image(ref) - require.NoError(t, err, "expected image to be present after build + push") + _ = pushImage(t, ref, nil, opts...) - // Then: re-running envbuilder with GET_CACHED_IMAGE should succeed - ctrID, err := runEnvbuilder(t, runOpts{env: []string{ - envbuilderEnv("GIT_URL", srv.URL), - envbuilderEnv("CACHE_REPO", testRepo), - envbuilderEnv("GET_CACHED_IMAGE", "1"), - envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), - }}) - require.NoError(t, err) - - // Then: the cached image ref should be emitted in the container logs - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) require.NoError(t, err) defer cli.Close() - logs, err := cli.ContainerLogs(ctx, ctrID, container.LogsOptions{ - ShowStdout: true, - ShowStderr: true, - }) - require.NoError(t, err) - defer logs.Close() - logBytes, err := io.ReadAll(logs) - require.NoError(t, err) - require.Regexp(t, `ENVBUILDER_CACHED_IMAGE=(\S+)`, string(logBytes)) - // When: we pull the image we just built - rc, err := cli.ImagePull(ctx, ref.String(), image.PullOptions{}) - require.NoError(t, err) - t.Cleanup(func() { _ = rc.Close() }) - _, err = io.ReadAll(rc) - require.NoError(t, err) + // Then: re-running envbuilder with GET_CACHED_IMAGE should succeed + cachedRef := getCachedImage(ctx, t, cli, opts...) // When: we run the image we just built - ctr, err := cli.ContainerCreate(ctx, &container.Config{ - Image: ref.String(), - Entrypoint: []string{"sleep", "infinity"}, - Labels: map[string]string{ - testContainerLabel: "true", - }, - }, nil, nil, nil, "") - require.NoError(t, err) - t.Cleanup(func() { - _ = cli.ContainerRemove(ctx, ctr.ID, container.RemoveOptions{ - RemoveVolumes: true, - Force: true, - }) - }) - err = cli.ContainerStart(ctx, ctr.ID, container.StartOptions{}) - require.NoError(t, err) + ctr := startContainerFromRef(ctx, t, cli, cachedRef) // Then: The files from the prebuild stage are present. out := execContainer(t, ctr.ID, "/bin/sh -c 'cat /the-past/hello.txt /the-future/hello.txt; readlink -f /the-past/hello.link'") @@ -1698,17 +1554,11 @@ RUN date --utc > /root/date.txt require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") // When: we run envbuilder with PUSH_IMAGE set - _, err = runEnvbuilder(t, runOpts{env: []string{ + _ = pushImage(t, ref, nil, envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), - envbuilderEnv("PUSH_IMAGE", "1"), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), - }}) - require.NoError(t, err) - - // Then: the image should be pushed - _, err = remote.Image(ref) - require.NoError(t, err, "expected image to be present after build + push") + ) // Then: re-running envbuilder with GET_CACHED_IMAGE should succeed _, err = runEnvbuilder(t, runOpts{env: []string{ @@ -1725,13 +1575,11 @@ RUN date --utc > /root/date.txt srv = newServer(dockerfilePrebuildContents) // When: we rebuild the prebuild stage so that the cache is created - _, err = runEnvbuilder(t, runOpts{env: []string{ + _ = pushImage(t, ref, nil, envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", testRepo), - envbuilderEnv("PUSH_IMAGE", "1"), envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), - }}) - require.NoError(t, err) + ) // Then: re-running envbuilder with GET_CACHED_IMAGE should still fail // on the second stage because the first stage file has changed. @@ -1814,6 +1662,120 @@ RUN date --utc > /root/date.txt`, testImageAlpine), // Then: envbuilder should fail with a descriptive error require.ErrorContains(t, err, "failed to push to destination") }) + + t.Run("CacheAndPushDevcontainerFeatures", func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ + // NOTE(mafredri): We can't cache the feature in our local + // registry because the image media type is incompatible. + ".devcontainer/devcontainer.json": fmt.Sprintf(` +{ + "image": %q, + "features": { + "ghcr.io/devcontainers/feature-starter/color:1": { + "favorite": "green" + } + } +} +`, testImageUbuntu), + }, + }) + + // Given: an empty registry + testReg := setupInMemoryRegistry(t, setupInMemoryRegistryOpts{}) + testRepo := testReg + "/test" + ref, err := name.ParseReference(testRepo + ":latest") + require.NoError(t, err) + _, err = remote.Image(ref) + require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") + + opts := []string{ + envbuilderEnv("GIT_URL", srv.URL), + envbuilderEnv("CACHE_REPO", testRepo), + } + + // When: we run envbuilder with PUSH_IMAGE set + _ = pushImage(t, ref, nil, opts...) + + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + require.NoError(t, err) + defer cli.Close() + + // Then: re-running envbuilder with GET_CACHED_IMAGE should succeed + cachedRef := getCachedImage(ctx, t, cli, opts...) + + // When: we run the image we just built + ctr := startContainerFromRef(ctx, t, cli, cachedRef) + + // Check that the feature is present in the image. + out := execContainer(t, ctr.ID, "/usr/local/bin/color") + require.Contains(t, strings.TrimSpace(out), "my favorite color is green") + }) + + t.Run("CacheAndPushUser", func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ + ".devcontainer/devcontainer.json": `{ + "name": "Test", + "build": { + "dockerfile": "Dockerfile" + }, + }`, + ".devcontainer/Dockerfile": fmt.Sprintf(`FROM %s +RUN useradd -m -s /bin/bash devalot +USER devalot +`, testImageUbuntu), + }, + }) + + // Given: an empty registry + testReg := setupInMemoryRegistry(t, setupInMemoryRegistryOpts{}) + testRepo := testReg + "/test" + ref, err := name.ParseReference(testRepo + ":latest") + require.NoError(t, err) + _, err = remote.Image(ref) + require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") + + opts := []string{ + envbuilderEnv("GIT_URL", srv.URL), + envbuilderEnv("CACHE_REPO", testRepo), + } + + // When: we run envbuilder with PUSH_IMAGE set + _ = pushImage(t, ref, nil, opts...) + + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + require.NoError(t, err) + defer cli.Close() + + // Then: re-running envbuilder with GET_CACHED_IMAGE should succeed + cachedRef := getCachedImage(ctx, t, cli, opts...) + + // When: we run the image we just built + ctr := startContainerFromRef(ctx, t, cli, cachedRef) + + // Check that envbuilder started command as user. + // Since envbuilder starts as root, probe for up to 10 seconds. + for i := 0; i < 10; i++ { + out := execContainer(t, ctr.ID, "ps aux | awk '/^devalot * 1 / {print $1}' | sort -u") + got := strings.TrimSpace(out) + if got == "devalot" { + return + } + time.Sleep(time.Second) + } + require.Fail(t, "expected pid 1 to be running as devalot") + }) } func TestChownHomedir(t *testing.T) { @@ -1935,6 +1897,91 @@ func cleanOldEnvbuilders() { } } +func pushImage(t *testing.T, ref name.Reference, remoteOpt remote.Option, env ...string) v1.Image { + t.Helper() + + var remoteOpts []remote.Option + if remoteOpt != nil { + remoteOpts = append(remoteOpts, remoteOpt) + } + + _, err := runEnvbuilder(t, runOpts{env: append(env, envbuilderEnv("PUSH_IMAGE", "1"))}) + require.NoError(t, err, "envbuilder push image failed") + + img, err := remote.Image(ref, remoteOpts...) + require.NoError(t, err, "expected image to be present after build + push") + + // The image should have its directives replaced with those required + // to run envbuilder automatically + configFile, err := img.ConfigFile() + require.NoError(t, err, "expected image to return a config file") + + assert.Equal(t, "root", configFile.Config.User, "user must be root") + assert.Equal(t, "/", configFile.Config.WorkingDir, "workdir must be /") + if assert.Len(t, configFile.Config.Entrypoint, 1) { + assert.Equal(t, "/.envbuilder/bin/envbuilder", configFile.Config.Entrypoint[0], "incorrect entrypoint") + } + + require.False(t, t.Failed(), "pushImage failed") + + return img +} + +func getCachedImage(ctx context.Context, t *testing.T, cli *client.Client, env ...string) name.Reference { + ctrID, err := runEnvbuilder(t, runOpts{env: append(env, envbuilderEnv("GET_CACHED_IMAGE", "1"))}) + require.NoError(t, err) + + logs, err := cli.ContainerLogs(ctx, ctrID, container.LogsOptions{ + ShowStdout: true, + ShowStderr: true, + }) + require.NoError(t, err) + defer logs.Close() + logBytes, err := io.ReadAll(logs) + require.NoError(t, err) + + re := regexp.MustCompile(`ENVBUILDER_CACHED_IMAGE=(\S+)`) + matches := re.FindStringSubmatch(string(logBytes)) + require.Len(t, matches, 2, "envbuilder cached image not found") + ref, err := name.ParseReference(matches[1]) + require.NoError(t, err, "failed to parse cached image reference") + return ref +} + +func startContainerFromRef(ctx context.Context, t *testing.T, cli *client.Client, ref name.Reference) container.CreateResponse { + // Ensure that we can pull the image. + rc, err := cli.ImagePull(ctx, ref.String(), image.PullOptions{}) + require.NoError(t, err) + t.Cleanup(func() { _ = rc.Close() }) + _, err = io.ReadAll(rc) + require.NoError(t, err) + + // Start the container. + ctr, err := cli.ContainerCreate(ctx, &container.Config{ + Image: ref.String(), + Labels: map[string]string{ + testContainerLabel: "true", + }, + }, nil, nil, nil, "") + require.NoError(t, err) + + t.Cleanup(func() { + // Start a new context to ensure that the container is removed. + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _ = cli.ContainerRemove(ctx, ctr.ID, container.RemoveOptions{ + RemoveVolumes: true, + Force: true, + }) + }) + + err = cli.ContainerStart(ctx, ctr.ID, container.StartOptions{}) + require.NoError(t, err) + + return ctr +} + type runOpts struct { binds []string env []string diff --git a/internal/magicdir/magicdir.go b/internal/magicdir/magicdir.go index 31bcd7c9..5e062514 100644 --- a/internal/magicdir/magicdir.go +++ b/internal/magicdir/magicdir.go @@ -76,3 +76,8 @@ func (m MagicDir) Built() string { func (m MagicDir) Image() string { return m.Join("image") } + +// Features is a directory that contains feature files. +func (m MagicDir) Features() string { + return m.Join("features") +} From b64b812c3026cb7fc777d22e078585a4a93dce64 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Wed, 25 Sep 2024 13:19:53 +0300 Subject: [PATCH 50/81] fix: allow caching run layers with no filesystem changes (#355) (cherry picked from commit 67cb94c574a1f3ff843e205f34c3f07fc963e00b) --- envbuilder.go | 46 +++++++++++++++--------------- integration/integration_test.go | 50 +++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 22 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index bc94d89c..b082ca80 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -456,17 +456,18 @@ func Run(ctx context.Context, opts options.Options) error { } kOpts := &config.KanikoOptions{ // Boilerplate! - CustomPlatform: platforms.Format(platforms.Normalize(platforms.DefaultSpec())), - SnapshotMode: "redo", - RunV2: true, - RunStdout: stdoutWriter, - RunStderr: stderrWriter, - Destinations: destinations, - NoPush: !opts.PushImage || len(destinations) == 0, - CacheRunLayers: true, - CacheCopyLayers: true, - CompressedCaching: true, - Compression: config.ZStd, + CustomPlatform: platforms.Format(platforms.Normalize(platforms.DefaultSpec())), + SnapshotMode: "redo", + RunV2: true, + RunStdout: stdoutWriter, + RunStderr: stderrWriter, + Destinations: destinations, + NoPush: !opts.PushImage || len(destinations) == 0, + CacheRunLayers: true, + CacheCopyLayers: true, + ForceBuildMetadata: opts.PushImage, // Force layers with no changes to be cached, required for cache probing. + CompressedCaching: true, + Compression: config.ZStd, // Maps to "default" level, ~100-300 MB/sec according to // benchmarks in klauspost/compress README // https://github.com/klauspost/compress/blob/67a538e2b4df11f8ec7139388838a13bce84b5d5/zstd/encoder_options.go#L188 @@ -1180,17 +1181,18 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) } kOpts := &config.KanikoOptions{ // Boilerplate! - CustomPlatform: platforms.Format(platforms.Normalize(platforms.DefaultSpec())), - SnapshotMode: "redo", - RunV2: true, - RunStdout: stdoutWriter, - RunStderr: stderrWriter, - Destinations: destinations, - NoPush: !opts.PushImage || len(destinations) == 0, - CacheRunLayers: true, - CacheCopyLayers: true, - CompressedCaching: true, - Compression: config.ZStd, + CustomPlatform: platforms.Format(platforms.Normalize(platforms.DefaultSpec())), + SnapshotMode: "redo", + RunV2: true, + RunStdout: stdoutWriter, + RunStderr: stderrWriter, + Destinations: destinations, + NoPush: true, + CacheRunLayers: true, + CacheCopyLayers: true, + ForceBuildMetadata: true, // Force layers with no changes to be cached, required for cache probing. + CompressedCaching: true, + Compression: config.ZStd, // Maps to "default" level, ~100-300 MB/sec according to // benchmarks in klauspost/compress README // https://github.com/klauspost/compress/blob/67a538e2b4df11f8ec7139388838a13bce84b5d5/zstd/encoder_options.go#L188 diff --git a/integration/integration_test.go b/integration/integration_test.go index f88829aa..42246f95 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -1312,6 +1312,56 @@ RUN date --utc > /root/date.txt`, testImageAlpine), require.NotEmpty(t, strings.TrimSpace(out)) }) + t.Run("CacheAndPushWithNoChangeLayers", func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ + "Dockerfile": fmt.Sprintf(` +FROM %[1]s +RUN touch /foo +RUN echo "Hi, please don't put me in a layer (I guess you won't listen to me...)" +RUN touch /bar +`, testImageAlpine), + }, + }) + + // Given: an empty registry + testReg := setupInMemoryRegistry(t, setupInMemoryRegistryOpts{}) + testRepo := testReg + "/test" + ref, err := name.ParseReference(testRepo + ":latest") + require.NoError(t, err) + _, err = remote.Image(ref) + require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") + + opts := []string{ + envbuilderEnv("GIT_URL", srv.URL), + envbuilderEnv("CACHE_REPO", testRepo), + envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), + } + + // When: we run envbuilder with PUSH_IMAGE set + _ = pushImage(t, ref, nil, opts...) + + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + require.NoError(t, err) + defer cli.Close() + + // Then: re-running envbuilder with GET_CACHED_IMAGE should succeed + cachedRef := getCachedImage(ctx, t, cli, opts...) + + // When: we run the image we just built + ctr := startContainerFromRef(ctx, t, cli, cachedRef) + + // Then: the envbuilder binary exists in the image! + out := execContainer(t, ctr.ID, "/.envbuilder/bin/envbuilder --help") + require.Regexp(t, `(?s)^USAGE:\s+envbuilder`, strings.TrimSpace(out)) + require.NotEmpty(t, strings.TrimSpace(out)) + }) + t.Run("CacheAndPushAuth", func(t *testing.T) { t.Parallel() From 2180807ae3a36bbbcc7d0c937ee81b8f06c285ef Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 25 Sep 2024 13:36:50 +0100 Subject: [PATCH 51/81] fix: update kaniko to add cache fix (#354) (cherry picked from commit 9c315aabfaef5bd599e7b7a05b20e909ef7f6dd3) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9b806fb3..b3fa7843 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22.4 // There are a few options we need added to Kaniko! // See: https://github.com/GoogleContainerTools/kaniko/compare/main...coder:kaniko:main -replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240830141327-f307586e3dca +replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240925122543-caa18967f374 // Required to import codersdk due to gvisor dependency. replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20240702054557-aa558fbe5374 diff --git a/go.sum b/go.sum index c1f1fcb5..07dc01db 100644 --- a/go.sum +++ b/go.sum @@ -171,8 +171,8 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352 h1:L/EjCuZxs5tOcqqCaASj/nu65TRYEFcTt8qRQfHZXX0= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352/go.mod h1:P1KoQSgnKEAG6Mnd3YlGzAophty+yKA9VV48LpfNRvo= -github.com/coder/kaniko v0.0.0-20240830141327-f307586e3dca h1:PrcSWrllqipTrtet50a3VyAJEQmjziIZyhpy0bsC6o0= -github.com/coder/kaniko v0.0.0-20240830141327-f307586e3dca/go.mod h1:XoTDIhNF0Ll4tLmRYdOn31udU9w5zFrY2PME/crSRCA= +github.com/coder/kaniko v0.0.0-20240925122543-caa18967f374 h1:/cyXf0vTSwFh7evQqeWHXXl14aRfC4CsNIYxOenJytQ= +github.com/coder/kaniko v0.0.0-20240925122543-caa18967f374/go.mod h1:XoTDIhNF0Ll4tLmRYdOn31udU9w5zFrY2PME/crSRCA= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 h1:3A0ES21Ke+FxEM8CXx9n47SZOKOpgSE1bbJzlE4qPVs= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0/go.mod h1:5UuS2Ts+nTToAMeOjNlnHFkPahrtDkmpydBen/3wgZc= github.com/coder/quartz v0.1.0 h1:cLL+0g5l7xTf6ordRnUMMiZtRE8Sq5LxpghS63vEXrQ= From 7955333b43df31b23bb167f43ba250de79546cdb Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Fri, 27 Sep 2024 16:41:37 +0300 Subject: [PATCH 52/81] fix: set env and run scripts when starting cached image (#359) (cherry picked from commit c11faf3a2ccceb91309f14057978c0ce0441a20b) --- envbuilder.go | 621 ++++++++++++++++++-------------- integration/integration_test.go | 144 +++++++- options/options.go | 4 + 3 files changed, 503 insertions(+), 266 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index b082ca80..8a1a0389 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -58,32 +58,65 @@ var ErrNoFallbackImage = errors.New("no fallback image has been specified") // DockerConfig represents the Docker configuration file. type DockerConfig configfile.ConfigFile +type runtimeDataStore struct { + // Runtime data. + Image bool `json:"-"` + Built bool `json:"-"` + SkippedRebuild bool `json:"-"` + Scripts devcontainer.LifecycleScripts `json:"-"` + ImageEnv []string `json:"-"` + ContainerEnv map[string]string `json:"-"` + RemoteEnv map[string]string `json:"-"` + DevcontainerPath string `json:"-"` + + // Data stored in the magic image file. + ContainerUser string `json:"container_user"` +} + +type execArgsInfo struct { + InitCommand string + InitArgs []string + UserInfo userInfo + Environ []string +} + // Run runs the envbuilder. // Logger is the logf to use for all operations. // Filesystem is the filesystem to use for all operations. // Defaults to the host filesystem. func Run(ctx context.Context, opts options.Options) error { - defer options.UnsetEnv() - if opts.GetCachedImage { - return fmt.Errorf("developer error: use RunCacheProbe instead") + var args execArgsInfo + // Run in a separate function to ensure all defers run before we + // setuid or exec. + err := run(ctx, opts, &args) + if err != nil { + return err } - if opts.CacheRepo == "" && opts.PushImage { - return fmt.Errorf("--cache-repo must be set when using --push-image") + err = syscall.Setgid(args.UserInfo.gid) + if err != nil { + return fmt.Errorf("set gid: %w", err) + } + err = syscall.Setuid(args.UserInfo.uid) + if err != nil { + return fmt.Errorf("set uid: %w", err) } - magicDir := magicdir.At(opts.MagicDirBase) + opts.Logger(log.LevelInfo, "=== Running the init command %s %+v as the %q user...", opts.InitCommand, args.InitArgs, args.UserInfo.user.Username) - // Default to the shell! - initArgs := []string{"-c", opts.InitScript} - if opts.InitArgs != "" { - var err error - initArgs, err = shellquote.Split(opts.InitArgs) - if err != nil { - return fmt.Errorf("parse init args: %w", err) - } + err = syscall.Exec(args.InitCommand, append([]string{args.InitCommand}, args.InitArgs...), args.Environ) + if err != nil { + return fmt.Errorf("exec init script: %w", err) } + return errors.New("exec failed") +} + +func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) error { + defer options.UnsetEnv() + + magicDir := magicdir.At(opts.MagicDirBase) + stageNumber := 0 startStage := func(format string, args ...any) func(format string, args ...any) { now := time.Now() @@ -96,6 +129,24 @@ func Run(ctx context.Context, opts options.Options) error { } } + if opts.GetCachedImage { + return fmt.Errorf("developer error: use RunCacheProbe instead") + } + if opts.CacheRepo == "" && opts.PushImage { + return fmt.Errorf("--cache-repo must be set when using --push-image") + } + + // Default to the shell. + execArgs.InitCommand = opts.InitCommand + execArgs.InitArgs = []string{"-c", opts.InitScript} + if opts.InitArgs != "" { + var err error + execArgs.InitArgs, err = shellquote.Split(opts.InitArgs) + if err != nil { + return fmt.Errorf("parse init args: %w", err) + } + } + opts.Logger(log.LevelInfo, "%s %s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder"), buildinfo.Version()) cleanupDockerConfigJSON, err := initDockerConfigJSON(opts.Logger, magicDir, opts.DockerConfigBase64) @@ -108,6 +159,30 @@ func Run(ctx context.Context, opts options.Options) error { } }() // best effort + runtimeData := runtimeDataStore{ + ContainerEnv: make(map[string]string), + RemoteEnv: make(map[string]string), + } + if fileExists(opts.Filesystem, magicDir.Image()) { + if err = parseMagicImageFile(opts.Filesystem, magicDir.Image(), &runtimeData); err != nil { + return fmt.Errorf("parse magic image file: %w", err) + } + runtimeData.Image = true + + // Some options are only applicable for builds. + if opts.RemoteRepoBuildMode { + opts.Logger(log.LevelDebug, "Ignoring %s option, it is not supported when using a pre-built image.", options.WithEnvPrefix("REMOTE_REPO_BUILD_MODE")) + opts.RemoteRepoBuildMode = false + } + if opts.ExportEnvFile != "" { + // Currently we can't support this as we don't have access to the + // post-build computed env vars to know which ones to export. + opts.Logger(log.LevelWarn, "Ignoring %s option, it is not supported when using a pre-built image.", options.WithEnvPrefix("EXPORT_ENV_FILE")) + opts.ExportEnvFile = "" + } + } + runtimeData.Built = fileExists(opts.Filesystem, magicDir.Built()) + buildTimeWorkspaceFolder := opts.WorkspaceFolder var fallbackErr error var cloned bool @@ -139,7 +214,9 @@ func Run(ctx context.Context, opts options.Options) error { } } else { opts.Logger(log.LevelError, "Failed to clone repository: %s", fallbackErr.Error()) - opts.Logger(log.LevelError, "Falling back to the default image...") + if !runtimeData.Image { + opts.Logger(log.LevelError, "Falling back to the default image...") + } } _ = w.Close() @@ -175,112 +252,106 @@ func Run(ctx context.Context, opts options.Options) error { } } - defaultBuildParams := func() (*devcontainer.Compiled, error) { - dockerfile := magicDir.Join("Dockerfile") - file, err := opts.Filesystem.OpenFile(dockerfile, os.O_CREATE|os.O_WRONLY, 0o644) - if err != nil { - return nil, err - } - defer file.Close() - if opts.FallbackImage == "" { - if fallbackErr != nil { - return nil, xerrors.Errorf("%s: %w", fallbackErr.Error(), ErrNoFallbackImage) - } - // We can't use errors.Join here because our tests - // don't support parsing a multiline error. - return nil, ErrNoFallbackImage - } - content := "FROM " + opts.FallbackImage - _, err = file.Write([]byte(content)) - if err != nil { - return nil, err - } - return &devcontainer.Compiled{ - DockerfilePath: dockerfile, - DockerfileContent: content, - BuildContext: magicDir.Path(), - }, nil - } - - var ( - buildParams *devcontainer.Compiled - scripts devcontainer.LifecycleScripts - - devcontainerPath string - ) - if opts.DockerfilePath == "" { - // Only look for a devcontainer if a Dockerfile wasn't specified. - // devcontainer is a standard, so it's reasonable to be the default. - var devcontainerDir string - var err error - devcontainerPath, devcontainerDir, err = findDevcontainerJSON(buildTimeWorkspaceFolder, opts) - if err != nil { - opts.Logger(log.LevelError, "Failed to locate devcontainer.json: %s", err.Error()) - opts.Logger(log.LevelError, "Falling back to the default image...") - } else { - // We know a devcontainer exists. - // Let's parse it and use it! - file, err := opts.Filesystem.Open(devcontainerPath) + if !runtimeData.Image { + defaultBuildParams := func() (*devcontainer.Compiled, error) { + dockerfile := magicDir.Join("Dockerfile") + file, err := opts.Filesystem.OpenFile(dockerfile, os.O_CREATE|os.O_WRONLY, 0o644) if err != nil { - return fmt.Errorf("open devcontainer.json: %w", err) + return nil, err } defer file.Close() - content, err := io.ReadAll(file) + if opts.FallbackImage == "" { + if fallbackErr != nil { + return nil, xerrors.Errorf("%s: %w", fallbackErr.Error(), ErrNoFallbackImage) + } + // We can't use errors.Join here because our tests + // don't support parsing a multiline error. + return nil, ErrNoFallbackImage + } + content := "FROM " + opts.FallbackImage + _, err = file.Write([]byte(content)) if err != nil { - return fmt.Errorf("read devcontainer.json: %w", err) + return nil, err } - devContainer, err := devcontainer.Parse(content) - if err == nil { - var fallbackDockerfile string - if !devContainer.HasImage() && !devContainer.HasDockerfile() { - defaultParams, err := defaultBuildParams() - if err != nil { - return fmt.Errorf("no Dockerfile or image found: %w", err) - } - opts.Logger(log.LevelInfo, "No Dockerfile or image specified; falling back to the default image...") - fallbackDockerfile = defaultParams.DockerfilePath + return &devcontainer.Compiled{ + DockerfilePath: dockerfile, + DockerfileContent: content, + BuildContext: magicDir.Path(), + }, nil + } + + var buildParams *devcontainer.Compiled + if opts.DockerfilePath == "" { + // Only look for a devcontainer if a Dockerfile wasn't specified. + // devcontainer is a standard, so it's reasonable to be the default. + var devcontainerDir string + var err error + runtimeData.DevcontainerPath, devcontainerDir, err = findDevcontainerJSON(buildTimeWorkspaceFolder, opts) + if err != nil { + opts.Logger(log.LevelError, "Failed to locate devcontainer.json: %s", err.Error()) + opts.Logger(log.LevelError, "Falling back to the default image...") + } else { + // We know a devcontainer exists. + // Let's parse it and use it! + file, err := opts.Filesystem.Open(runtimeData.DevcontainerPath) + if err != nil { + return fmt.Errorf("open devcontainer.json: %w", err) } - buildParams, err = devContainer.Compile(opts.Filesystem, devcontainerDir, magicDir.Path(), fallbackDockerfile, opts.WorkspaceFolder, false, os.LookupEnv) + defer file.Close() + content, err := io.ReadAll(file) if err != nil { - return fmt.Errorf("compile devcontainer.json: %w", err) + return fmt.Errorf("read devcontainer.json: %w", err) + } + devContainer, err := devcontainer.Parse(content) + if err == nil { + var fallbackDockerfile string + if !devContainer.HasImage() && !devContainer.HasDockerfile() { + defaultParams, err := defaultBuildParams() + if err != nil { + return fmt.Errorf("no Dockerfile or image found: %w", err) + } + opts.Logger(log.LevelInfo, "No Dockerfile or image specified; falling back to the default image...") + fallbackDockerfile = defaultParams.DockerfilePath + } + buildParams, err = devContainer.Compile(opts.Filesystem, devcontainerDir, magicDir.Path(), fallbackDockerfile, opts.WorkspaceFolder, false, os.LookupEnv) + if err != nil { + return fmt.Errorf("compile devcontainer.json: %w", err) + } + if buildParams.User != "" { + runtimeData.ContainerUser = buildParams.User + } + runtimeData.Scripts = devContainer.LifecycleScripts + } else { + opts.Logger(log.LevelError, "Failed to parse devcontainer.json: %s", err.Error()) + opts.Logger(log.LevelError, "Falling back to the default image...") } - scripts = devContainer.LifecycleScripts - } else { - opts.Logger(log.LevelError, "Failed to parse devcontainer.json: %s", err.Error()) - opts.Logger(log.LevelError, "Falling back to the default image...") } - } - } else { - // If a Dockerfile was specified, we use that. - dockerfilePath := filepath.Join(buildTimeWorkspaceFolder, opts.DockerfilePath) - - // If the dockerfilePath is specified and deeper than the base of WorkspaceFolder AND the BuildContextPath is - // not defined, show a warning - dockerfileDir := filepath.Dir(dockerfilePath) - if dockerfileDir != filepath.Clean(buildTimeWorkspaceFolder) && opts.BuildContextPath == "" { - opts.Logger(log.LevelWarn, "given dockerfile %q is below %q and no custom build context has been defined", dockerfilePath, buildTimeWorkspaceFolder) - opts.Logger(log.LevelWarn, "\t-> set BUILD_CONTEXT_PATH to %q to fix", dockerfileDir) - } - - dockerfile, err := opts.Filesystem.Open(dockerfilePath) - if err == nil { - content, err := io.ReadAll(dockerfile) - if err != nil { - return fmt.Errorf("read Dockerfile: %w", err) + } else { + // If a Dockerfile was specified, we use that. + dockerfilePath := filepath.Join(buildTimeWorkspaceFolder, opts.DockerfilePath) + + // If the dockerfilePath is specified and deeper than the base of WorkspaceFolder AND the BuildContextPath is + // not defined, show a warning + dockerfileDir := filepath.Dir(dockerfilePath) + if dockerfileDir != filepath.Clean(buildTimeWorkspaceFolder) && opts.BuildContextPath == "" { + opts.Logger(log.LevelWarn, "given dockerfile %q is below %q and no custom build context has been defined", dockerfilePath, buildTimeWorkspaceFolder) + opts.Logger(log.LevelWarn, "\t-> set BUILD_CONTEXT_PATH to %q to fix", dockerfileDir) } - buildParams = &devcontainer.Compiled{ - DockerfilePath: dockerfilePath, - DockerfileContent: string(content), - BuildContext: filepath.Join(buildTimeWorkspaceFolder, opts.BuildContextPath), + + dockerfile, err := opts.Filesystem.Open(dockerfilePath) + if err == nil { + content, err := io.ReadAll(dockerfile) + if err != nil { + return fmt.Errorf("read Dockerfile: %w", err) + } + buildParams = &devcontainer.Compiled{ + DockerfilePath: dockerfilePath, + DockerfileContent: string(content), + BuildContext: filepath.Join(buildTimeWorkspaceFolder, opts.BuildContextPath), + } } } - } - var ( - username string - skippedRebuild bool - ) - if _, err := os.Stat(magicDir.Image()); errors.Is(err, fs.ErrNotExist) { if buildParams == nil { // If there isn't a devcontainer.json file in the repository, // we fallback to whatever the `DefaultImage` is. @@ -385,7 +456,7 @@ func Run(ctx context.Context, opts options.Options) error { // Since the user in the image is set to root, we also store the user // in the magic file to be used by envbuilder when the image is run. opts.Logger(log.LevelDebug, "writing magic image file at %q in build context %q", magicImageDest, magicTempDir) - if err := writeFile(opts.Filesystem, magicImageDest, 0o755, fmt.Sprintf("USER=%s\n", buildParams.User)); err != nil { + if err := writeMagicImageFile(opts.Filesystem, magicImageDest, runtimeData); err != nil { return fmt.Errorf("write magic image file in build context: %w", err) } } @@ -411,9 +482,7 @@ func Run(ctx context.Context, opts options.Options) error { defer closeStderr() build := func() (v1.Image, error) { defer cleanupBuildContext() - _, alreadyBuiltErr := opts.Filesystem.Stat(magicDir.Built()) - _, isImageErr := opts.Filesystem.Stat(magicDir.Image()) - if (alreadyBuiltErr == nil && opts.SkipRebuild) || isImageErr == nil { + if runtimeData.Built && opts.SkipRebuild { endStage := startStage("🏗️ Skipping build because of cache...") imageRef, err := devcontainer.ImageFromDockerfile(buildParams.DockerfileContent) if err != nil { @@ -424,7 +493,8 @@ func Run(ctx context.Context, opts options.Options) error { return nil, fmt.Errorf("image from remote: %w", err) } endStage("🏗️ Found image from remote!") - skippedRebuild = true + runtimeData.Built = false + runtimeData.SkippedRebuild = true return image, nil } @@ -556,23 +626,17 @@ func Run(ctx context.Context, opts options.Options) error { return fmt.Errorf("restore mounts: %w", err) } - // Create the magic file to indicate that this build - // has already been ran before! - file, err := opts.Filesystem.Create(magicDir.Built()) - if err != nil { - return fmt.Errorf("create magic file: %w", err) - } - _ = file.Close() - configFile, err := image.ConfigFile() if err != nil { return fmt.Errorf("get image config: %w", err) } - containerEnv := make(map[string]string) - remoteEnv := make(map[string]string) + runtimeData.ImageEnv = configFile.Config.Env - // devcontainer metadata can be persisted through a standard label + // Dev Container metadata can be persisted through a standard label. + // Note that this currently only works when we're building the image, + // not when we're using a pre-built image as we don't have access to + // labels. devContainerMetadata, exists := configFile.Config.Labels["devcontainer.metadata"] if exists { var devContainer []*devcontainer.Spec @@ -586,117 +650,130 @@ func Run(ctx context.Context, opts options.Options) error { } opts.Logger(log.LevelInfo, "#%d: 👀 Found devcontainer.json label metadata in image...", stageNumber) for _, container := range devContainer { - if container.RemoteUser != "" { - opts.Logger(log.LevelInfo, "#%d: 🧑 Updating the user to %q!", stageNumber, container.RemoteUser) + if container.ContainerUser != "" { + opts.Logger(log.LevelInfo, "#%d: 🧑 Updating the user to %q!", stageNumber, container.ContainerUser) - configFile.Config.User = container.RemoteUser + configFile.Config.User = container.ContainerUser } - maps.Copy(containerEnv, container.ContainerEnv) - maps.Copy(remoteEnv, container.RemoteEnv) + maps.Copy(runtimeData.ContainerEnv, container.ContainerEnv) + maps.Copy(runtimeData.RemoteEnv, container.RemoteEnv) if !container.OnCreateCommand.IsEmpty() { - scripts.OnCreateCommand = container.OnCreateCommand + runtimeData.Scripts.OnCreateCommand = container.OnCreateCommand } if !container.UpdateContentCommand.IsEmpty() { - scripts.UpdateContentCommand = container.UpdateContentCommand + runtimeData.Scripts.UpdateContentCommand = container.UpdateContentCommand } if !container.PostCreateCommand.IsEmpty() { - scripts.PostCreateCommand = container.PostCreateCommand + runtimeData.Scripts.PostCreateCommand = container.PostCreateCommand } if !container.PostStartCommand.IsEmpty() { - scripts.PostStartCommand = container.PostStartCommand + runtimeData.Scripts.PostStartCommand = container.PostStartCommand } } } - // Sanitize the environment of any opts! - options.UnsetEnv() - - // Remove the Docker config secret file! - if err := cleanupDockerConfigJSON(); err != nil { - return err + maps.Copy(runtimeData.ContainerEnv, buildParams.ContainerEnv) + maps.Copy(runtimeData.RemoteEnv, buildParams.RemoteEnv) + if runtimeData.ContainerUser == "" && configFile.Config.User != "" { + runtimeData.ContainerUser = configFile.Config.User } - - environ, err := os.ReadFile("/etc/environment") + } else { + runtimeData.DevcontainerPath, _, err = findDevcontainerJSON(opts.WorkspaceFolder, opts) if err == nil { - for _, env := range strings.Split(string(environ), "\n") { - pair := strings.SplitN(env, "=", 2) - if len(pair) != 2 { - continue + file, err := opts.Filesystem.Open(runtimeData.DevcontainerPath) + if err != nil { + return fmt.Errorf("open devcontainer.json: %w", err) + } + defer file.Close() + content, err := io.ReadAll(file) + if err != nil { + return fmt.Errorf("read devcontainer.json: %w", err) + } + devContainer, err := devcontainer.Parse(content) + if err == nil { + maps.Copy(runtimeData.ContainerEnv, devContainer.ContainerEnv) + maps.Copy(runtimeData.RemoteEnv, devContainer.RemoteEnv) + if devContainer.ContainerUser != "" { + runtimeData.ContainerUser = devContainer.ContainerUser } - os.Setenv(pair[0], pair[1]) + runtimeData.Scripts = devContainer.LifecycleScripts + } else { + opts.Logger(log.LevelError, "Failed to parse devcontainer.json: %s", err.Error()) } } + } - allEnvKeys := make(map[string]struct{}) + // Sanitize the environment of any opts! + options.UnsetEnv() - // It must be set in this parent process otherwise nothing will be found! - for _, env := range configFile.Config.Env { - pair := strings.SplitN(env, "=", 2) - os.Setenv(pair[0], pair[1]) - allEnvKeys[pair[0]] = struct{}{} - } - maps.Copy(containerEnv, buildParams.ContainerEnv) - maps.Copy(remoteEnv, buildParams.RemoteEnv) + // Set the environment from /etc/environment first, so it can be + // overridden by the image and devcontainer settings. + err = setEnvFromEtcEnvironment(opts.Logger) + if err != nil { + return fmt.Errorf("set env from /etc/environment: %w", err) + } - // Set Envbuilder runtime markers - containerEnv["ENVBUILDER"] = "true" - if devcontainerPath != "" { - containerEnv["DEVCONTAINER"] = "true" - containerEnv["DEVCONTAINER_CONFIG"] = devcontainerPath - } + allEnvKeys := make(map[string]struct{}) - for _, env := range []map[string]string{containerEnv, remoteEnv} { - envKeys := make([]string, 0, len(env)) - for key := range env { - envKeys = append(envKeys, key) - allEnvKeys[key] = struct{}{} - } - sort.Strings(envKeys) - for _, envVar := range envKeys { - value := devcontainer.SubstituteVars(env[envVar], opts.WorkspaceFolder, os.LookupEnv) - os.Setenv(envVar, value) - } - } + // It must be set in this parent process otherwise nothing will be found! + for _, env := range runtimeData.ImageEnv { + pair := strings.SplitN(env, "=", 2) + os.Setenv(pair[0], pair[1]) + allEnvKeys[pair[0]] = struct{}{} + } - // Do not export env if we skipped a rebuild, because ENV directives - // from the Dockerfile would not have been processed and we'd miss these - // in the export. We should have generated a complete set of environment - // on the intial build, so exporting environment variables a second time - // isn't useful anyway. - if opts.ExportEnvFile != "" && !skippedRebuild { - exportEnvFile, err := os.Create(opts.ExportEnvFile) - if err != nil { - return fmt.Errorf("failed to open EXPORT_ENV_FILE %q: %w", opts.ExportEnvFile, err) - } + // Set Envbuilder runtime markers + runtimeData.ContainerEnv["ENVBUILDER"] = "true" + if runtimeData.DevcontainerPath != "" { + runtimeData.ContainerEnv["DEVCONTAINER"] = "true" + runtimeData.ContainerEnv["DEVCONTAINER_CONFIG"] = runtimeData.DevcontainerPath + } - envKeys := make([]string, 0, len(allEnvKeys)) - for key := range allEnvKeys { - envKeys = append(envKeys, key) - } - sort.Strings(envKeys) - for _, key := range envKeys { - fmt.Fprintf(exportEnvFile, "%s=%s\n", key, os.Getenv(key)) - } + for _, env := range []map[string]string{runtimeData.ContainerEnv, runtimeData.RemoteEnv} { + envKeys := make([]string, 0, len(env)) + for key := range env { + envKeys = append(envKeys, key) + allEnvKeys[key] = struct{}{} + } + sort.Strings(envKeys) + for _, envVar := range envKeys { + value := devcontainer.SubstituteVars(env[envVar], opts.WorkspaceFolder, os.LookupEnv) + os.Setenv(envVar, value) + } + } - exportEnvFile.Close() + // Do not export env if we skipped a rebuild, because ENV directives + // from the Dockerfile would not have been processed and we'd miss these + // in the export. We should have generated a complete set of environment + // on the intial build, so exporting environment variables a second time + // isn't useful anyway. + if opts.ExportEnvFile != "" && !runtimeData.SkippedRebuild { + exportEnvFile, err := opts.Filesystem.Create(opts.ExportEnvFile) + if err != nil { + return fmt.Errorf("failed to open %s %q: %w", options.WithEnvPrefix("EXPORT_ENV_FILE"), opts.ExportEnvFile, err) } - username = configFile.Config.User - if buildParams.User != "" { - username = buildParams.User + envKeys := make([]string, 0, len(allEnvKeys)) + for key := range allEnvKeys { + envKeys = append(envKeys, key) } - } else { - skippedRebuild = true - magicEnv, err := parseMagicImageFile(opts.Filesystem, magicDir.Image()) - if err != nil { - return fmt.Errorf("parse magic env: %w", err) + sort.Strings(envKeys) + for _, key := range envKeys { + fmt.Fprintf(exportEnvFile, "%s=%s\n", key, os.Getenv(key)) } - username = magicEnv["USER"] + + exportEnvFile.Close() } - if username == "" { + + // Remove the Docker config secret file! + if err := cleanupDockerConfigJSON(); err != nil { + return err + } + + if runtimeData.ContainerUser == "" { opts.Logger(log.LevelWarn, "#%d: no user specified, using root", stageNumber) } - userInfo, err := getUser(username) + execArgs.UserInfo, err = getUser(runtimeData.ContainerUser) if err != nil { return fmt.Errorf("update user: %w", err) } @@ -714,9 +791,9 @@ func Run(ctx context.Context, opts options.Options) error { if err != nil { return err } - return os.Chown(path, userInfo.uid, userInfo.gid) + return os.Chown(path, execArgs.UserInfo.uid, execArgs.UserInfo.gid) }); chownErr != nil { - opts.Logger(log.LevelError, "chown %q: %s", userInfo.user.HomeDir, chownErr.Error()) + opts.Logger(log.LevelError, "chown %q: %s", execArgs.UserInfo.user.HomeDir, chownErr.Error()) endStage("⚠️ Failed to the ownership of the workspace, you may need to fix this manually!") } else { endStage("👤 Updated the ownership of the workspace!") @@ -725,22 +802,22 @@ func Run(ctx context.Context, opts options.Options) error { // We may also need to update the ownership of the user homedir. // Skip this step if the user is root. - if userInfo.uid != 0 { - endStage := startStage("🔄 Updating ownership of %s...", userInfo.user.HomeDir) - if chownErr := filepath.Walk(userInfo.user.HomeDir, func(path string, _ fs.FileInfo, err error) error { + if execArgs.UserInfo.uid != 0 { + endStage := startStage("🔄 Updating ownership of %s...", execArgs.UserInfo.user.HomeDir) + if chownErr := filepath.Walk(execArgs.UserInfo.user.HomeDir, func(path string, _ fs.FileInfo, err error) error { if err != nil { return err } - return os.Chown(path, userInfo.uid, userInfo.gid) + return os.Chown(path, execArgs.UserInfo.uid, execArgs.UserInfo.gid) }); chownErr != nil { - opts.Logger(log.LevelError, "chown %q: %s", userInfo.user.HomeDir, chownErr.Error()) - endStage("⚠️ Failed to update ownership of %s, you may need to fix this manually!", userInfo.user.HomeDir) + opts.Logger(log.LevelError, "chown %q: %s", execArgs.UserInfo.user.HomeDir, chownErr.Error()) + endStage("⚠️ Failed to update ownership of %s, you may need to fix this manually!", execArgs.UserInfo.user.HomeDir) } else { - endStage("🏡 Updated ownership of %s!", userInfo.user.HomeDir) + endStage("🏡 Updated ownership of %s!", execArgs.UserInfo.user.HomeDir) } } - err = os.MkdirAll(opts.WorkspaceFolder, 0o755) + err = opts.Filesystem.MkdirAll(opts.WorkspaceFolder, 0o755) if err != nil { return fmt.Errorf("create workspace folder: %w", err) } @@ -755,11 +832,21 @@ func Run(ctx context.Context, opts options.Options) error { // example, TARGET_USER may be set to root in the case where we will // exec systemd as the init command, but that doesn't mean we should // run the lifecycle scripts as root. - os.Setenv("HOME", userInfo.user.HomeDir) - if err := execLifecycleScripts(ctx, opts, scripts, skippedRebuild, userInfo); err != nil { + os.Setenv("HOME", execArgs.UserInfo.user.HomeDir) + if err := execLifecycleScripts(ctx, opts, runtimeData.Scripts, !runtimeData.Built, execArgs.UserInfo); err != nil { return err } + // Create the magic file to indicate that this build + // has already been ran before! + if !runtimeData.Built { + file, err := opts.Filesystem.Create(magicDir.Built()) + if err != nil { + return fmt.Errorf("create magic file: %w", err) + } + _ = file.Close() + } + // The setup script can specify a custom initialization command // and arguments to run instead of the default shell. // @@ -773,7 +860,7 @@ func Run(ctx context.Context, opts options.Options) error { envKey := "ENVBUILDER_ENV" envFile := magicDir.Join("environ") - file, err := os.Create(envFile) + file, err := opts.Filesystem.Create(envFile) if err != nil { return fmt.Errorf("create environ file: %w", err) } @@ -782,7 +869,7 @@ func Run(ctx context.Context, opts options.Options) error { cmd := exec.CommandContext(ctx, "/bin/sh", "-c", opts.SetupScript) cmd.Env = append(os.Environ(), fmt.Sprintf("%s=%s", envKey, envFile), - fmt.Sprintf("TARGET_USER=%s", userInfo.user.Username), + fmt.Sprintf("TARGET_USER=%s", execArgs.UserInfo.user.Username), ) cmd.Dir = opts.WorkspaceFolder // This allows for a really nice and clean experience to experiement with! @@ -826,16 +913,16 @@ func Run(ctx context.Context, opts options.Options) error { key := pair[0] switch key { case "INIT_COMMAND": - opts.InitCommand = pair[1] + execArgs.InitCommand = pair[1] updatedCommand = true case "INIT_ARGS": - initArgs, err = shellquote.Split(pair[1]) + execArgs.InitArgs, err = shellquote.Split(pair[1]) if err != nil { return fmt.Errorf("split init args: %w", err) } updatedArgs = true case "TARGET_USER": - userInfo, err = getUser(pair[1]) + execArgs.UserInfo, err = getUser(pair[1]) if err != nil { return fmt.Errorf("update user: %w", err) } @@ -846,28 +933,16 @@ func Run(ctx context.Context, opts options.Options) error { if updatedCommand && !updatedArgs { // Because our default is a shell we need to empty the args // if the command was updated. This a tragic hack, but it works. - initArgs = []string{} + execArgs.InitArgs = []string{} } } // Hop into the user that should execute the initialize script! - os.Setenv("HOME", userInfo.user.HomeDir) - - err = syscall.Setgid(userInfo.gid) - if err != nil { - return fmt.Errorf("set gid: %w", err) - } - err = syscall.Setuid(userInfo.uid) - if err != nil { - return fmt.Errorf("set uid: %w", err) - } + os.Setenv("HOME", execArgs.UserInfo.user.HomeDir) - opts.Logger(log.LevelInfo, "=== Running the init command %s %+v as the %q user...", opts.InitCommand, initArgs, userInfo.user.Username) + // Set last to ensure all environment changes are complete. + execArgs.Environ = os.Environ() - err = syscall.Exec(opts.InitCommand, append([]string{opts.InitCommand}, initArgs...), os.Environ()) - if err != nil { - return fmt.Errorf("exec init script: %w", err) - } return nil } @@ -1157,7 +1232,8 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) // Since the user in the image is set to root, we also store the user // in the magic file to be used by envbuilder when the image is run. opts.Logger(log.LevelDebug, "writing magic image file at %q in build context %q", magicImageDest, magicTempDir) - if err := writeFile(opts.Filesystem, magicImageDest, 0o755, fmt.Sprintf("USER=%s\n", buildParams.User)); err != nil { + runtimeData := runtimeDataStore{ContainerUser: buildParams.User} + if err := writeMagicImageFile(opts.Filesystem, magicImageDest, runtimeData); err != nil { return nil, fmt.Errorf("write magic image file in build context: %w", err) } @@ -1241,6 +1317,25 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) return image, nil } +func setEnvFromEtcEnvironment(logf log.Func) error { + environ, err := os.ReadFile("/etc/environment") + if errors.Is(err, os.ErrNotExist) { + logf(log.LevelDebug, "Not loading environment from /etc/environment, file does not exist") + return nil + } + if err != nil { + return err + } + for _, env := range strings.Split(string(environ), "\n") { + pair := strings.SplitN(env, "=", 2) + if len(pair) != 2 { + continue + } + os.Setenv(pair[0], pair[1]) + } + return nil +} + type userInfo struct { uid int gid int @@ -1311,14 +1406,14 @@ func execLifecycleScripts( ctx context.Context, options options.Options, scripts devcontainer.LifecycleScripts, - skippedRebuild bool, + firstStart bool, userInfo userInfo, ) error { if options.PostStartScriptPath != "" { _ = os.Remove(options.PostStartScriptPath) } - if !skippedRebuild { + if firstStart { if err := execOneLifecycleScript(ctx, options.Logger, scripts.OnCreateCommand, "onCreateCommand", userInfo); err != nil { // skip remaining lifecycle commands return nil @@ -1466,6 +1561,11 @@ func maybeDeleteFilesystem(logger log.Func, force bool) error { return util.DeleteFilesystem() } +func fileExists(fs billy.Filesystem, path string) bool { + _, err := fs.Stat(path) + return err == nil +} + func copyFile(fs billy.Filesystem, src, dst string, mode fs.FileMode) error { srcF, err := fs.Open(src) if err != nil { @@ -1490,43 +1590,36 @@ func copyFile(fs billy.Filesystem, src, dst string, mode fs.FileMode) error { return nil } -func writeFile(fs billy.Filesystem, dst string, mode fs.FileMode, content string) error { - f, err := fs.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode) +func writeMagicImageFile(fs billy.Filesystem, path string, v any) error { + file, err := fs.Create(path) if err != nil { - return fmt.Errorf("open file: %w", err) + return fmt.Errorf("create magic image file: %w", err) } - defer f.Close() - _, err = f.Write([]byte(content)) - if err != nil { - return fmt.Errorf("write file: %w", err) + defer file.Close() + + enc := json.NewEncoder(file) + enc.SetIndent("", " ") + if err := enc.Encode(v); err != nil { + return fmt.Errorf("encode magic image file: %w", err) } + return nil } -func parseMagicImageFile(fs billy.Filesystem, path string) (map[string]string, error) { +func parseMagicImageFile(fs billy.Filesystem, path string, v any) error { file, err := fs.Open(path) if err != nil { - return nil, fmt.Errorf("open magic image file: %w", err) + return fmt.Errorf("open magic image file: %w", err) } defer file.Close() - env := make(map[string]string) - s := bufio.NewScanner(file) - for s.Scan() { - line := strings.TrimSpace(s.Text()) - if line == "" { - continue - } - parts := strings.SplitN(line, "=", 2) - if len(parts) != 2 { - return nil, fmt.Errorf("invalid magic image file format: %q", line) - } - env[parts[0]] = parts[1] + dec := json.NewDecoder(file) + dec.DisallowUnknownFields() + if err := dec.Decode(v); err != nil { + return fmt.Errorf("decode magic image file: %w", err) } - if err := s.Err(); err != nil { - return nil, fmt.Errorf("scan magic image file: %w", err) - } - return env, nil + + return nil } func initDockerConfigJSON(logf log.Func, magicDir magicdir.MagicDir, dockerConfigBase64 string) (func() error, error) { diff --git a/integration/integration_test.go b/integration/integration_test.go index 42246f95..66dfe846 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -18,6 +18,7 @@ import ( "os/exec" "path/filepath" "regexp" + "slices" "strings" "testing" "time" @@ -39,6 +40,7 @@ import ( "github.com/docker/docker/api/types/volume" "github.com/docker/docker/client" "github.com/docker/docker/pkg/stdcopy" + "github.com/google/go-cmp/cmp" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/registry" @@ -1312,6 +1314,133 @@ RUN date --utc > /root/date.txt`, testImageAlpine), require.NotEmpty(t, strings.TrimSpace(out)) }) + t.Run("CompareBuiltAndCachedImageEnvironment", func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + wantSpecificOutput := []string{ + "containeruser", + "FROM_CONTAINER=container", + "FROM_CONTAINER_ENV=containerEnv", + "FROM_REMOTE_ENV=remoteEnv", + "CONTAINER_OVERRIDE_C=containerEnv", + "CONTAINER_OVERRIDE_CR=remoteEnv", + "CONTAINER_OVERRIDE_R=remoteEnv", + } + + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ + ".devcontainer/Dockerfile": fmt.Sprintf(` + FROM %s + ENV FROM_CONTAINER=container + ENV CONTAINER_OVERRIDE_C=container + ENV CONTAINER_OVERRIDE_CR=container + ENV CONTAINER_OVERRIDE_R=container + RUN adduser -D containeruser + RUN adduser -D remoteuser + USER root + `, testImageAlpine), + ".devcontainer/devcontainer.json": ` + { + "dockerFile": "Dockerfile", + "containerUser": "containeruser", + "containerEnv": { + "FROM_CONTAINER_ENV": "containerEnv", + "CONTAINER_OVERRIDE_C": "containerEnv", + "CONTAINER_OVERRIDE_CR": "containerEnv", + }, + "remoteUser": "remoteuser", + "remoteEnv": { + "FROM_REMOTE_ENV": "remoteEnv", + "CONTAINER_OVERRIDE_CR": "remoteEnv", + "CONTAINER_OVERRIDE_R": "remoteEnv", + }, + "onCreateCommand": "echo onCreateCommand", + "postCreateCommand": "echo postCreateCommand", + } + `, + }, + }) + + // Given: an empty registry + testReg := setupInMemoryRegistry(t, setupInMemoryRegistryOpts{}) + testRepo := testReg + "/test" + ref, err := name.ParseReference(testRepo + ":latest") + require.NoError(t, err) + _, err = remote.Image(ref) + require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") + + opts := []string{ + envbuilderEnv("GIT_URL", srv.URL), + envbuilderEnv("CACHE_REPO", testRepo), + envbuilderEnv("INIT_SCRIPT", "echo '[start]' && whoami && env && echo '[end]'"), + envbuilderEnv("INIT_COMMAND", "/bin/ash"), + } + + // When: we run envbuilder with PUSH_IMAGE set + ctrID, err := runEnvbuilder(t, runOpts{env: append(opts, envbuilderEnv("PUSH_IMAGE", "1"))}) + require.NoError(t, err, "envbuilder push image failed") + + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + require.NoError(t, err) + defer cli.Close() + + var started bool + var wantOutput, gotOutput []string + logs, _ := streamContainerLogs(t, cli, ctrID) + for { + log := <-logs + if log == "[start]" { + started = true + continue + } + if log == "[end]" { + break + } + if started { + wantOutput = append(wantOutput, log) + } + } + started = false + + // Then: re-running envbuilder with GET_CACHED_IMAGE should succeed + cachedRef := getCachedImage(ctx, t, cli, opts...) + + // When: we run the image we just built + ctrID, err = runEnvbuilder(t, runOpts{ + image: cachedRef.String(), + env: opts, + }) + require.NoError(t, err, "envbuilder run cached image failed") + + logs, _ = streamContainerLogs(t, cli, ctrID) + for { + log := <-logs + if log == "[start]" { + started = true + continue + } + if log == "[end]" { + break + } + if started { + gotOutput = append(gotOutput, log) + } + } + + slices.Sort(wantOutput) + slices.Sort(gotOutput) + if diff := cmp.Diff(wantOutput, gotOutput); diff != "" { + t.Fatalf("unexpected output (-want +got):\n%s", diff) + } + + for _, want := range wantSpecificOutput { + assert.Contains(t, gotOutput, want, "expected specific output %q to be present", want) + } + }) + t.Run("CacheAndPushWithNoChangeLayers", func(t *testing.T) { t.Parallel() @@ -2003,7 +2132,7 @@ func startContainerFromRef(ctx context.Context, t *testing.T, cli *client.Client rc, err := cli.ImagePull(ctx, ref.String(), image.PullOptions{}) require.NoError(t, err) t.Cleanup(func() { _ = rc.Close() }) - _, err = io.ReadAll(rc) + _, err = io.Copy(io.Discard, rc) require.NoError(t, err) // Start the container. @@ -2033,6 +2162,7 @@ func startContainerFromRef(ctx context.Context, t *testing.T, cli *client.Client } type runOpts struct { + image string binds []string env []string volumes map[string]string @@ -2063,8 +2193,18 @@ func runEnvbuilder(t *testing.T, opts runOpts) (string, error) { _ = cli.VolumeRemove(ctx, volName, true) }) } + img := "envbuilder:latest" + if opts.image != "" { + // Pull the image first so we can start it afterwards. + rc, err := cli.ImagePull(ctx, opts.image, image.PullOptions{}) + require.NoError(t, err, "failed to pull image") + t.Cleanup(func() { _ = rc.Close() }) + _, err = io.Copy(io.Discard, rc) + require.NoError(t, err, "failed to read image pull response") + img = opts.image + } ctr, err := cli.ContainerCreate(ctx, &container.Config{ - Image: "envbuilder:latest", + Image: img, Env: opts.env, Labels: map[string]string{ testContainerLabel: "true", diff --git a/options/options.go b/options/options.go index 5b2586c7..18bd56d1 100644 --- a/options/options.go +++ b/options/options.go @@ -573,4 +573,8 @@ func UnsetEnv() { _ = os.Unsetenv(opt.Env) _ = os.Unsetenv(strings.TrimPrefix(opt.Env, envPrefix)) } + + // Unset the Kaniko environment variable which we set it in the + // Dockerfile to ensure correct behavior during building. + _ = os.Unsetenv("KANIKO_DIR") } From 9ee20e77a78067c5058a4016030004c54f674790 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 27 Sep 2024 17:33:10 +0100 Subject: [PATCH 53/81] fix: add preExec to Run (#368) Fixes #364 Adds the capability for Run() to run defers defined outside of Run(). Currently the only use-case for this is to close the Coder logger. Also adds an integration test that validates this behaviour and ensures that closeLogs gets called. Co-authored-by: Mathias Fredriksson (cherry picked from commit 481e3a90135d4b87dd65921b46d925c062a19436) --- cmd/envbuilder/main.go | 12 +++++- envbuilder.go | 7 +++- integration/integration_test.go | 67 +++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/cmd/envbuilder/main.go b/cmd/envbuilder/main.go index 410e0897..720c0c85 100644 --- a/cmd/envbuilder/main.go +++ b/cmd/envbuilder/main.go @@ -37,6 +37,12 @@ func envbuilderCmd() serpent.Command { Options: o.CLI(), Handler: func(inv *serpent.Invocation) error { o.SetDefaults() + var preExec []func() + defer func() { // Ensure cleanup in case of error. + for _, fn := range preExec { + fn() + } + }() o.Logger = log.New(os.Stderr, o.Verbose) if o.CoderAgentURL != "" { if o.CoderAgentToken == "" { @@ -50,6 +56,10 @@ func envbuilderCmd() serpent.Command { if err == nil { o.Logger = log.Wrap(o.Logger, coderLog) defer closeLogs() + preExec = append(preExec, func() { + o.Logger(log.LevelInfo, "Closing logs") + closeLogs() + }) // This adds the envbuilder subsystem. // If telemetry is enabled in a Coder deployment, // this will be reported and help us understand @@ -78,7 +88,7 @@ func envbuilderCmd() serpent.Command { return nil } - err := envbuilder.Run(inv.Context(), o) + err := envbuilder.Run(inv.Context(), o, preExec...) if err != nil { o.Logger(log.LevelError, "error: %s", err) } diff --git a/envbuilder.go b/envbuilder.go index 8a1a0389..3cea4f65 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -84,7 +84,9 @@ type execArgsInfo struct { // Logger is the logf to use for all operations. // Filesystem is the filesystem to use for all operations. // Defaults to the host filesystem. -func Run(ctx context.Context, opts options.Options) error { +// preExec are any functions that should be called before exec'ing the init +// command. This is useful for ensuring that defers get run. +func Run(ctx context.Context, opts options.Options, preExec ...func()) error { var args execArgsInfo // Run in a separate function to ensure all defers run before we // setuid or exec. @@ -103,6 +105,9 @@ func Run(ctx context.Context, opts options.Options) error { } opts.Logger(log.LevelInfo, "=== Running the init command %s %+v as the %q user...", opts.InitCommand, args.InitArgs, args.UserInfo.user.Username) + for _, fn := range preExec { + fn() + } err = syscall.Exec(args.InitCommand, append([]string{args.InitCommand}, args.InitArgs...), args.Environ) if err != nil { diff --git a/integration/integration_test.go b/integration/integration_test.go index 66dfe846..79b678d5 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -23,6 +23,8 @@ import ( "testing" "time" + "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/codersdk/agentsdk" "github.com/coder/envbuilder" "github.com/coder/envbuilder/devcontainer/features" "github.com/coder/envbuilder/internal/magicdir" @@ -58,6 +60,71 @@ const ( testImageUbuntu = "localhost:5000/envbuilder-test-ubuntu:latest" ) +func TestLogs(t *testing.T) { + t.Parallel() + + token := uuid.NewString() + logsDone := make(chan struct{}) + + logHandler := func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v2/buildinfo": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"version": "v2.8.9"}`)) + return + case "/api/v2/workspaceagents/me/logs": + w.WriteHeader(http.StatusOK) + tokHdr := r.Header.Get(codersdk.SessionTokenHeader) + assert.Equal(t, token, tokHdr) + var req agentsdk.PatchLogs + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + for _, log := range req.Logs { + t.Logf("got log: %+v", log) + if strings.Contains(log.Output, "Closing logs") { + close(logsDone) + return + } + } + return + default: + t.Errorf("unexpected request to %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + return + } + } + logSrv := httptest.NewServer(http.HandlerFunc(logHandler)) + defer logSrv.Close() + + // Ensures that a Git repository with a devcontainer.json is cloned and built. + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ + "devcontainer.json": `{ + "build": { + "dockerfile": "Dockerfile" + }, + }`, + "Dockerfile": fmt.Sprintf(`FROM %s`, testImageUbuntu), + }, + }) + _, err := runEnvbuilder(t, runOpts{env: []string{ + envbuilderEnv("GIT_URL", srv.URL), + "CODER_AGENT_URL=" + logSrv.URL, + "CODER_AGENT_TOKEN=" + token, + }}) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + select { + case <-ctx.Done(): + t.Fatal("timed out waiting for logs") + case <-logsDone: + } +} + func TestInitScriptInitCommand(t *testing.T) { t.Parallel() From 03def87627672d3fe726304539699b450cc73d53 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Fri, 27 Sep 2024 22:47:04 +0300 Subject: [PATCH 54/81] fix: use openfile to explicitly set magic image perms (#370) (cherry picked from commit 0a11dad4ca99fd606f392340680f0d71447d92a2) --- envbuilder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envbuilder.go b/envbuilder.go index 3cea4f65..94998165 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -1596,7 +1596,7 @@ func copyFile(fs billy.Filesystem, src, dst string, mode fs.FileMode) error { } func writeMagicImageFile(fs billy.Filesystem, path string, v any) error { - file, err := fs.Create(path) + file, err := fs.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) if err != nil { return fmt.Errorf("create magic image file: %w", err) } From fc853061215d5654a2e0ccdd8d47717e7d5261f7 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 27 Sep 2024 20:53:24 +0100 Subject: [PATCH 55/81] Revert "fix: add preExec to Run (#368)" (#371) (cherry picked from commit 5c150b3c1e35b7b331caa464393156eac4561159) --- cmd/envbuilder/main.go | 12 +----- envbuilder.go | 7 +--- integration/integration_test.go | 67 --------------------------------- 3 files changed, 2 insertions(+), 84 deletions(-) diff --git a/cmd/envbuilder/main.go b/cmd/envbuilder/main.go index 720c0c85..410e0897 100644 --- a/cmd/envbuilder/main.go +++ b/cmd/envbuilder/main.go @@ -37,12 +37,6 @@ func envbuilderCmd() serpent.Command { Options: o.CLI(), Handler: func(inv *serpent.Invocation) error { o.SetDefaults() - var preExec []func() - defer func() { // Ensure cleanup in case of error. - for _, fn := range preExec { - fn() - } - }() o.Logger = log.New(os.Stderr, o.Verbose) if o.CoderAgentURL != "" { if o.CoderAgentToken == "" { @@ -56,10 +50,6 @@ func envbuilderCmd() serpent.Command { if err == nil { o.Logger = log.Wrap(o.Logger, coderLog) defer closeLogs() - preExec = append(preExec, func() { - o.Logger(log.LevelInfo, "Closing logs") - closeLogs() - }) // This adds the envbuilder subsystem. // If telemetry is enabled in a Coder deployment, // this will be reported and help us understand @@ -88,7 +78,7 @@ func envbuilderCmd() serpent.Command { return nil } - err := envbuilder.Run(inv.Context(), o, preExec...) + err := envbuilder.Run(inv.Context(), o) if err != nil { o.Logger(log.LevelError, "error: %s", err) } diff --git a/envbuilder.go b/envbuilder.go index 94998165..683f6a54 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -84,9 +84,7 @@ type execArgsInfo struct { // Logger is the logf to use for all operations. // Filesystem is the filesystem to use for all operations. // Defaults to the host filesystem. -// preExec are any functions that should be called before exec'ing the init -// command. This is useful for ensuring that defers get run. -func Run(ctx context.Context, opts options.Options, preExec ...func()) error { +func Run(ctx context.Context, opts options.Options) error { var args execArgsInfo // Run in a separate function to ensure all defers run before we // setuid or exec. @@ -105,9 +103,6 @@ func Run(ctx context.Context, opts options.Options, preExec ...func()) error { } opts.Logger(log.LevelInfo, "=== Running the init command %s %+v as the %q user...", opts.InitCommand, args.InitArgs, args.UserInfo.user.Username) - for _, fn := range preExec { - fn() - } err = syscall.Exec(args.InitCommand, append([]string{args.InitCommand}, args.InitArgs...), args.Environ) if err != nil { diff --git a/integration/integration_test.go b/integration/integration_test.go index 79b678d5..66dfe846 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -23,8 +23,6 @@ import ( "testing" "time" - "github.com/coder/coder/v2/codersdk" - "github.com/coder/coder/v2/codersdk/agentsdk" "github.com/coder/envbuilder" "github.com/coder/envbuilder/devcontainer/features" "github.com/coder/envbuilder/internal/magicdir" @@ -60,71 +58,6 @@ const ( testImageUbuntu = "localhost:5000/envbuilder-test-ubuntu:latest" ) -func TestLogs(t *testing.T) { - t.Parallel() - - token := uuid.NewString() - logsDone := make(chan struct{}) - - logHandler := func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/api/v2/buildinfo": - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"version": "v2.8.9"}`)) - return - case "/api/v2/workspaceagents/me/logs": - w.WriteHeader(http.StatusOK) - tokHdr := r.Header.Get(codersdk.SessionTokenHeader) - assert.Equal(t, token, tokHdr) - var req agentsdk.PatchLogs - err := json.NewDecoder(r.Body).Decode(&req) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - for _, log := range req.Logs { - t.Logf("got log: %+v", log) - if strings.Contains(log.Output, "Closing logs") { - close(logsDone) - return - } - } - return - default: - t.Errorf("unexpected request to %s", r.URL.Path) - w.WriteHeader(http.StatusNotFound) - return - } - } - logSrv := httptest.NewServer(http.HandlerFunc(logHandler)) - defer logSrv.Close() - - // Ensures that a Git repository with a devcontainer.json is cloned and built. - srv := gittest.CreateGitServer(t, gittest.Options{ - Files: map[string]string{ - "devcontainer.json": `{ - "build": { - "dockerfile": "Dockerfile" - }, - }`, - "Dockerfile": fmt.Sprintf(`FROM %s`, testImageUbuntu), - }, - }) - _, err := runEnvbuilder(t, runOpts{env: []string{ - envbuilderEnv("GIT_URL", srv.URL), - "CODER_AGENT_URL=" + logSrv.URL, - "CODER_AGENT_TOKEN=" + token, - }}) - require.NoError(t, err) - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - select { - case <-ctx.Done(): - t.Fatal("timed out waiting for logs") - case <-logsDone: - } -} - func TestInitScriptInitCommand(t *testing.T) { t.Parallel() From d9fcd53421e84ba4a0342078c50342fbb5529cc6 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 10 Sep 2024 12:20:09 +0100 Subject: [PATCH 56/81] chore(README.md): add notes regarding Coder integration (#341) Co-authored-by: Mathias Fredriksson (cherry picked from commit 7fe19002839ac03b9e97faa8d310131a67831f95) --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index e5cc3cfe..7ae8ab99 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,27 @@ Exit the container, and re-run the `docker run` command... after the build compl > If you need to bypass this behavior for any reason, you can bypass this safety check by setting > `ENVBUILDER_FORCE_SAFE=true`. +## Usage with Coder + +Coder provides sample +[Docker](https://github.com/coder/coder/tree/main/examples/templates/devcontainer-docker) +and +[Kubernetes](https://github.com/coder/coder/tree/main/examples/templates/devcontainer-kubernetes) +templates for use with Envbuilder. You can import these templates and modify them to fit +your specific requirements. + +Below are some specific points to be aware of when using Envbuilder with a Coder +deployment: + +- The `ENVBUILDER_INIT_SCRIPT` should execute `coder_agent.main.init_script` in + order for you to be able to connect to your workspace. +- In order for the Agent init script to be able to fetch the agent binary from + your Coder deployment, the resulting Devcontainer must contain a download tool + such as `curl`, `wget`, or `busybox`. +- `CODER_AGENT_TOKEN` should be included in the environment variables for the + Envbuilder container. You can also set `CODER_AGENT_URL` if required. + + ### Git Branch Selection Choose a branch using `ENVBUILDER_GIT_URL` with a _ref/heads_ reference. For instance: From feec2247ae260696d8ee2ebe1799ba2982b24bed Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 10 Sep 2024 14:38:14 +0100 Subject: [PATCH 57/81] chore(README.md): add instructions for local iteration without a Git repo (#342) (cherry picked from commit 9b83cf5b799e1ea45860896b2ce0b5814a897892) --- README.md | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7ae8ab99..28bfe098 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ docker run -it --rm \ Edit `.devcontainer/Dockerfile` to add `htop`: ```bash -$ vim .devcontainer/Dockerfile +vim .devcontainer/Dockerfile ``` ```diff @@ -53,6 +53,39 @@ Exit the container, and re-run the `docker run` command... after the build compl > If you need to bypass this behavior for any reason, you can bypass this safety check by setting > `ENVBUILDER_FORCE_SAFE=true`. +If you don't have a remote Git repo or you want to quickly iterate with some +local files, simply omit `ENVBUILDER_GIT_URL` and instead mount the directory +containing your code to `/workspaces/empty` inside the Envbuilder container. + +For example: + +```shell +# Create a sample Devcontainer and Dockerfile in the current directory +printf '{"build": { "dockerfile": "Dockerfile"}}' > devcontainer.json +printf 'FROM debian:bookworm\nRUN apt-get update && apt-get install -y cowsay' > Dockerfile + +# Run envbuilder with the current directory mounted into `/workspaces/empty`. +# The instructions to add /usr/games to $PATH have been omitted for brevity. +docker run -it --rm -e ENVBUILDER_INIT_SCRIPT='bash' -v $PWD:/workspaces/empty ghcr.io/coder/envbuilder:latest +``` + +Alternatively, if you prefer to mount your project files elsewhere, tell +Envbuilder where to find them by specifying `ENVBUILDER_WORKSPACE_FOLDER`: + +```shell +docker run -it --rm -e ENVBUILDER_INIT_SCRIPT='bash ' -e ENVBUILDER_WORKSPACE_FOLDER=/src -v $PWD:/src ghcr.io/coder/envbuilder:latest +``` + +By default, Envbuilder will look for a `devcontainer.json` or `Dockerfile` in +both `${ENVBUILDER_WORKSPACE_FOLDER}` and `${ENVBUILDER_WORKSPACE_FOLDER}/.devcontainer`. +You can control where it looks with `ENVBUILDER_DEVCONTAINER_DIR` if needed. + +```shell +ls build/ +Dockerfile devcontainer.json +docker run -it --rm -e ENVBUILDER_INIT_SCRIPT='bash' -e ENVBUILDER_DEVCONTAINER_DIR=build -v $PWD:/src ghcr.io/coder/envbuilder:latest +``` + ## Usage with Coder Coder provides sample From 6ffa02145de3f12399697cbc3a488a8b6fe42da8 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 12 Sep 2024 21:20:37 +0100 Subject: [PATCH 58/81] chore(docs/docker.md): update docs for DinD with Coder (#346) (cherry picked from commit 287080c6659b9abe7a4a4f635c08cc9de2b814ee) --- docs/docker.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/docker.md b/docs/docker.md index 4ed032e3..ca09c724 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -1,7 +1,23 @@ # Docker inside Envbuilder There are a number of approaches you can use to have access to a Docker daemon -from inside Envbuilder: +from inside Envbuilder. + +> Note: some of the below methods involve setting `ENVBUILDER_INIT_SCRIPT` to +> work around the lack of an init system inside the Docker container. +> If you are attempting to use the below approaches with [Coder](https://github.com/coder/coder), +> you may need to instead add the relevant content of the init script to your +> agent startup script in your template. +> For example: +> ``` +> resource "coder_agent" "dev" { +> ... +> startup_script = <<-EOT +> set -eux -o pipefail +> nohup dockerd > /var/log/docker.log 2>&1 & +> EOT +> } +> ``` ## Docker Outside of Docker (DooD) From e4a8e4b7f0e8eae2dc0047e68a65d2063477e643 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 19 Sep 2024 18:42:53 +0100 Subject: [PATCH 59/81] chore: update dind examples to use onCreateCommand (#350) (cherry picked from commit 7c8e6a4da574e87af2a67b189120cac6bb5c1ceb) --- docs/docker.md | 25 ++++++++++++------- examples/docker/02_dind/Dockerfile | 25 ++++++++++++++++--- examples/docker/02_dind/devcontainer.json | 5 ++-- examples/docker/02_dind/entrypoint.sh | 7 ------ examples/docker/02_dind/on-create.sh | 22 ++++++++++++++++ examples/docker/03_dind_feature/Dockerfile | 23 +++++++++++++++-- .../docker/03_dind_feature/devcontainer.json | 3 ++- examples/docker/03_dind_feature/entrypoint.sh | 7 ------ examples/docker/03_dind_feature/on-create.sh | 18 +++++++++++++ examples/docker/04_dind_rootless/Dockerfile | 11 +++++--- .../docker/04_dind_rootless/devcontainer.json | 5 ++-- .../{entrypoint.sh => on-create.sh} | 4 +-- 12 files changed, 115 insertions(+), 40 deletions(-) delete mode 100755 examples/docker/02_dind/entrypoint.sh create mode 100755 examples/docker/02_dind/on-create.sh delete mode 100755 examples/docker/03_dind_feature/entrypoint.sh create mode 100755 examples/docker/03_dind_feature/on-create.sh rename examples/docker/04_dind_rootless/{entrypoint.sh => on-create.sh} (79%) diff --git a/docs/docker.md b/docs/docker.md index ca09c724..56ce9d05 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -9,7 +9,8 @@ from inside Envbuilder. > you may need to instead add the relevant content of the init script to your > agent startup script in your template. > For example: -> ``` +> +> ```terraform > resource "coder_agent" "dev" { > ... > startup_script = <<-EOT @@ -43,7 +44,6 @@ docker run -it --rm \ ghcr.io/coder/envbuilder:latest ``` - ## Docker-in-Docker (DinD) **Security:** Low @@ -57,8 +57,8 @@ Example: > Note that due to a lack of init system, the Docker daemon > needs to be started separately inside the container. In this example, we -> create a custom entrypoint to start the Docker daemon in the background and -> call this entrypoint via `ENVBUILDER_INIT_SCRIPT`. +> create a custom script to start the Docker daemon in the background and +> call this entrypoint via the Devcontainer `onCreateCommand` lifecycle hook. ```console docker run -it --rm \ @@ -66,7 +66,7 @@ docker run -it --rm \ -v /tmp/envbuilder:/workspaces \ -e ENVBUILDER_GIT_URL=https://github.com/coder/envbuilder \ -e ENVBUILDER_DEVCONTAINER_DIR=/workspaces/envbuilder/examples/docker/02_dind \ - -e ENVBUILDER_INIT_SCRIPT=/entrypoint.sh \ + -e ENVBUILDER_INIT_SCRIPT=bash \ ghcr.io/coder/envbuilder:latest ``` @@ -75,8 +75,14 @@ docker run -it --rm \ The above can also be accomplished using the [`docker-in-docker` Devcontainer feature](https://github.com/devcontainers/features/tree/main/src/docker-in-docker). -> Note: we still need the custom entrypoint to start the docker startup script. -> See https://github.com/devcontainers/features/blob/main/src/docker-in-docker/devcontainer-feature.json#L60 +> Note: we still need the `onCreateCommand` to start Docker. +> See +> [here](https://github.com/devcontainers/features/blob/main/src/docker-in-docker/devcontainer-feature.json#L65) +> for more details. +> +> Known issue: `/run` does not get symlinked correctly to `/var/run`. +> To work around this, we create the symlink manually before running +> the script to start the Docker daemon. Example: @@ -86,7 +92,7 @@ docker run -it --rm \ -v /tmp/envbuilder:/workspaces \ -e ENVBUILDER_GIT_URL=https://github.com/coder/envbuilder \ -e ENVBUILDER_DEVCONTAINER_DIR=/workspaces/envbuilder/examples/docker/03_dind_feature \ - -e ENVBUILDER_INIT_SCRIPT=/entrypoint.sh \ + -e ENVBUILDER_INIT_SCRIPT=bash \ ghcr.io/coder/envbuilder:latest ``` @@ -95,7 +101,7 @@ docker run -it --rm \ **Security:** Medium **Convenience:** Medium -This approach runs a Docker daemon in *rootless* mode. +This approach runs a Docker daemon in _rootless_ mode. While this still requires a privileged container, this allows you to restrict usage of the `root` user inside the container, as the Docker daemon will be run under a "fake" root user (via `rootlesskit`). The user inside the workspace can @@ -129,6 +135,7 @@ including transparently enabling Docker inside workspaces. Most notably, it access inside their workspaces, if required. Example: + ```console docker run -it --rm \ -v /tmp/envbuilder:/workspaces \ diff --git a/examples/docker/02_dind/Dockerfile b/examples/docker/02_dind/Dockerfile index 70a215b0..aa29519b 100644 --- a/examples/docker/02_dind/Dockerfile +++ b/examples/docker/02_dind/Dockerfile @@ -1,6 +1,23 @@ FROM ubuntu:noble + +# Install Docker using Docker's convenience script. RUN apt-get update && \ - apt-get install -y curl apt-transport-https && \ - curl -fsSL https://get.docker.com/ | sh -s - -ADD entrypoint.sh /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file + apt-get install -y curl sudo apt-transport-https && \ + curl -fsSL https://get.docker.com/ | sh -s - + +# The ubuntu:noble image includes a non-root user by default, +# but it does not have sudo privileges. We need to set this up. +# Note: we chown /var/run/docker.sock to the non-root user +# in the onCreateCommand script. Ideally you would add the +# non-root user to the docker group, but in this scenario +# this is a 'single-user' environment. It also avoids us +# having to run `newgrp docker`. +RUN echo "ubuntu ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/ubuntu + +# Add our onCreateCommand script. +ADD on-create.sh /on-create.sh + +# Switch to the non-root user. +USER ubuntu + +ENTRYPOINT ["bash"] diff --git a/examples/docker/02_dind/devcontainer.json b/examples/docker/02_dind/devcontainer.json index 1933fd86..6649501c 100644 --- a/examples/docker/02_dind/devcontainer.json +++ b/examples/docker/02_dind/devcontainer.json @@ -1,5 +1,6 @@ { "build": { "dockerfile": "Dockerfile" - } -} \ No newline at end of file + }, + "onCreateCommand": "/on-create.sh" +} diff --git a/examples/docker/02_dind/entrypoint.sh b/examples/docker/02_dind/entrypoint.sh deleted file mode 100755 index 38ac3318..00000000 --- a/examples/docker/02_dind/entrypoint.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -nohup dockerd > /var/log/docker.log 2>&1 & - -exec bash --login \ No newline at end of file diff --git a/examples/docker/02_dind/on-create.sh b/examples/docker/02_dind/on-create.sh new file mode 100755 index 00000000..8b369e23 --- /dev/null +++ b/examples/docker/02_dind/on-create.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Start Docker in the background. +sudo -u root /bin/sh -c 'nohup dockerd > /var/log/docker.log &' + +# Wait up to 10 seconds for Docker to start. +for attempt in $(seq 1 10); do + if [[ $attempt -eq 10 ]]; then + echo "Failed to start Docker" + exit 1 + fi + if [[ ! -e /var/run/docker.sock ]]; then + sleep 1 + else + break + fi +done + +# Change the owner of the Docker socket so that the non-root user can use it. +sudo chown ubuntu:docker /var/run/docker.sock diff --git a/examples/docker/03_dind_feature/Dockerfile b/examples/docker/03_dind_feature/Dockerfile index 12f1c1a0..49c6646a 100644 --- a/examples/docker/03_dind_feature/Dockerfile +++ b/examples/docker/03_dind_feature/Dockerfile @@ -1,3 +1,22 @@ FROM ubuntu:noble -ADD entrypoint.sh /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file + +# Install some dependencies such as curl and sudo. +# Also set up passwordless sudo for the ubuntu user. +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + curl \ + sudo \ + apt-transport-https && \ + echo "ubuntu ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/ubuntu + +# Add our onCreateCommand script. +ADD on-create.sh /on-create.sh + +# Switch to the non-root user. +USER ubuntu + +# The devcontainer feature provides /usr/local/share/docker-init.sh +# which will handle most of the steps of setting up Docker. +# We can't put this in the entrypoint as it gets overridden, so +# we call it in the on-create script. +ENTRYPOINT ["bash"] diff --git a/examples/docker/03_dind_feature/devcontainer.json b/examples/docker/03_dind_feature/devcontainer.json index e1b5a18a..58616a6d 100644 --- a/examples/docker/03_dind_feature/devcontainer.json +++ b/examples/docker/03_dind_feature/devcontainer.json @@ -2,7 +2,8 @@ "build": { "dockerfile": "Dockerfile" }, + "onCreateCommand": "/on-create.sh", "features": { "ghcr.io/devcontainers/features/docker-in-docker:2": {} } -} \ No newline at end of file +} diff --git a/examples/docker/03_dind_feature/entrypoint.sh b/examples/docker/03_dind_feature/entrypoint.sh deleted file mode 100755 index d18fb7dd..00000000 --- a/examples/docker/03_dind_feature/entrypoint.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -/usr/local/share/docker-init.sh - -exec bash --login \ No newline at end of file diff --git a/examples/docker/03_dind_feature/on-create.sh b/examples/docker/03_dind_feature/on-create.sh new file mode 100755 index 00000000..96bef1ca --- /dev/null +++ b/examples/docker/03_dind_feature/on-create.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Known issue: Kaniko does not symlink /run => /var/run properly. +# This results in /var/run/ being owned by root:root which interferes +# with accessing the Docker socket even if the permissions are set +# correctly. Workaround: symlink it manually +sudo ln -s /run /var/run + +# Run the docker init script. This needs to be +# run as root. It will take care of starting the +# daemon and adding the ubuntu user to the docker +# group. +sudo /usr/local/share/docker-init.sh + +# Change the owner of the Docker socket so that the non-root user can use it. +sudo chown ubuntu:docker /var/run/docker.sock diff --git a/examples/docker/04_dind_rootless/Dockerfile b/examples/docker/04_dind_rootless/Dockerfile index 5358ce60..2d88aa17 100644 --- a/examples/docker/04_dind_rootless/Dockerfile +++ b/examples/docker/04_dind_rootless/Dockerfile @@ -1,8 +1,11 @@ FROM ubuntu:noble + # Based on UID of ubuntu user in container. ENV XDG_RUNTIME_DIR /run/user/1000 ENV DOCKER_HOST unix:///${XDG_RUNTIME_DIR}/docker.sock + # Setup as root +USER root RUN apt-get update && \ # Install prerequisites apt-get install -y apt-transport-https curl iproute2 uidmap && \ @@ -19,6 +22,8 @@ USER ubuntu RUN dockerd-rootless-setuptool.sh install && \ docker context use rootless && \ mkdir -p /home/ubuntu/.local/share/docker -# Add our custom entrypoint -ADD entrypoint.sh /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file + +# Add our onCreateCommand script. +ADD on-create.sh /on-create.sh + +ENTRYPOINT ["bash"] \ No newline at end of file diff --git a/examples/docker/04_dind_rootless/devcontainer.json b/examples/docker/04_dind_rootless/devcontainer.json index 1933fd86..6649501c 100644 --- a/examples/docker/04_dind_rootless/devcontainer.json +++ b/examples/docker/04_dind_rootless/devcontainer.json @@ -1,5 +1,6 @@ { "build": { "dockerfile": "Dockerfile" - } -} \ No newline at end of file + }, + "onCreateCommand": "/on-create.sh" +} diff --git a/examples/docker/04_dind_rootless/entrypoint.sh b/examples/docker/04_dind_rootless/on-create.sh similarity index 79% rename from examples/docker/04_dind_rootless/entrypoint.sh rename to examples/docker/04_dind_rootless/on-create.sh index 6c8a6260..ba2fced5 100755 --- a/examples/docker/04_dind_rootless/entrypoint.sh +++ b/examples/docker/04_dind_rootless/on-create.sh @@ -3,6 +3,4 @@ set -euo pipefail # Start the rootless docker daemon as a non-root user -nohup rootlesskit --net=slirp4netns --mtu=1500 --disable-host-loopback --port-driver=builtin --copy-up=/etc --copy-up=/run dockerd > "/tmp/dockerd-rootless.log" 2>&1 & - -exec bash --login \ No newline at end of file +nohup rootlesskit --net=slirp4netns --mtu=1500 --disable-host-loopback --port-driver=builtin --copy-up=/etc --copy-up=/run dockerd >"/tmp/dockerd-rootless.log" 2>&1 & From 47de9d1ace9b80ada81751d1ac05a2f2f9473fa0 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 25 Sep 2024 17:20:27 +0100 Subject: [PATCH 60/81] chore(scripts/diagram.sh): export in PNG format as well (#356) (cherry picked from commit aff347a100e7be119823ec10f82ffd7e7cb0f739) --- scripts/diagram-dark.png | Bin 0 -> 56966 bytes scripts/diagram-dark.svg | 161 +++++++++++++++++++------------------- scripts/diagram-light.png | Bin 0 -> 56922 bytes scripts/diagram-light.svg | 161 +++++++++++++++++++------------------- scripts/diagram.sh | 7 +- 5 files changed, 169 insertions(+), 160 deletions(-) create mode 100644 scripts/diagram-dark.png create mode 100644 scripts/diagram-light.png diff --git a/scripts/diagram-dark.png b/scripts/diagram-dark.png new file mode 100644 index 0000000000000000000000000000000000000000..50476628c0ece2f4e14d4af1ceb47c3bd01077de GIT binary patch literal 56966 zcmeFZby$?^*ES3Yf}ntcAdMm*-QB1lQqmpLjW95DONg{0Ie?^e=TM52bPY13A~C?w zJ;Zm9TlW4P&-XsZ^L+33@BTwXXXd)swXU_!b*{MP%?lOzTlkdt7#JA06rMd%$H2g? z!N9EKDK{Ea10mPk4bh694`k{bnZM4+b}< zAg3UP7~IaeeNB-z9+63K>pD8tI{AgWukIUt1d46#V=FYcL+TGHCRE=tI!O&@HBv zG|_iTP>%y+>&x9?!#(FbDu4?nWA z1|IxxmQ=vRhqI)R14T5*d2IOSLMcDoF$pyIvoC2^`g zCj`OYq?BF|i?qgk@fp|ui+mC@ap!#Ujp2b4k^&62od65Awio>4^M!Fl;=S8{%r?l4 zBr6NzL}A_fUqAj(IxDwfhRO6^82YVdOmQane(Bow^CR%C9vuO}pc^%_R#5$~AEn<8 z$G~y?fQ5zUsT>$wH_0p4{O2ukoPe-njAQ*ISBqT9fsN14X_M*F{o@8TtUB%{z>9ldSS$TkPVUPoEi=7x!^#`)SQ>aHQ9=*tAG@UIzl}@VDGxla*r{BR&gHL$UAqDU2R307x)nKV+hZcqORoKZUnZL$()S1 zzi+i#rpU1toOO>U9sm@#}GlkGewi`(;Qxadpse`LzVdoiqnFBbA3P?{77 zb2;tVELSqcJqfDevN5n%R&eop!()}yB(7^OaF}SY-{EY+%p-6!8dx_sDKLISB;2Sp zdU96W_ov`PG0({_%|~O3Aya>c#5o|)BG0LLuVJ5|nZXu?mH(e8xj>G7g4i2&mZI-p zz76uoHM+kss_`*g*;R&+$SmYz)QswTtPLCTARRX`H^P z`(t6FIoV&)@CBE&Oj;)pi3@?}h)wW4@x2QF?fO7$3@CY>kR2J%d)T@i%40q^ox1;7 z7>1AjurwA9c)dO|=6h|$PK5a$L5MXY_sy_>SmSCIpl(8}DiJ{4Mu56!-cbC_5hC6I zuW@%J{M;=}IJ}YM_iJK$ANj?L=W{*@TFXH_*eYV%>W(ADnZ=cfhb)h7wvy-()%>&e zvpTdXSO@Ohmd@e;q;$NoBYL%=I7;A1I(E6uvypj4kzQS&>}SW=zJ0Lm&n?!}(~760 zV8BP4$t~{OX}Pz?BcwwQc^)=3~kcQ_bbN8pth@-DjvA@2W}VJ z3*65Afsd{TS;!%pB^?@1Gt6L=nfF`+(!`K3Y^ zNMJYr=w0QzM{eD;gY{8F*UetK4=6LN4%0_hB4Y%B0rPairA!0PNEXd>-lu*3!zF>& z2wsJO`P}H*OrcsyNkd6WLrqD2Y&z?MifSR;)c~Y<<)fjsjXPUDe3f9 zX|3WlriOtm**zb(mcJbB_hHh<1NtE;9MA=vg%&snq?$R^l_2yxz-tEFTz*YIF{rA_ zL)*kJ3#Pnxqj}y8-C;l*x-EOff!w)(+2vb9fZNvs+&*M9Ikb@BiWaf}rYW|h@gxv0 zw*@%w8WSvxKMXEV9>A9RginA0bu&ieIq#nB7igX?L1`7ZUZPjNAQraT#zR(3WNc>F zjZolp1jinmhxKD!x%+k|z=AKA-c#IP5CWw3?|ZZ-^T#;b@B!s9R_Ad|i9@r^%Ay2| z?d$T3)XkYpN|%b0j@^~HxzOgbgk8m3EnwGUeW*sw{#Pv|vD_m^CKT>a1%hVv?pD;Els-T>^Td+t5$ zZE5;C;1tc*6`Zd&CEX0@UY_1S6lOKWdAIuEL9xBDkw&SHNwZ2@`LRGStxYgmN@9Yb zDCw|4sh`&m46Qr1H({T;HsS)zHk);$WazHe!oaG$4aiE>y=^SGS{qL&y74Hn)VNe9 zU%Xt4P3NmWvG@LKX?o(gYFe04c>&qwMH<<~X|?Rf7mgdBn^i;?;_mC0ubg@#rMOnR z=~|DzrYGEf2mr$;KBlX!5Th*zZ{T^i4j~?nEMN3{z4~4*MY7c(79m!PN&lA%zcUQt zfFR>pp^?ULBesHu+3kwz>z3DXQ$OZ2UEJmd(!js0Zb3CDA-hT;1rzjRup@ZB(=3d% zhCxUCc-!1<^>%lY;IYeRkJIjGopC9TA=DnEnv=u-i^*==?lr0%0p+$$N00|0YYyO%_oXu_4V{zWK7 z)IRb8wL_q`FROvJy`UKUjrv7mWc}FfXlSEI8@H#sJj%-F$}H)PaO?<%VVoN6GY~0u z&&TpKXD^259Jl9qgQPV{rF>Df{--G821067>Nk~BC&$Nadbir_N4{j%n4RoCgDQGMqlKoFgw7U? zb*Qzpv_yTrx+uhQjAHDHy2Bhjq_SsBE2fmm&QGg9u}PdjdTw^UWhu4KTs%_G(? z{Obj09Y8v1DS2D#pP|mt{G{_WIa$awP2SsoHiOP|k&o^;FvFup*LG79ran6Ht!!-I z7DT~@6dQasNZ<261w2OdAu)Flyt+ncqtf=Vl=l61@$pit#~Ud$Yw6K{V>SO>JT7Si zgqS)1g8zt=>4~jT#<%W>lY*v< z={!@SyBl@Z=OC%e8tHeLX~bIMDvM&9KC-h-LL4^X9u}n&*`aN#9f+;9+DoMb4PoVp za5{W={oJ`~DRl0X<4)Ti%zkawTG4UNnoWm_S+yY3Q}Y}|T0`Xtu&T8--JgGl#Tmd; za|~}YF0!7oc8DI0m0glGdM-VfP%GNr}i1)?lqKNidMs^wxOeCs7sc_en8rF`#_4TM+7B!=F(?mJ^?zvwU+r@ zQwHkPp2RFwypl;lck+v5d~_WLc}^s1c!1)kDP_%*u27uO!b`{Fc) zV|*E(st?E1)T8K97Mi+UvkBlT=^pdo5UZh6;n6sf(Lq3P5a(Iibwc=nWvv6^j*;t} zNzP$T)~vLgjM&;z$MwG#R!2wIcYa;3tCW>{`V-X(nmD!JbBrY`=oD75x^GcT3?iy1 z0LY1}!diD*`>6IGHO5+_CwQm}|I*iu;d^{5UVp zg#fGsP^Pr)A`G5C4=xj&4>*b@JKHX;0=fP&Oz@s{pPZZ`H)JxWBs|j8nY1}uB!W2~ zp?*F+u->h!!%l<#I_~3=D5y8SjJ>|W|I80{ILQg!{8&;_!ZiCnDGAX$Mqps{;~IG8 zDRi^4-rK1n2#D{s7CR3H(!>m$TQBHBWj{t-x;uRd1F{Y9nUJE%St$~(R#}JDA6n1P@ zaCYAu)nfa=HNhptG5o%FsHS0#LB;G2mHM#C1IfpKGcGlNImTcWyNo5=SPDp-F#jW8)i%Uot!*jtLW4ASxf%RFq5Vi-A6|&&H`_>iM=8wxy91IaF9Kmk2fcS z4+WB<-)lUJI^_AezgN?3900zXwto5wKV}ws@XCFvAn~!A;tNnu0b(+Ke7xRm2_3oS@{)@;Xp*OR(!=rj|6`r$wkle)F zdShd_MxaOJ=O2swlY=bynZ9cx-`2+)r~K2ikEzO)NBBBb$BjQoBm~!i3J=qqd|sra zP?SmPl68mX6MPWq9|eU)NT!0&c}FZ-)BKLxkAG zm<VL!e98p`u6Qg#9!%n4mDJ8*meD@0~eEN)tgjm3$);H$`PoGbwr?J)5z5bcr z;=kvfB#7FzwT>Qj^4X+`Um2J*%6D$dn75LdW|^7upEq9{%_}Z8SxG?Zm&6MLHp-wHv1OX?FQYBwJaVmVwj6d6v@9giv-S6Nrh}+N1A$vFC=jC^A z648>{($)NqH0XR)UCZ?F$#i4Btk=hq$Yxfm)!e{djZaKNx*DjNIynwGY<&C<8TwGne8ks}qSvxxjG!uvUP`>MNb3!KKC_S{K7}5Rv$4jjJmt?dJLn$XD4i&$|R_T%A-y z*!C4AI@`1!gQ9h=gMgS05<1&#Ax;u@-FlV2jhwVz&mBKbc-3eROFDgL5#1hUNc$>y zu@io9Xi(mi(RP;FX20IL>60RME4{j~?aZt!y}BNPQVy3&S9FaH`1ylu(MG(2q#8uG z4=|vlx_OtZbd;O5rL~YA^B~27mOX6TNpkZ|`2Ob_57J`38Y0OJ8yb-Bn3!;4Vrf|m zokw}J;7ER&<0ih}k8-K?BE>;ld)vJ6^=JY(Z_uK}W1;78LGc%CiB!dLiFQ#AJ(QtQ zn&x>gI`;BUR-$DS+?{Tkn3wKd)?QBBGlv~oYjNbb_I^;vSJJlR&F|NTHXVA|esBGH_peXQ*D9FydCHQ{%v#sDM@Ji+WNoE}aO z++{sIhFXsrto)I!I_gnjIluGyATsQ*O|Mii+|^^U7-sk6^Y+5KtiRE#@PTQm0G;x1 z6H`ULweq8yR-T{?82li*cHO|-htk%scmI8jKXR{2SV;HqtH*IuUtFJ^UBeNDM*~4d zVZvM3PfxeA+)rN|42Nke7CD<1M(tDgcg{WLSC{R%;x*#jHZlE{4(f|0!>FL}ry9d$ z)5DDp2Pq7*NeD20i*@MT4-AkO{5XyU$w<=L4muUJ!+nxTE*4eV zZEol_?a@s(nMG;IOrQLauHW25H3dkbG-eNC*VnqCR{KV4Ehl6L0nTR(Vb3P@d8Y+f zY!pFMd)X6f`%3yp19|#NYWuKY?6pC|RdKu@ ztgNlLiV5uySsluH&@L;6zZseeR>mYIp9!p21WGH#94K$ z;+@K*KQ8s!^Ry&CR zVzANT$g6-HCh0NMu{d4`N+L6HcAOQ`c?QN`7z{{BNhwo?V4Xnf`Ooiv@yDgjf61Hy zv+!L>`1vE2fC~z9KiS-@yGHuT6@Jn0Ep4TMf_MhxsrqO#&}V%hfr zCbi{B?bXkHG&3aGTU+<;)R^@10^rBmsph#Cw;k3{Oxoa&F8B)xAXCs3TneeaB2qxl zX`iIKcowjF6nKl>&{=tK$WJagtWW0HIO4{iXfFK#U441sBj<85C+98J+_y#UmseR{ za?Z#@gg&vLuXy`YS>-BHt5sbg=-N76E#pc6f3coK^EpMLv_`D8@Zv53SKI4}HQPeO2Y7v|!<>WOG#SA*H8F)5m zwu6N<(`9s|Ho$SyxPRVPz$Bn;?mWfsGK6sQQb2S}tT{70(^n-WWph4A>3pk?)q5?| zv1}4H7)3bW(9|Sn9V=Wn^=0=4!{DeSs&>ms&Klxsh&-P#h+O=z&NVq{GGOmQ`-a^{ zEJ@RYvve+_+ni{&^r)bbFuJxN?X~0B7shVqgmJEyGRHRb_O-HdZQs^(7|zY2&$V1H zocwQDBxgqV2Y-`3^f08Hl~=H4{VNncd+~-!_wYR>C~QEW5CkGf{F;^4Vg) zl?TI)dGu)9X?k{UbN`t4OtDiK_K406HmXR zE$URZU*0WQen44Bvtl}f^IhbwYqYo@ati7p#jgT^HVl;zw!Qb94HMf;FW=*AEqYT5 z;@XV6WxMfiNWcqO*iAcrncq6MTohfCg>>UbG~P(v{nD1U@AboS(iTMnb^?_wQyO|- z6zVrwxV*(dHdRjh!ivB6!A(Ux!&)eTx7)BxWE6>MeM&3 z{9Y&V9lD#{XPlqa3{K3=O-+A!{vW7bU$f=yRGxJ5Kma@8b1ZSUjT;;DuPodQ~wE-FwGOwf5_*4R&!!7r;71X)rA( z=OqVrq*_tU5$|nqp3As7jMrTQhQ)PWbaTXZmUDels8ai` zTs}}Xm~oE#!-s(Ke6q;MVM!6^JgeydOdH;GsANA?TG|im>M`pE_2kp^6 zw~Er!M0wbg>zVr#VE1RgP&QSwqPHK}HP)}$YRwll{wRDd$SEz)52H&WAEG03aDAQ+ z5TilMq<|YxA&|Yw6}{AXw~b<%Y*{!627_?y3q03JT_D{?MB-XLs;k@G)|tM!KAW#p zUORejs8@s!O1AYTey|(b2Xjtb^fZvCzEZt+^l8r5@p)RGKq`>wXdhdWr{`NL&eq3K zd5t8(ZB1p;%mpU1O8IIu3bR^JnTHdF*2cqSUNHe@FI&!G#Yngy1r<<2P?0mFq1am_ zcMvl8C~jbsVqJv=DUzM=!o(3Z2yLRmX`-T9Bje3ai*GV0gjTLuSwg8&HhJsPK#EZ@ zQFLz|*YW2PPGFRN>!dB$wCA;~SEr3)hahxsc~-CeVfD?7KwY2kgw@Y`K-xe<<3WzJ z(UBC#g$j`h1qT~d_-zEbvnZV<5)|#d4@39K0t*Agz=W^;-kaR(7wXy$>#zC?>VPIr zI^TMgeGxdWl!CYl92iewYU#nJ|io%Fu?wl%9Y1AOI zm9*eCg6z)$T?0$%qc&cBT4!W4(s^;O`8LlVMzNjL9!}DSvVdWOi*AI11-l621jKr5 z-X;%>ers$#B7&{qQgFoW71Xq2?;B;o&l>vPG@5ielst4Jb&-Ll^t}}#=frU=Y{dFx z?mmT60FCr#P?1s%IRejfTJLPNA$AE=%L`gIw><2}1_V9ShS>c~i01t4&q0Gr6_7qN zA%A{au7oA`Za>ve21lD^`CM)L=^2+t4W}zW`O9ca!M)I%xBtbh@VL>y?`CY^H6nx$G7JII_Jw| z8xxW)Uo8GVQ+o%iqAnpD-gv!sk%qA8#%7PjaE%JTpYf<{eY=6XHP%C?x_irm+HJ_|HjB#M zLM)z>w<;FGx7<4tZzYv%0>yG8&=^Fy>+&Ee(`{^m+C1mxyj554=2Tm#> zu{i=?QN8!0Mx;R@Es;%UwJxo!q=nSCepAA2Erq=3h|FLu8>FJ`^aSZbrAPJXW|we4DyEz(awZ3Vgw-Ot^}~jP z9-lm>*mQD~h?Uo2v)j4vu&WFd3AnrU)?_f)V7fdFgE&B;CQ`v36QqR8-VLba2MCyt+BE%AJ z%yn2ACFepCki)E$oCePxI`HBG`2k2(qV>?rI%d!?&8Nh;R(&uQPt<+WFrcd$ggDq6 zd{km&ZtM-rk+eAYp)S|PmX>{m0sg5n=@b2EAowt7|)~&yXfSc8y zYw9)f;ocJH(@SpxZr{0%4|lmtQd2hsr&v&yT{0qCM9SBfM)Q-L zB7L}TqWd7ZmF>pUbh#m5J)erAimh1s@kI8n&MmXge9b8NF7NF?G&J3J-ahk+BuQ|@DsVcS&x}dT+93| z#Ogx_Lc_}S65ltLdV%EN4|(16K52g(JMPk<^5%6#U4u7rX=Ng#aaLPq8u9A;n*#f` zYqRxit{H7BF(aoN??&M~kl+&tk@(23(Unphx4qh=HK5>H;+TMsgOo!01gc0Lq+2}( z4~~MH)8#=&fzFN@(Oe32Y6%H55y9XDlHr&i)~YoSw&F>Lmw6nr_(;=|*|~4~u|J56 z$e$gW+bi3K3DBiQrKcdgot;{pYt4hIZUB*zSU$*2YOekB2Vu^#rZUCIz5)e!&gRx2 zc~NEgOg5cAN)$7P4;vj)r6GU1InH+p%^p{UcyRRl&@lo|nSp_;+niMG79l*_W(t1T zO&4AK{hQTyKUrW~`Ct^xG_%VoH6uV0+ZQ$O)hD1| z>-zNVp4;Y*gzI?XxBhQ-Ag@;O6FrH`{$s0Y2=~ttD=yW!hDyH4Z04>tWW90pr!3_0 zJZdQWw7Pix6vQdw|6_`|DBl}7ahQ(sHF23OP|AQ|@{KhK{AK-JFcKqiqbdB6P5C&Xu6{l*kG>Qgoz3IcNsPtNWU83CgO zp-m-Ia`{$1p-KT1&=!kywiO54+^CqF)|TXy_Ez6)@@s=q^n!-_A>lY}>(4UZdWjf}D8y91EdQyPbZV^$sukV}Af5bD z_z^UL+!gs6m)N$ksJSmOsp^moZ>dY9-*bKU!{zs##~ht#d-y4|zhz%D}ec zqo=9dn>>EIBjJvDQcK3xQW>r+Z@i>US{*>zoaL0nRRUsq_8v7|8;^=+e2+(O_V)Lv4Wh(u3Mn$oh;I%&kNn2en|vQAXZpN_`+tw^rb)#_8;vmjwn%5R zqKsI~TaM=SgrAXojHp+~pgfZc)ge{qqu5_}NJV~kaWC#X`flY+(}+@tW&$H3YF`gGlIfF-To6Z~J)a zC?!4XQ&J#s_wMOZQs(~2hH6cI6|a-I9NY!05>rRV{esFFW*{OWa*F7t+wV-*Q}fx^ ztuHt|Tra;R1b;FiFmbxp$F)Ck)_4Pwqd`4aGDNlpuQfonbgT?Pa&V-4saT$l!Y!?= z3b(-1#j2U5_Cv4>$B}YY4lk1>%i^?XTc9an+vj*Xk@1nXsBA>XWME=)Z>6xQiPaXg zy4t?cEylV!i)&N;svwOE-sjT}8?6o*H{=`VVyA_PLs%>5R#7s=bylS1C~FW}veAu+ldo@U$IZ4fiwsZIa&(dpd4GGsPt;Of zh8~!33&udXe2{Bs$6a)sl+H4#!0b_V6d8%hEcB3m~p?LNpi@gUVz z1CJ2neK&634-7O2JQrrm<_JQNIC4Yt?QmIYKs_vO24cha7SCiQH;k{4x!`W(+qjh-8r!xD2a&RRT@9E&zZ{sM@Ie184SYWa3CENasom9jHx787p9 z6^H!(sND{S9_p8QAvulAd9|lAJt~R4$*D;x61A5vs=p4cGtMppj?!eAe)3>J0`qWj zu1}BkiMV^wz{V+PB1ZdTaGXOB5Bwn3pzw)b$!ud@WITki*ft@5Dlq@<%|3(Sc&`tn zNo^dqTg#^r`JV>-n;G;>(dkmwB6=fKaxL$Wp7^Vu8@W=VDDH6XYvnYLLtb!;>%MR2z6Z>$z**WcKY+hf>M7xj9uQa|grOgAY0@(2t@sSg z@qUZMhUu5aZBWTAFB!fQvy}d*2lpRMyqX2L*9)%M$;a3lo9m}iV~A71LNw8-7Em+W zl~pst2NNgm;s{jXB-mZ@@lw@;&T#ArW~@@Nf^0*0tE^Ihk+sp*RA2+Npc(Ndu(p5#<<}8Z)gxx(J<1MVR;l3Q_4rrfYv6_ZoQw*GtN4c_R)5l5B|73$#3}*4n)JiYwQ^ciRIUn3%HW`B4HaA zSOSkNAP?$XE+3R6vK!c{w5RGkL{(`owF<2&X)LWj{rqbU$1~H42k?L*g4{8G@d?n6 z?qe&J3aZeJ)ZjSTL5dliDoWCUM29t{^uG6##@UA6I$fj)Q^K=GqQ+UO zZD2l+pWNPeKlEa3q|Zu@pD9{h9dMlYN(3_Z1ErG#=Z_X2iq&G5)Gp^_3rw?h5Z1v? zR6`gpeD4b0MfdQCNQcFkay=#y8Df6xkCpPqa)Ek^H-*vUazDc(yJ|XTjd=9jjkF=o zar)2Yyeyi@C7AaLs;$FI_$?&~=CLY!>^myl}>O#+~in@%+5i+l&zY!<2 zktjd&I-lG>j#CIs_WJX;I@G61FX);hD|y#w6fR0O$`ia;7%YtYdk4MzP#>+7A5M>S z&0{sIqQu(z2{EPcj$NcS!<%!*sI8Q0T9gJ-B`odj2xfS#tQjojY!`etSp`PL?%Q7u z-K&23WV&7rKxf+c2hr$Y*>D30mdZe|bW$uu#weOpd4ZpH8d5lFPc{|hKR;4xl2g<^ zhk}DZ=|)Y>d+L+2{Ku_HKea~FXiH-1Jw4Wm);GS8|5Kiz)%DR^DAhu{7-^K z%w?x&nR3l7`zsR!qvL)O=<$hI$i5>8!;#|d`i@thqnC=eEeT@R z9HzRVlxY*9CvX?~&8{NxkiQD5Xa#AIsZ~cyE#xkWR^)>_3!uofH^Ja1eN6t*lI)z* z67-8R*YYIE|1Lzv;C8BJ{4TQ8w_8cnj{~Dv_suE6(J*){kRzQ?XIajzme%J7B12?{SbaZKe?{mR0Z5% zAaDR2+8ZrxOM0&P6XVlh1T`@SOWE-|9+(IL>RzwIxb|R)rg_Pn0Pj%GFLAz4!)J3a z!R8og2!KQ-z@55h?wDN2ZjIT5<2%gTBV&ovVX|PZ-c3i#BVJxAu^h1ixmuKI03TZ1 zH+?J&=!*OR+BQl@H~pyl*uYqM=8hkb>O6|-ZB54ZvI5?w-=a_55wVq25gC9XVPfmw z?ZLBHMVB0g+yzUVt@GBTA924Lz9EwN8E7mPG6IM?{!P9~ZZVWAnUMsx;oxY~pk||+ z`_w_N#WBjO>(Poxh?l24&qnJdGD(sg?>pN(#Y9pGAO!z<#F6bG6D0kl6as0^{n!9bQ~!u+3FaZMNQ>#vfGz-wYt9bKSr*TM!tW6 zKJx&AdlV4R_X)V?>_7*z-Jh&@e!kiojOQsIDEr34b!p|}=?sM!GqON&V~L-kQ?IjAy2bKVbyglz^1oKmClJm@SB+b4isF@$37( zb&>%>fdM4r836Q%aum_`jiqzZQC?APIL2-1;U^;nwN5f^a1774QYzQ~NDp6--V8zW z%i$c5s?n=o%HO`>6%**5n>H;wiLJiU2m>beADRTpD(r%!vDK?u0r>rRrus2XrEb&( zxO`&JSW%TpSt#6r-AzPok6v9s8 zDK_?9L5zsNe^MBHxY+xRX%XGKwnv}4@eR@%jeQe21(y>|1=Q>Cqn_e=#B}JMI=|T$ z^r-R~%PV+LlHFV5bHjJlqmnqGpFPe*%&kKIiNvFx0iV+)N3K^FvxVswzq7P!qpLZk z8PtdNJR^B5*Ez$~F~l}R1zPljQ+YjjjywUe&JEA-9xE&+`;lOy~U{_kXLaTLGLjA`_iWSSt?26MS7H$ghRHmo%!Z zuNCcl*8Oa4M*j)SE-#)*&pdLE4|HJu`$DM&ES=D*Qp7cxVj$MgiZ^wd7f=2H3+BigUJR zsA^lv$AC*`hbXx$D!?+(umgshdE7c5v)#90#rm)@9W~c-rR3D|pL^bcxlqAE=Ht%Ro_ZpQz;OuWmTj607uf|EU zA1%`0zmC4W8W@PWm;yH@raUaJuikZKM*6nX0CuV&7kIoIzSNq`dA*+`NRtFWry|RT5{rmc z^8P{%N2{dFzF7R;bla$lnPtY#Jg(?{&9GiSvjy%wkarQEeK|k?kcmpSnXwRT!z1UO zr{8$?H18y<87@;l^8e#ONZrmZe{t=EeY6|br~+0HP1so<8((^daeIfL&cNO4#J6|9 zgsQsPwRY3@$MZz3vgexjqR{WoZIN~YXQ1Z_*8mbc_eCGdZyG#(la;@%K!O+p@1YDp zf)%8#dzi40=fCMnK9j{@RoBHt&%Pa`?ocW14Uvy1l(y3CY(V7qVbu8gM*0-L z#zQKj)vR9}g8exQWy>%!@W-y}CCgCGlsJj}{1&=%&vlorRfjB;4_%Mk`E^fXMo{o#I{Oi=EeKxZN5VP0Xf|_X+Ij(!N(?gC?RUx?8kV( z;ANaHPz!ko+_g_7kN8Ef{M+r=2IFcAsLTazf$7u?kcorkvZNaK_*tYFmND6aAG~jh z0gOHRm_Fg$)B72>B6cc%1)a4PT9n?qXi+kQ!LF0$UvrXLE%r7dBO7_)B~Cd4T8Zd# z9*0TG!=L`>I0xW0Pf$s}Lo_9?{;X*E^li*PCOS!4 z(S*CS*2>Cd3G+UQXx}$#tK5EVGhN2wfayTLW>ej{eb3J4ZFmp$0p|bYEgvlw-V?U= zpr1CKHkm3|8{K0p)BSgW0ZKK%vNlVAw7yddE~`XGmV~5@bJeQH6teUOn>is^07&s(hUhUYFk%vt`mh*$O|=2i;5TH}=!>}R96;p+AD(d` z1T+Q&<#@@%RDUidyz$-w0hhrfB;cOV!AeqA9wVqLiD7<*p)h{nF!At_QX;dCPOpIc)KWInWt%s(Ck28QGC zVo``*Mo^7|Ah$ZKE9C^N!p}%RD_)*grEg~^8- z;)0JDR-~*x748V&zk9qu1jJW8e8BqFu&5uY9NfHZ!i)1gmRTdS674_u@aRe4{vXi= zAdZPhd+IPVFaCJwWnTggnv>#B#o<~%=|KxFgyZc#|Ac|4O$Afq61J@z_ zzVq_?Pb~l;C39ZtD<2V*ByK&t=`gk)d*aVA6|l^DJRT@pJzRdhpCIT1Hjwc;0w`Pr zd{4#yK@#9gGIqll-B_Z{aehSXMyuHkP7L>mwz&@OIi8!1_;OSJr?A(B{OM$s$ zOtl9KMu1y+3VHZPHtz&&N4>}jaHr>gqCJKmhC6>SW|%^NH3IlGP3b)(aD#6IXb_Wd z40;;hF_qfPbGQv)&GBEh>KaxjuhR&z;d@@(XZ_qNyop8F+FDB3vc}(dw?*Fi>F?t^ z!cMQQ9+KG5cl>TX7Pe3-%f!GCFuXIx*CNF5;6GANpk{NPZnO+j1C%eu+H(ky8)v{4 zcgXSsxCkofj$98TfFi^h{coG}*X^zaJ)+PV(V>7fW`e=9xn$*rxjsxrIcqO!otJx` zphKD_`PEG&D~qu3Q}=qO#&h)4M!xsi*YH3cfX@7!=s#g|YBfD&%j{>W1MsyQ_HGW~ zNr!;>nL9s^y|@i*In$6AxK_NKkA_#T#hZ@KWs9eufB`YN-el;P0v^9js#IR{;BITV zW3LhC2ZB64MLDMf_%p0O2!4KSAwX|O1e{F;0G7`l_9a5QT=iAME4gLS6RVKA0rAL>= zCA0(qKNid9C;7G{*4|O@-aBV%{d!KI|7!pC8Xb4znzFGXvnZTfe>y z#qZ*b4NX}((cE3|Ysv|B0Ism?keN!+I=?(UssakVT-LGQZ%elm0gT($&2yGW!IID( z=lNTt_k9D&`Fmq6gSr3Eh2R_Lff#xjEH4p=`;JbjCkLCMWDU_Y~aS?zDkfN0PoY=&lNKB;$^`)~c0Ut$Uw={htI=Mh; z46AHf2RBuU?rI@I`?eqNetq9HS!xE}vDh8XI-Xr?akM7a>-T|r; zs|~>JM1i!MhJ1m*s{0n=a~rzABvjvj6SEBu@P9?76nSxg?sPv+lteSz9(1-)mz_m( z0^d>U`_HjsoR)vajy~hH$u(?!fJvlI0ByZOPC&8fuQli;HI>Rx^6NsEe}w0TJ#PWu8-!?!1oA^4Dg<&H zadiVbb1>N!UA!n{8MY2-6+O@Dqr?@GXyXe1p}BsVNv0z#JjSQ1g*9c&t)axyH1FKuifaE&gT@~Wh$MvJU?wO z(F%=6WJ-x2rHw~^up$1x!J$sbQ{YCu?nG(-$9;srHAiLTH&Iyb1OEDF5ZZKJeQnrE zwY4_@w9;#M4&yoXgCZMtI7;W!l_j-wOXS!0qIsI{7pN3Pe0_}W1-z*8rDiqZ_Di3+ zwB8bL&tYwnyvS6>Q&?qfc^YS&%NUIJNRZgJHB$2~qUC7LXN#VU{v(U>LPr=@c3fcA z{v32vfnn+-1JvQPIop!9;Uw6*x!jLBZnoI!PvL7zLH`9JB;x4AKHhSRyj_LC)Q+v5 zf)Nl~z-vmjHG+NYGm}#gn`n)t)J}l6?SoHM(_%^?CipsH_|pyIdqqwMdq0%FpBx>< z-1rGs3cSIM$rSehYfJM2qg=-!EB=0v+I@~|UHQ+rg{-hB-eb^JSPZ}MsD8P3eQbc~ z^@Ix>)|m&qG(RNqC6&an&o9o2L&fU=)Y;m3 zf9KaBdP6}#`wI|3O3li9W4(}b@APD-O_iSse<~ixRlz(@U}7Edh`Pd4HinykPImau)DL+Hdd&R zWXXrgw1AIwCb5Yh$=+cAXrQ5fkUjP-u&nC+_AM58idjfqL5U&xD#dLw5n^JE@$5rt z)a^?T(^7)mYCN>AFi0wrq>;M^FUr^C$tF7c7d~8^ZzO&`0A9d24h+=^5+h!?l`qzh zwZ2dW9Ci(K@c&`st)k-Ewyy0!2!xzgoZwoxy9W>M?uEO% z`?s?9Ie&XkYyS<68w!|f&N=$%{TU1Spc^jHH~8lq!@5mR zRmNh-J=%hWO~zE~g-#u9(|GbcqQtPf`Cq7NU#3Y-u9kI|!eS79Fse zl6yWy=pG$U}PL*IrT!}WFVlS!m5gTsBi)AF*1t`;xemObO2Z_@kfG(e<0 zJqeFjT0$v&NYtJ;*1y7PRhx+SPi%3D^^3oZ`FUv&VKeApxnE$j_&jrPX}BtewBBMs ze8}n>i2ENN%AZ*9b1masMZUIrz8UivGgAot{W~95C?n;y%5OSMX#!X>%7YAE zVL-FN9uuHlVwKE=j30f9;x|t9GABe7Mr?2pi;qriT4@BmN0yVG8 z`AvcI{`Rm`h(=4_K(-&k+I|A7K*Y6Q^uwbph+HjBCB|nxu|D3Dz zR~1|j6!#^xkveP+1@uuz+V5O5KHgur=rl$>7aD%>c_Q-k!pzPbBOFQRMGSg@o*X)X ztoVU7JDjRq2kea$V9{WNj(-4)#z6|)Cg!h_nohsKLY@Wn>cl8bm`#cWyf5G4j6gaz zdLso~BAOM>9I)xN(>V28oZf3P>5m~#e`8sVOLpc+PvdvSsJC4i^Ni=M7B2Fi8uN77 z*vHmsw&T(r4^QSH2ndBUcbaQ-C*mqalhbVZ?RZ1+_nP1}0M|W~j&qFfiX?ir(`1h# z`W@z>rhgLd2n!c~Pm}`sWwYX5&bsV&Ao$i`W#VDy6sAP45_$9PU@zCG-kauHh4zAF z9nxcP`=`vDi$PysKPK1|m#5DcE=f zazwrXg;udkdn)9He3FXY;m=F1_^VGn!%>1#s41$qD{ZbUM(fhW!=SGtyaY9v<`?kn zjGDwZTjp?=FUKBGnlJjm#m9!>w;B&~P4va&BaOfevfQ7^8|}#*uYlABD}ovYOadI7 z$btfD9-nJ04X;|!koInvb&)KFUP(Plke)^~*yS>1ECtBcw>$8r}X~NtRAg@a$T_5`lN^aT4NskXw5I6D}$EM^WS+(c};H?pGs5D)XV6nksW6g@x zrA#U|s5aPhU5r@9l!?BfZ|69yc<=%jC(*u4eev@28uR#L7E7zy&IFaret*!}+M&eX zT3_5+OkXL=|BJ?!MYpB#3xxaqH*g*IFjZs#qA@b8)z}E`#ZY!lq;LL7{UN7Wv3Zp0 z^g#IsgB!)0cgZ&{Pd&H4)wPGa!ik|sn#}rDwrrCHGrk5JpNr%Tu{wU|D~&fCNx9rC zw~V5z72SvAJwJKejI{;1943$-AopM-9IqbytA3M;gz zQb!m<;YwK-ZQkgr{X&55+~;;4Emr#FvhoUswY9*9i9N%<;dC2G74q8({hlQ3(Pr-0 ztkG14-4TRF@@>5UW_uJI5+|}l94tKfy0M+dWHHLAdldc%_>z_n#17nz*;c;>@DWwA zoTZC>$FP2G<;njkef!aL&W_`sZC*fadR{{5Mpl1K1#*ayg1{|7xy3zoJRQ_^pHoqw z!P4&Fz4NVZ*p{CnZe~-gWXdz4fnDN&K>8DkC>^{(Rth`deunODCjW;F7Sg)n(34>e z%i*z)OZc=CCEl!3eQrtwTCL@?wP>`H0@vjR)n>yVxe{=+OEC&fhOsj-p{5BxC|}_5 zYJCPV4Nl}9Nx9u~j9a$M`u-|7b--4rC8@LSxS!U~-*%#;PeFn9ldr!UefxJ+X)8qe z!7mnsP#VV`35hg#|9NK~KCXAwTqvE{1qe);;%nW#3TbHPyR{6F#6n#p9xjF}Bu%mC zwAl3O;@Nq_*vy9SWY}Eqc@!`Q#1$Ht^@Jq z9ieRY$0)5y^OiH~2?PnYJ;50@KLyQ!^v+#hn!tq{K}ios^FDCzuo$dz5tft*q2<-Q zj~=D7E27{YXhQHLb>!M0rK3}C6t%7QRLK*SUSAB3pC96(txnSqU49>Fwokn7VLKW` zr>YS#J|~MR_cVH0SQaxN%+Z!80yL7vURO&^(m%qlS`fnP+|JxX)?PQ73(+fQu)M>f zGt4bES12S=Ln;i;Sf|8#u7Q;6Hu9^n;XgWrA{n-RJJrv(0e|ohBE&{0NfvJIDN^XWd-F zYjrWb?hEGTS3WNyiw(}I{R9^`nWrDGGNP_H51IpTLgUoV(IljV+n4xX)+1MZx@(J% zoe8FFaBVckSD4YCk`+IFc(M9EoYN(4e9nwLA;0Lv?yN;bwTk7pi3|-EJ3 zJF7dI!{l^kib;Y@x>o2l##3J8lNqk)h)OgiD5DA5Mh*7Z0Fg`VgQA5dUaLVzc+HEjXN<* z9u1Kz;G;HPJq@PO`syJyMuY;rNGR19*;eTO40g#RmtX~DUClexy9~d_#I%lYx;?NJ zHI0im8DA{q!F>+8+d!Pe`O#5ZJeaz-VZS-ARq9y%>s-TX44AtOBljC6s9_(5KMJcl zkHOvX;~YPc&VCft-Fh63xm^!3XUe=7D0UvZ%+}DT%F51Wv7Zv52b1Di5pdkSUEnI6 z-Gm_no2|4P0Yk%ghC*y6=bgQ!3&2a0&M5r_;seok4a?;8?b2&c{MISu4v61Qin8KO zbNdkF@qcGa<+6anx?fnfy_$~eCw*B;#^!OdYrEU%A*uTmo?O1rJw8z(9}e+2ep7P- zmBK92Suhot31H$TkdHa|iToU2zi4P=$l_p!<~V7uibJYObufsaW^lJY&*QlNTO}b; zv+ga-#YM!)*##POEWZKbL43q_(~i_|v0Ef{IGQFDT;7%#-s87?7)c77v2G&ua5?ZSaH=E&L>!OzWySPdKaZk;{g2|dJP)$2 zsnzwn>RXF>fnp?V3qL13e}^MDIE$pDecW&i57(C;N7^&mTP5W;&jP-`Q?}UsYDmAeiKeJn4nDB_oNs4@ zt=d;EP^|WT;n#kpQfSCqVByA4{c1*mSi}$>? zEK4BJ7v6u{J)j5q+i+7Oy4`!rjb@ZCOO5BQ(M=a+U+w5dF>1Te%MuCSy^7tJDK{Sd z_464G`Lok@!ESrggY7G~$6bTyu71&dph9HYjgxMUq>zk%{xvYaJ5CW~ ze|>51m|l}lVZ^{&=egWKU1c%}p|@GCfWLWe$-cxUxWQWUIjq}HAncTlr;Ws*l+Er7 z?fylv<2cZIyng36uzY%bM9|n64UUKUdLPW7TqLpVU5TP<8zSXfT=rn(1QCt@3|)Ftsr&%oyBL~a2pE~w|30E?W)lvlo=L!hF^ z`Rg^G)!w6&Z{?yXhwm&>5cWzSNRzlUCvIun2btg6$p^e2j(zGyBTQci`7(H(wc-?}kZK;byhD%jp&6$v7>t4Rhi0#T>Huoa{&xiG-66_1{})7g@bADaB?W%;8~J z{tbGwGn(F$-u4n$&_~@rknyf&d8KUj?cF=9PuGMESiP;)ri$yU&n=BcpOH`sKg->v z<*m@KKLK+0bkl#ZGF*AQ(u73Nrts5yF?+=htLD^BkI$#EM>|mMX zw~m76#wYK^A|g7?$?f#Y+|?giF|j{i=S?*1h`_ZZHjATdHROBV=%^@W$XqF*s}x1J zc!@h9neU}1b_s1QRlre2F~JeCnF&v_6tZkWy#I)a9}3!G4QNi~5oC;xrU?efWFu$| zV-gWhm8jzo%uq|ZSC)J_A-P(45xMDUJWnIQ4&Dmg`Di|umvZ(L6ow#CU&$~9;i|7$ z)5Ot+w;s^GR(zu35+r5P6^J~exOYG7rUAT=!F0^y*ncgfe_Shh(zw5;n4TA0he(2# zZ_0Gr6S7^~TfnH?c)G=ahVZlt2`65u$gunmyx|4|5b=(h^!I6fl_g zHg=QaQl*~_SHK}!w^}~XmF-W;JG?jA^+#b2*mUy(3pv$OX~NdM``Cje_qe>?A41?L zAwe+)c3)n~aQQ$*+m04)Tjvs3Oc=%qkZZTr(XW@`ac7Vbg?d|=w;AKNK8#}(d%cZM zS_AbQ5rAsTI@lR5f#w;bDU2weaLNu%D(LXnn1{?*^RHcT*D5N_mh1KOwF@4xnYkVb zzEv-pu9%4H$W@(lqT7s>tV7~c9Shx;bf!VLtCC>nkld5dFP+s<&Og039xHpe_Gon2 z^)%eS&1~Eo*>%k{NkMJ=rK~V8*+|vf<>lczuR{%-C9%;~@n5|i*?LQ>UE@9#X3J{D zX6123%|BdpL0Yl*A-*?Zz!J@|p3aca_U5$%s0 z&$QcC_GIjT(H zjI6s)+U<2{RhzZ2EXmL+2F|l~-hcFL=iAFvkaS~XI0!JGJxFO@q7KJP%?TH@rwH8kn(s*B@!?|Kd^oLScS1Y6x_`OKKe{#W z=m_<%hPReex*K|i0iJW7ZUT%vpI4hb%JuPb;pekbSAz9Y5K4@Axj;S=xxoqp=dAU; z0e!QIs^u32*|AY!tL{<$y18i_%_;@bm~xVP4&umEt#oe9XpC{#EFC-gmy4kYL^n1V zQ$X8o)S~{Q?Tpd&ZnyY~JHw}U6Q3auSXuD##NOcb5&f_>*M$jyVogM&AM_v zil6Q|XPA!Vy~gGHDN{BEpL{;$Zn%b$M?HdnF!Zm5`%hJ)iX!|+-dT=Op@z17Mbi)c z8Ha^y+6_H%-=M+8}ayrQ)jJr5-XaL<*34B;W4O> zaIKX#sSQH!{Uo&kR2!O-jmxW>wbyQ1y=(I2J|4(WkVoH7uQsFp$Gwogoa<(bP9a_R z{HTq6G1CR{2iIxO_zA(wAVwjH7I+0tHrJPX3X8V02y9r{rQ5A_-hTX=70(;58neme zqNmoDrnBy>r0r2DrR40K{F?lN*_>sNl48$aU!X#Ye^|n)aaH*+H@csbQ&_*%jO%&v zAw~V^t8Fuw%bH(XH+dG*$~{8#Iw4S@gqikBUG8Q6Yfz2reroqIOrDs8knNb53D|{> zflUd!L71T`5atI4Yw~P&?F*CP+RRy$G>Zz~R~qB<-?TGiBHk-4nA+TB_R^_>@Ybpk z{VxyRuU(E7{rH>@GQB&y7R>Wd`(Mcu=Qdnyt9HZ|AZ?jKgzWwADA?#!k5OY1SOR|L zDdV)ZdiTW(HXWDd+G_fMU$1$5zBz(-dm09@+~PWgPwt;mnJN->0_{Hilglxya({xA zR^O`JT5i9`!i16Rkzg9u7ru-qvO^ z-wWmF(EwHAapvS}SIG=@aGfvV#XSYJXKi}($ZF?Bkme^GV-Ws1@*NyZJ zRKG2MF!Qm*U~)S-wFnF2>G`S2?bv<5CoM6uM{UJ;iu+0aX)H8APoE>@jw&=lp>_+}&jlG5rb=|_lYDA}d4tGKg?@4ah*d#K<;y z=nVd2PvFO1KD;PO{Jb8Khy8HECF`WGxbjL{E57_izt zl0@D)UVqN+l&_NElc@uoAUpoYY4-kTa?v2urw)dTBrA*TzF4(L=mcB75XR=)Gx;FX zblQ8`mk(+Q&(ymfvCj&WndSaIaxzszqp_pv!t_VGKC*p17G~fZZ2PcXzX9~>Eoe?O_>QYIM{ZMi2A&*Vw^C$-@K>HKS*d)Q}LZq$KWQH z$=Ng>-unD)#to^*;0?j7;e~7QQRy+)l^b}EOSL`ZpG4;0^6HEDJL*3s)g1)u*TsmNndZNaU%dyVvXA(K zGdy03f&}x&I!3=Hm*;P;HL}1ZGW^vm!gZf}!bwVR)hO^!fQKluQD?T1*Q zgNuk@(`mKd3Ad9F!5WC@!iVDWgptmZCLfuJ3aVe$Buk={+Ls(?7o2oqLF1i?r zA2oLuTYVx&;~cH&MsRBimwKEt=ju#a8k*{eJS;}aVkRlh`;9ISr@pF&GyOrsXB{tF zL6OUD%(i^$Xkbrdu@G^vXql(ZK=}vb;P5WGOurVaB%zN+?Vde1VJgqaGC+c zO3%-ROelk6^RY-Cx4t&XT$@|6y#67J07j7HEWp^)|COI2k381~BDv0WRNbHXl7GRyN%{7>w`RFy1NiCbeSnh5$;o5bZ zl&rkyw9$VqxiWRuDMD7)D?bsK(CWUCJYOBs94e)hYRw6sO#P1DCM;4D84AeZ%6CLI zczg@-qA5(Tqs|^iSA5B14=RHgxfl>mOcT@B%K(-2(AMuz+uz-DK0%|pH()?i(UvAF3K-9I3lIY z1gzg*(QkVT-QHsQt0@hQilH%}43N~{6UG&**MEIyI-1&%5Nev-dw%Sp5R22uUvHay zFafGX-cf?TvZuID4quK%x*B#}p6$yA8hc)sm&(-;4Y+;Hr?<5OmMEes12E{n8FCU| z`E44tvujeV=EBJv^pR!uV@}vNdcCJze1jBlQROL2x+G6vCv`d1DBnR7JFQ87bJTx}Fpvd@ib%4UO$u53sX**YqEEnEJ zMN!9qC8RAhc&4zN4}SQ_mlT%5Hp&p$bGrKM)zvGkK4Ns$k@n)X>)T_{OAO}frxf3d zC?&uSVgpA;ZkIrFn5S9ejHJ*E)~puxmrIiL*CDovYcDUf6g7=OjnzXah1mT`o5w>s51$U z2obwMWgH@ju{KQqbahJkX^zEsV6&a z)$WeLQ3uPTy3X(t9^u*Cg{a4R62+T5_IYbGM90LK4rV9>6~}hA zU;Z$N#zaqilTGyzzf2*0G*#do8u22xs}w}uc6M_4J}dQ|lTn{WX4EP+Jpw&(AGThZ z;g**Id?)jV5f*OOt_pk?3gr^CWJa$PKj?~ z9nh(9G0Sc?nXoZ+CVr~S8A&C0IfIA?+LIfdot4V2-V#V6o^Yd<*SEr^OUG4>xJnbY z)nt<2#TEo}wG_w`mbN@iS85AmRlh|{StN|sbCN5UZ%FveQIM#<>ogHG<3t_R>NC)!p#}3laqd@f$vL z5yaWcKPrj7K^RfL;a&?}IDuU*xQ@L) zA1YBj)KHQ)r1!3|CYLE^^3dEU4vbIdg!y7S64UGS_AK}= zO@WysF&D=_n{Xmq$d+Xi%y-lOKQb5!*F2t~;1P|0cG5Y!3TDu%rhVDNu9adDOcy$l z+Z?(|1>U`y?E@Z3QV1Xn*liwB>zjiDuKqwIsArl!fhloPLnSSN9wA%S4#l(9sVq)b zzeHBrxqvA<=(=NiFv01HS>;lqd;Fjr{P6zz3PV{fPg5T6sXBT0{8_$|{q+%tNx!(zrtwjyctrcQM%?v+{rna< z=Xm)F0nOEW(Qa*I8_Ib)Z^Bb6JMU?F6zR&K;?yQ%c~f5rX3INrk#=H#rc*Jy3AQ@@ z%7oIFe;r2A6(gCbk1>3!nNX>^lPTioFNdC2_4II3VQ?gSF^VE=hRP=Pa_N)8d36aC zE2PZWJZ^{tWd(M~(gdkI&iYE>3a;?;%6F@!8bw}kzSsqit4!jwyTDeW{uP64DI=O? zqp#>SMb(3HpNtdSS?B7=gWI%C<`9-IK7ISMdQtgg+}*Rrl#@+t+; zq9xOxS4RI|1p^eIUEp8wcPSRJ(CJ5Lo)=CM;%JIFLdDqfLxD{a`7q+)<|Ii%WqE_@ zMeb5B7r`m><}5piMmrN@w`9CZS#jq`yH&MAnOdEY(@cASTnQ?xE{;W=6cYENRA8ci zve~p(WBYp@GBk8sz8#35oYkGl%DQc<@MYg1)I^ z$xVKtF~r6DrOpF> z9Qv_d`7T}!aHjq2;wUh6H4^zulhiM!>D&G=3tQFFrZ^0%s*M{m$4O2{;cMtUP)888 zhshxB={=wlgR8dm>!LYpY;hJ_y(s!u5L=Lt@|B2to0Fhe$&))wvn9Fz#;FhD=hTIt z@C7z6;2Dhp0H{6dB8Q2*gWMEGEXLEoqYMAhdyzL%?p?Z|%YXnq9N;W!2PZfltC7V04%`L1%E^r$GncdK|b%0GBWu%y;Qc4uA#nOB5^$w9org4(Pbx~0v zxA_pMfO3oUJcgM5u0`*^th5UqU?-`a7anX4)^XP74ZgzKHeJ7z(3o!3Zc@oE zhY-FK{l$+x5=R#BfT}6Vt9zS7N>paSao~@7L;a1W`*u2ebgPk?}H;J_?Qv}@Tc=PZe>em}B9X`=my*sVBbLlQx5Y()&lSE4i#ZRar>xD1UG|Oi z!+js@pPn1pA%CCjb3H9^L8p(il2zC0Ca$+H# z!As!cu@{2Ef*TuqzFp`bFj%4Vgpa8tgz)*y}DIyHJm zLN}V->|ifu5o^*>aHB!+XrdYwD5o((H^cu^PQStlc(rYXiKc{wU`nBIcobqR0E7`8 zuXh^lAkHvgl4BD30-#T+&2L+fv|m)Z0ev`l^N{PQeR ztZv7-#6qqFN_Sq{g=SQKAE=G`UtP>O(ey=xcT&^4P|z{2+Fv8x`O_X7wv%{VF?L2x zZk|&)z=MdiKEC0+{}sfe`$49RCwfwkF;rlQ_OdA-3+N3;un1}dO}6ss8aT zn)yJ>2nKc+<*Q6duieaIQh#lj($ym=1PFjfA-sOkxkJCU&mNA`<{Ir$7%o2c3Wj%H zStf9uAO+paX! zD=1mVtY4mgBAYFovmC3#j$d*T1v|8(5%W$DJPc1!>xzxBJEb5)YpAXq5~5A{-G5W7 zTh-Q@qCkY)k@cCzw>9hYiDt(!-VSJE5FKa*4A0|C&{e=!8^S{5|D20-l~oELF#`cJ z=^B=Zx&ROX+MLz{*513C@+CLsg?lAA zV2|7OeF6M?kSmA&S9U@9fedRo5|8baW>Ttf;02}Og6FE)z=l)>(+wK`-_j&)`wzr1 zm7yTnC!{O<2CsHH*Ja7|w0{S&p@N9Sf=I5$86?kgOrI6Ujs2*r!sU8 z!de2I99``uCFJ>@hszhY;T%en#V!L53ZsO zRY8$y23J|4cLZGCP4DuR2s;r-e6swjPsN!A9djZlq9zmdCc6x50&7`VAiEJkH7E@VQO)sOmb7xPIluC7aU+ffqUjqq&p3V!e(<=k| z66W_`B7B+^wcqt)+-=H4m!Oio>n+@nI@Ai(>4k^!X%o4$5kQ@Pjj4~D+a;bIyH!+= z+B~qD=u3@hHGdUDvqGE$h+D~==$(p z>%u+%_=$6EZg(wP3T-TZFyF263nDTB|FCR?p56iA0@Ynsnli+v7IT{F z<-f+jKgL?2=wI3n1aHljH_odE+%!>b8C)+ctZMn6*;q?IAE4A6Pw90_wC-L@Ig^284mUSKZr2zdLXc9xpr=5eo)SAR*9=K`)Z`~pX_u##f z+mNbY&du)=AB8c=Ry%Xjo8-IND*emLPja3o{kbPuyN1MuG)Vv&uHlih5X~JKo>3h6(aO*8`FK} zLOAK$#yE0M0yA)GSBz%M_1pXW1g(};^f_GVPy=5wf5ia#vY^*)-c^iLWHjNxvzLe6 zr>nB4#otS7fAI-}4_^@e2cOWyJiUC1hlqMS+^oT=n||EJLr0t<9<54X-RpqH<_QIEeKq>q7gcy**XETvU(V*JW;~ek0Fc%7_H4j zKp{zK1xsoNgi>vs7BS7v{gm0qiz*c77o;F)tpkqo8SZE3H67;SaVzTBMTg)H%jP-uLykORu)D zZO>#7ikVv$xXX>UW5>5!bxg*Vn;vk+2EX;%JON>yN^)hCXtKc2Ue%An=Gk3wO5?$K z_I&i~VGMLkIzK-xR)-FG1rO&w=f8f`rV4TD%fmDO?d@%rZh{_gnaa>Z_OFD#4+#8& zKm3QSTA?2jIv9JMI}lqRoq0WGbN*RQ&1af#6Dhz)pUpOz4Gl}9MJu$9MvX%nu}Qq1 zppe4~O32Ab{$^~4ib$Sl5(*$Hs#{xu9Z8Ndcz&|o78zQMp@XCZPnzB?rEG4^;5)k9qb#vBdWK5k{y$rP`ecK)3Slv8~fgV zUaLVc(diRl^gQH`#STgVg^Y>*a+VK3J_~Ktv89wc8qRRlPFB$aY$Th;l9s8%SnkCC z%&z=1fAWuOoP+Z>x1}^0qsk-Uums=WQ*MWI+1G_7ZYA)te_+zLum2dCIyeC<2|3!;$-(Z zRB{K^rLn;-cOh}(+SIV~fAol)!teX-C!0Q%+GJnr)KOn@Z7JkkIBT9{>Y9&HJ@sV{ zvz6D-79~@%^OfhDp4p#mOV#c-2WSAf>y~By&zjh6)%hEr+U0$=-ibHt=+8UE)v2z_ zmOs8T3RfpnRVjTsTqyR(qeioK4i5ADv7ENx%OtHVGm@e3*+9RH(CVp))6268XQ}Ci zC`49WjKlin;fBks)I)U*aEz>~L>E#t>M@Px>*=bZM1LBIkv0|Hb@n|Z)|%rQLL~Xg zKDQy_F>8D8FOtNZ%#?lVY86z(tBKi95Th1$KYt73r!}Nkf(1=+kWEC5l#46%ts??` zK?w<4KkMP42WJ;%<%*^V)o*Rd^Rn!g+;=ogZZCFrg*eU9Z+6Qq*Une%uz1{l1xZtL z+sAq&3HV!M%ej{dpZqw-rdudcTKgJ7B3R-MQ9HI@x4gsnZqdA7+|6@8s{dW9jKvId z1K?3<*=WNaD_GgTv6#I)jvtkpgkm612ZT1ieK%+`@#`{I8WxR2Pz17gC&FB{e}3BT zB~K`*d?!7ck+$eL=D2S(k=W(A43?CKb^odTVZe1grGHe`M^&&ENNH4%3D}pYEZMq> zd6kLn1`rejkZ(4-P5Xl(;}Wq5Hz)0*qi9;yTP(9lyza4vA9vOX{RsUU#){8)ZXe!! zd>3^tSyuuBykz@>OcMv+EM>hREbz*;I#K?+({<1Q=pF_sJecVm1^?GKc_42vjOfVv zYbr6vF@z4N>x+(O7|L?qFDLRF{cw-D`5ZcWY6GUyV7orPf1~fis?W=V$Li8(^Q8rm z0%5h|j$L+P@1k|8rAv~#qTg4MV4EzP>HvgMfCBxub>*s0rsr3WQ zuwqRnxeLy{5LAHZ2BwR^{}=fAv-I{~;D;;iVHijNlAfM8cWDz`Mn_6Hd4sN+Mf+*} zC#Z_Tp-%Bqr~+79ERSfV2sZ`6j>;9v28Z`12pt*>?CHX7>g8m~^@_*{3Ik*F=NZdH zLmic`J29?NT6RFNroB%bh0@mwR%5XdIgu`Z0xxOZx*f#=F)E(mCh> zUEFFk&88BiZ`Jv^#E(g?75#aN%_Of;+}X`xatAPLn+7rzT!P7-_{dn~xW+)$0{kyq zz1Ck^$n8f7(b0eIU9`b{L~k^teV%KX6xj3@@5z*Jc!$6JaDCi^Gdj|G1vF*b+rxzA zo3`c3h4_tE#D-gTQiM%4PtsaVC)`iC3eWlB4EZBUx>Zi@mv|hq!GRe}Kup88gDWs` zFW`T#)#?^+$dm7QizZ|dKD$y9)Nd5Dx*kv+!qbS%W+iXf^+lEBFCnNiN3AO#K=`^S zpgBZ~j+!qB$C!_E6>zfL04Hlw%qhX9Q=H-jOH57kpxUVyOQ+cxx<%(bx6ZzW7L?t+ z`F=kvi0f*)&PYjDNkbd-cs+=EIM*oZez=o>mrXrfYc5bjDGqq6cb`KN7_1#{_XVbm zKN0eMZ&d%in=@BDOS^Ae56fbkqChum^RDfl ze>?Mc+pU>>Ca5W7bWe(@Zx9U7liUqmGLBWWS!OhW#zZa>3$j_HjhvuJ{91&v{87nL z1V#`xjlP(unF-iLQe*&KGkf#zxrm*AK=L`z%xt zh6|+WW%>$=CiNa81oHJM7#ctSXS=`o>$}9Iil(5q=$4c1B9B!0pF9xBrEekTnaXVC zdY|9Er%~hO9ZwYVQpGPMCelt0p9vQ-L%knW>-#3PoMRj|lF$}4Ep%M|+Q)G1kaqh% z<~x(o|@CS z7t_lEj7l_Vn5K(n$C;JYg+Mow$)%Z|K`q0L@w_>OS%4Q?#%5-_^2kW@NiYrwekp$6 zTPB~O3p>m;I$t!N-;Ir{0{m*%+YF92U_MKYa8<7_t=9rFO*{v*L8_)QM-P#NLO*#Y zF*~MX#+x)>x}QQb%D_C6Q>ZZPe(wC-7lJvCHtCpmnj(*2g^Il6}@a zyqdj0!T{{)3pdL7OB2=_ebb9j)TP~n3CD^hcmp{KCkMwFz|YtmiBYPDcJRj7->th} zu!G357d$``M^9FFsbUST2ZB>&tU5eC_;Nd=74J%u@!_qLi$7zUYHw9&oM)q*6fcI9 zoI0pC&CqUv(z%VA-FE?yVek9=dIOZeWK)LQqJyE>?%JXUt7D0cp#i93``k|z*m{Lp z)D4uG6)Tt4xYsY0gE}`$H_5A4_4^CmKbm``Q0pShqO5v#U}^`arCKcFanLpoSL6M> zaM3VNsbG^|zenf{J+MCoD3Zy8b=0xmPvr`dxU@q=AA)FM6b2V zk3P(+!#&JxI_DL0-}v;8AghDgqsMuv3`Zz!b;8;mj>AGJwYn3KAB?07#ul4Z+rBGQ z2sHBu*`j>qOu!ZulV8!eJDk0D7o7d_t!ML>#!{mp+7TW2dS>`QWbL>xO7ncNySSv| zN6LBNf%_G8cdVPU-xd ze3Wk*fyk86f(W1ZNuF#Js7$;3<<9%D8+BH)jgS<6UT;EkZvG-ABd-W*COstbRA1g( z8V<;1pGCbI3NAen_#ND!ElJ1Ss8YSzpF)Cy@8b^;)PBJ1jk@M#w+ArO*{zx&#+woX zY*Ez*O+7Cn+t$fHdG>KO7M~+b0G?zDA^_4ay$W=0-O=W*@I14sIhl|$BAj72qkjCw z_%lNHfr3TugKk$mS<0SoQ~%4?EQi{C4TCb7S1mfajc%yGV1X0Ql;fZhUzVZCLB%#A zvsGHj<>cU=eBAM~D}rh-T6Kf588T3wCAW&Eodr8+(=SFy>irya4zPin;tIKxSrbJx zbml`t2+ZN#k*O76wvGzkz$W`lzy!QQ*wYisK*DJQK)Q5StMP(T0Aq}*bdqDTeTGeL zH2z7tQCYe;dD%3PX(?;!Gj&W($d3il*{D&wj)1wm+fLPY-V*n zDhBsHV;4DN1lwmfoBR=oKvp1K6X|LI6wfN)SxN{}-M1djnEq5prO`&(Zhv}^EqZqa zgEVdZ*!ID#O{UtI8;m7s+n!3>=w7%>6pl*HAPz2cCX0mC*T1mLIqw62r1Z~b75CwX zO+(TvaZ6BY5Tn}wGmpESnE0v$H1w|1fmo1~>&h#S@ zOE1hZ12={WH(Z{W8pSF9o_tuj#0t3*YJ?~`YQ={&*9*y1y05WB*7Z5FYLl*t1qHL;;)-PIszm4W zNsa8p+Hts_Z@!km$+%Kt-n}j{xhHZf{AB)jdSt?^YnWg*HnsTkFQsS#SmAn6->|uV zQ{iMY#7Y+2B6#l6)K!Q607SRkt3k08pb=O6*8gZe(3DUrb~hRrmTa-bsd)6V z_(0(DT!GTrt1!!v&Sn35UR&qMx3yE~t{Dj!s>@m+3f~JVO{uUR1T~>@=~TMt(1N)W z%X0dw{)-C!!woJ5{1X`iI}k>$M`6%#uy{V!g{ zR5ivHf5A)lT*e+1eED&#cQT%h@TI|%m1gm`US?|lQ&6}R&}}r1a@&3ZpKzCahJ2PS z2DVuZ+xcg=%kGQ3Obcn0=I8HT)r6XOH8S7+PN;p`bu;cSx&m=Kf4427ZM;+<8@eKI zc|Ln6zjZhxL)dP6oe}O%VW{uf_0{I3g&PZC;=FAK|(9MV}xh_{H zQ&(vZB@29QU`S{75p`4r3=YAZLG+`hYl+HrO6Q%_=7%;NHUg;^UgKQ&v|88w84^m_ zgh|ejm?dh%my<6o>qwu3WONr|ieJ1h+`;iQ1U0%X9RebHZeKR7V8-Q2_andul@Gh9 zXk!p{W7uN_5kOb#|HhO;v$O5iYH}hQBhBHph<9JevgM`uZIZzS@6~A5o5lunL%j;* zg);B+JoTJl3cqD$o!;DNaz#o-KE4UzGqlYN{bDkp@#$GCKs069Qj>``r7bbh%tM04 zpo>nUEF=R*ZgumF;%|qmNRVb8qXkeYoAk=KY|B53HJNF7rv$9U%@HHH6J408%liIr za=0oA*X-W56=L3(QvM2%|0oXrtUcWuhx3>`uJ>wHBlN}@D0CXI#;ey?SM70fIRCG` z?~bSX4gW?etB@png^-=SM~IM3X7-3LN zkM2a?)V_eAgnsrrsQ2%@*^><}p@QpUnqv8L`eKhB-?oKTWJ*apnJ6V_RK@SwH&F=W zIb|NUJL*;4y6GdxzpPhdcR8F=xK~{U7>(lah%8tvZjFsOLWFU0J_wuzdgD&7X`v9E zcOUak$hFT+*ybyUv~!AGTzb%(s&Lb*H?^)G`k_`l{&7o@Owh>jmer`Ir`P>5UAn6` z)wJ$%yU%s9%zPKt<6Q0igzNL%pw<6K@@RXw>%wzukrmpn}Egy~;)W3HgeOBC5fYr+!iDBJma2yZE}DzglRW47nrI)P+(Iqo-BXZm zoX&2(jv*9SLhitMj(ReU)9)&Qp}%u5z8MDUp|{`ss4H2~#o628nsP$7<+StN7aMYj z(a>;TVOIEpu_P5JG$FDa$gw2Xsj>f@c0`^PkwmK*cldnwVN|RyHb#N?$a%aTS#(f| zi-Qe-7uS*n9~1oA09Y(K-T-7El%qQnN`hmi?}h_sP*=wSU}eX-+cwRJ`|UxfkO5 ztUhE=&+JZU_;pommF3zlhux0eHsP=wQnbhc& zLxWBHl#d?0C?V40x*H|9^)1=nV^Q7{LuiED&va!z+nTwo{FQX5=t3mZsFu)Lzx{ee zte6$hKDw8t&4C;Y-?=AaVXya>_F&4*9*a^OWB+SRK50X6R&oUwqxbd$A<)%7Djnw# z*A?u;T&Z|k;COl#z|H>}2N2ZjONP(IA88RjSk&1$+}c&fSgd0LNrztxFCanYha;7` z?jfms!^OTMFheTor3Pv>7v2R5CIUNRR1rgaV!3OPXlOMb@1Y9We*G)j%YZj#%2YoG zgUiP+#gFg5zZpt}&0rsiSxi$gHkxd*BPfpWM`-%pcUs+Cj}eWaxnHg8)tj@G7NFwb zY}eKx0?%Ny{){g493alMQ`fCHv*zQ*?9dGIM)PUMlxphA#DPY0k_QDpa4B5ew~G{9 z`lDdxxKt9TMWCur^5QxAg!KZv(?0>s8bU5}EmKilKAs3GMRigdUiHKg2@?BV8!E+J z0nh40;%%g52<-zxsICfDC>+e?Eot|wI zW9-43nw%;yT-1VA`ExClHDE7lBtLgmshzd5cA+detQF_SrGJ$K&eU)$vvc!dmWh8jYuVFkV@{D0VQ>g zemsoOL6?%;Ea-=6Kl?m~a2t|!<2%7A{kKe8+(<^|7qG)~a`EArtA05fo5(Oh!i&@nKR8N^>Zh`)ka z+^5o)UrF54z%^Z7r7UDI$|DL5dw%>WfWni$2jzpCx6Ch=<19__ryDB7`Oe{iSnrBs zhV+9y8RQT%o)h~YyU#=yq_nRUj$O3KF>b(Z1w07k>=l!xFiW(wKeaU8C_4R-Zu7yDz14b3h+HMW7^^ zc{7Wy5*0U3EHXzcrN1$Fb~WlL`vufDuh3ZoN^rK@H)#!n{?h(lLp<=YtNO{)oM^Pg=gBe}4t=^_I>C-geC|@)0|3UKo};ld>8{4eJYx zD`?^<`U8v16{aZ5qprl9f2bG?A0q}B&mgvfA2%;k<*DfAE{z37;@%N*)%g6G#vD6P ze}Q_5uY#gF9KJH4T)1$oQy1+iJxSOji`5CWF49ZZ?Pqy^>k<2BNn>Wr);H|enZ>`# z`LMRl8-`tOlG2}AF4*8E!B)P# zAC@qTd2rC?()n<8D~LjXMk4|F*@-yxCu`ACE$;rezsfTr0!pxQf5g!m<()-GuY*4$ z-7wa<;8#B$$gNdfi&7Er-s)6j6u0gALucSoH-R1aCQ((r?1Fx3mTD@R}% zsmgXrXzYIo?aT~5-+BWocV3F;OMcT9G55h-kkcb7p&(#n^#GB+s`AvX@su^H)&PH zP9D)eR(8BBzl)v@0mMWf?uGer29$=m93M0=B;-Qoi%n!9y1+F{@rog}{38x-h|n!6 zeZ{&`a_+ZKGAL6)5IJ|{UnvAsgGfMB;?Q9qX9vDJzQ7QVfFZ1iGYryy(O&;VYZFH9 zSSjB@4a8`!FhWe)NbYApHeLkKCqSaq7aOxcTn8K6NnzF1 z&0=8^y1k}EE#>4SD(`adW0TH1)oSbLUyx$@O^>ms^mr3I;yW@2yd}4Wf8p_a%M%OI zRk3QMl8#a3RU^kl$ZU#%4gDKeir+c*I5xx{fOMzvAY#`0`0u;ugQiqvjl6r2qr`PDv9-y7RIv@nDOfbZfdHTCybYhLCxFjdss8 zPb~gJ<$BcjuO%*Us2GX*j!FcMDtU1obzXp9y)rX<29$;wUfg+j1=yC3&;`7V3UQ$a4@n4=e=X~L<1czuo_p)S4BnuE#IcG*_()Z- zuP|wQV4v~_$w^JkO-nkNPdCpiVi?v(8r6s6?PVu!2{K7z8)t8RdGc3=uR*eZjk~*> z9k8q`3M$w%5;DBbs*0TAB^f91WMd5uJE2+Gq{}Ds7JV`2uUO!HC*y9bv01%$8OxTK zFDe2FWDz8gy(%0(JV4vBWv7)$mzFe))B`xiOwGIMs*2`>ik_3sJ;XSLiGCRc{Af_1 zSDn%Um}?zWVWX*&59+4VG7dcJ2j+@5^Jg~H3rODz5ELM4sFV8c6~lpV&EAtQ-xRu? zU4}Q9Ve9s*aI}921?4jmn~C&zk>xi8`bpnfvSmP#Crh98NK{e-uQ1X#T)f37-HN6w z)(~+ivv#M%U+03wmZi+Vm)!1E4yU)>+nHD2GgC1QUzSa7OnqB}8u%IIGG^=NPZoS? z8BVi~5*S!^oxdiGo(dZa1PviNhjcoX7bc&>*vq>3Ooe4Iq=>ZhGUCnja7d`YW$LvA z5}6k1hn=(du6S-;58a@@xHXY>Vwos#>_rQ;hoNFI3ghaxo2HcudFMU|FhuD9n}SiK zzb9gn4H*t6D2;qiwaJ<~giNoC)G0e<;00eSN0H+byv(>Ni*kgrhDTf-5Z*=YBbk{# zlMtk3Zt5U-f}>ycV;A`450N*7VqJ% zqsgXxp56xWi=W7kGczaW??YqgCtR*BJ^RVn91Si8AP7Ys=aS)#{gGA}^)yh_> z*Ot1@r}E`@p2fn1ACi6jUNs@tpGGWL{_Qyq>XJA8)?=yayvJF*%MBZ>j2)$r8$=m;o;Nh-Wn|HpCAysWfXoiHt%~CPoRM`m*rmkV2kODWXsZBN0y} z`Pl})cAnELE$gc;qciFNVv#i*XNnUhFtTbPHBpe=sRAW;zHAT!Uh5n0<_oZ^pXN@e z_#!5s$Jm4+FNJCna0ib%vIFp-PWIL)$9W=1k~5bA7(j;1QZ5AZ`(yXP6@6W6^Ox@l zzB3t?yM!mov>?5K?5+UrS2gF2l@o!r&VRYTCil<1aIKAO&j4~N+i;+Ah6;X`gzN)ne3C| zxo!?+NWYsd6dz%H1U)V+CvKyfcabK-zjcs;bpC>V{#Q?-RL&#(D9{GqdQ?_k<2c|a z4$!S)a>CvFL`tr|Pb7zO7K9_hlVw`Z$ln7BT$XjqchALjAhw5?QI&7-^*$QnnPzlr zDPk#KdpUzHrBBpF=jN0`*g#C(s7-6vjN(Smh1RKN!Dw4q!FH|X4C+5Hh{X_2d6}@F z(mi!$0rS`9Q!^TAsc~%hrz_H@D@2fd(@=`8;xIPxKmjUAf?P*X;1$wm*4!q+#cUXO z`~6SG8(HJ?_H>SUo$fK^hX>M}zEt;rcQS<3Oo&Uz02 znpByj9d~IIZ{|j*scQGPOR`dOtpsR%5KiYkj6_0d-Td!N6R!jvjYiMd{-p-`%?zB- z=6Ds7FRljyR1She2#Ct03u>XfN!Pwi=`)&F2pz>&(`Lu{oCsg`r<;FBuN^4|DCzvO z_|B|3lK!kY9M4_SK959`SToh>n&Yy<+di%9IO_fh<|6SNH&<@{Rb&&AwyzYt{~-C` zg6C{jqM~i|5QU5gI528JBF2~=nL#DRh#HA9F}GlUmr5VyYW_af4OYp(HgMmYj&+X& zCx(lZQ;%`r9@NRH{#N*zRl(1@zMU^$45dH0B;ET;VznV3liEQb_jM$Kxo6wUN)C)EZk-9+&Eg&3yD&QD;*UMGRr==0}yIj$j0`$4X+vIPq4&ZLC5zG96FgJ0R3HBn9IFl%U@VmK=% z*3i;zVO;m#T;*8G`VL)&cucQ&w^7LH9+T2bchGVWZ;QS(oK@bx^oopvL|q4CNky~n zq#13Y>2aFN^Qli$F+TmTRBY{T$OwF15s&OU#NTXle1_~hq+QtG&DrR%7p^i)6nlw^ z@d{1+dmxqg5iUCF&9{mz)4FVT%RxUT041QYO77_H-6OVqtch&#z?{Q6Z_S||LGIS# zd%txOQxFS_es(UyxD6}4Fgvm+9v!?h<~-iSRmPxVyg^zzXHw6`WUG#YE`HkQ2%XOR zQ#{8g9Kc5&P?87B8e1FwTR+eAi>%1VNdx#3WG2L?h0j)If7Vw)5moaqM^1A0&zA|< zAUthlk$KThg)>zS^fhbmik1j4&1Xp|ugzb(a5VtG3H9nZ6OK&^rp?36x=2>QJ zP|BZ*Ltm*t@$t$-X%5Sj5`Q>HFU3KkY;RFyWU}5mUR)j*g`kr5$qQyAWee@2GBK6K zzvY)L(J;Q#{l({PBSd0*X=`)moJ%=vu{7Z z)c#0c{T-`xUpd~nROQBC88eNOJObAuLyqwv{Q(KXZSAavvYQK{)tK^MhMyJ4$@;1j z#IW4x#BJHS;nj&9KbB&h+34m0jI=T|@ry$=$uXXo~D??e?o12ECav%}TnX zUGDPI`<@A~KH}B>z#0Lj{IJlhb>SlMZSqxJ(|z{z%QDkx!LF=1$!+wPO=pU2dV~fj zQe0;Z_7gl;J;RlJ&@qm`dxvQL(_+9QAC6+%+^;Tk6Y#~fBheWj>6roA$sa#ENb7mB zebnYW7xyXI+;0F`RZs^=6-6n~hl%D9#z$jgu!z-wj8Rc%J0B?G+bEHRR23a~*Y4Gd z9}5Y_10JHzyPcZC1;$;y`QSd{2h3awR3a^nF5SGD2!W5fap2@pZK)TJyQcX}->7kW z`ea9hy}Fd1x_?XDTobe5^ig?sa!087o3$Z}Ntsc-@xEEEq3L3dvX%B1)tTM}Xw8-w zzw{u+5UO&46J0B+<;Zb!F@?Pkb}?_>Abzyllb}{}h1e&B7+16L2)Y{a<%mPAws>%G zr(fI9QqC8}4YMEwvtO*GH`x+nEa$)CQQ_gU7VzO{Jn@)nvBbOys?4FcT5QB@W@v%bk>RQZu9s|* zp5HkS^X1%jQQfM(ft2m)?U4f?C+C0?dhi{7Y3O4v2Ok|G;T{J;|+tCb{c<49!|ou4iF~09h29*=z_Q3|nD@%(j_c-xDEIRo!@i3^)|O zQfjb;JTNUK=%Y8XY(RuRVAk)XytCEnI zeB&fZd0&Bx_y~}CYF@EO`J!BW&t}VKbz3mUhO#5@(mOd3pN*mI*B|!vH%1OQH{aj@ zuwv9CH*DEJLdR=n5XQO09;KE|h_xAo<*HR)hr`V>fM~xcYM}Mk*yo!fSPZd?2N-br5W-c7*PALjZhQ$siG38d#!DHQqW0U(BkOVaVX~IT5MJRrU+3tfCp3^!T{j-k zA2p!SD_AsH)w*roCat-7&*avan_fqG_D>({IhoAE<)S{(@PI@}lFTDLd@8XChT}*L z7TXg#h$q`FCi&3e7=EXj`(2gpJX>SRVaL0vZxNKq@091CAp^AvZ?D`N)?+`^&m3)( z7?|nS6*j^nq8Nzc*GCIfUWm&n6;fe{wRUr6+piNoAS(SC5HuyKso-&^e=sV-=?dMMr%Z}h)@{mfAzStCr=hx2456Z@0n zQbu8!kE_*s=I4R2c&-mRI_Hk{#={>U9@#U;jvFv-ELC>WGu4ez9=J1|(T1hA>fgGn zAw?1RRi>qx7w&_|_w?4@tMF|~gy@NR>H%m|(da_>4%qQ5Qr;Pacp*ByS!tKGhDZI! zJ*g)9pdJ6-EqLo{i)-1}G z#$*~N=d@eitXT}O7W4OqZ)r^r7A8W%`o?_vzCfORejtK;rc!%wYXyX5)5uda7xG^JIZb8<4U>INr~XdARFJ1RRY5$995Msq}9P{wbw(VYSm$eD?lc#k7(iWq`iPct(wwdS8FvQ zjdK420nV}L2%>O{`EZ%O*Z4b9QeG~P9({hjTxT=s6aM}Kp+Wyne{c%Txf^1JFR(#M z+C|4Mk@0L0-5n^kd02*M{N7UEku={#>9gZ9HNPZJ3)gcy(Z9gbh{=1ezuKs+H)p^g zXHrYY=|F)pX_r9-9V1r+nQzAv^P2DlKk9}QTcvbLO*GUxS#E|KsE`iJH}7^d^*#RK zqW4`x|0;e6wQF+|p_8%(kx*|_jtIsPCcFBkigc%%G(vVmHO!T^%_WgCAB)#|W=O73dr6+U{eO;S-wdK;RW9q^c zZ32@EWKc_T2nnH-3pwC8oj4`4UV?}Y7v7T9X*T%KhL3dLoK@bUr*&y(qCEjsfd)E& zzZd=aVOo(Q#Hiq2RA`NsI=p%R()+n$JS)< z>>K*UNvp-luT3k0v~1Z66008<;fgg6m8`K*P{elz zsB;nWzYcK;?NaN%M-=Sh^3r_f-Uv%Nf3!l!#)xyKWwTlHF08$IRe@_ckT1Tv$S2>p zkQ`Y{NKW(`f6Q4}fEP~mjhZG61@uGowpyMXJw?^*E!N&KF%^#M6Vg@?uB7G>-hqct zhesP9^-Erw(^-5N4WmUx%eiYmU+uayaaggORxLA_^BsVZ zYUP0+X?`8Qb1lO7osaHjLzu8?wFN#^fjg)zwQC*Aqh;#Gl-LK>pu^?Z%*VNDL*h}K zO&EmygPFYOZbEUqjXEqk5PC)VU9=9f(TWJyyBsTP=S9bY!|&MDM04ecY78bUGy(x<*bzU2QG= zsC2CGK??+Gt(3}?JXmSIwp&_mz7^Ood}Vvw;C&m>+IBPvq^#uNaJCf+mWuJ2*bO5; zL!r=SDvT8Plhy3R&d{IpPu7BeKEP?KmdW|hmcl9(oU1Rj#rcLvz4ogo9dp>gmoC~j zF@&{&3+mI%aU3zqweK?roZB+pi51wG+=*=L*_6g;_HVVb#@d$2^v)<3ZfiHq$k!{bdg2sT)yvRamTv4(-h6zAtewn5 zAi>kUh*;G1AYaH+q`5G8&#taC*?hbXMBH}}ae;Ui;q&jIRcJ?ZIi>cqGlSn#L`8&o`tNCOmY3KlYW66)xV+$1?}_3k zkJ4eX5@0Tqa`tm23LA*dO$^Hw9|;`Xt|HI>d?6&<$N;A~y-1VTG2}yaDzisKcTRRC ztM0NsuEe@U6&Qxb_aT!PV;~*-TeVtASI0=~Ia8Zz3>?>{YoittPLompQebV?ZJ3~8K?+arg3`U0r@UQtkqSp6QvD1{K?v-^Xe?u*meA1DmJ95i`Hty%2h?mMbk-2C<^~bi%{0u*uTy@bH5sAkqRgCgJoG)HwFxaV`FF!Da zEAS)VxNV6xQeMm9y&gRtUTC(|M^8B37QSd);#^)U@ZreCLgp-3Uma z1w#Vdg~qXH*m-5|K7Im?W42DrXjO35_5Hd;-STO~`#sN9g!zkoB(--Qm3FL?CIroml?vkO1&$MWT{!`#wW?TZZRooDM3+?!PnYmD;(Td>tq6Y~|Eoroe-9I>0nD2l@ox=DnV6cAEIwh>FOuw*XG{_u28qk{{IuKvs#n@7O=A+PT(quHqi_w}>C1%HQ zv&+`aL65~UraQn_v`%|sNyZvfh@G;j3$jua*`afs3-|8`4b(z7Q`%Z3ZR(G7RXfsl zpxaOd$_`_5HC5VQplRU5#wdxwe>`4W4BjXh9qqZ2fZO;vT(lyz_9oB(745^BQoAP? z4lVRp2-k<`EF#NaZ{7ZxNoqBvHi?Rn4`va#QC zg-iE#=L%iXmZ7aexV?Q7COy+lU;V9$pQMi>&umGy1lH~RaH=~=sYQN=vRF9baw+J> z9807NZuV|*W-Xr<(oo#wqvsWSIX(sVgl|eB-(ybmr;z5(%a(D5$5rjzD;qdoOr7|~ zZ(|dQDz{!l(W;VJx?K~X4=KRfzhf^-rQwkEl|*pA_`?TIn#+aw(~di)e$1*)%UBgQ>#ty zP!QKZ)-2S>$wNHT9G0TsPSp=5TO}I1A5x6gl|{jjr4taSpdTH5xQ#tM{LslpCl)4* zAE%kj#Ov~g$Y+0M{6%^(8;c~a9MUBXaTt%{V}r-*HuDELOJOY?e`@4+Tv=(CP9As4 zu?@)Ok+Vp14&_v7mQE*F;Y_LXQ;-7Em+bxI8am^OIQZ6ge%Zf3sD*2nz#pZSe^Z2> zX1Ix}_RFf3v83sv_ST^H&NL2qWlCyIQbcv;y^X0F<=zFMmk2qc)s06vIeAQ`wH_Zh zr?6>D-zxhl60%U%)-*t@`gzxRoH?1$#%cTqd7p-K0=|uUiT#?B)Ru9fLV}xvn}e6s zM*2v;qV0f9Pie;z8|%eoA|pJ30V-Gheb)5_)7!OQ<2&A0!kx;&b%;r{!D7C#yqjC#)Kv@>W*Ay^>_ z_n3!MGmj)9Kh!Bb@hk@trnh`B_%lK6q%&BYmCkb32WN+pa^0zk~vQhY$m%`GC3xD*^c2Roq98NRW z6CITKTqd-UKzenKrod~2bcz?Ub8y|dS%0)<(qm^W(`9sg!@NHm6tFKxw+s;_@ z-r=TO=p?UxcW4n?Rdz~Xzb$e4LLO;ws6YHDsKl(Q9=-(Sa#}AawNW~u#M1};k@?uj#GOs)pfz{R%tJ|PHtHQu zoM3l|=;7LObN5h8XF;$T(mU+A4Vgh%+c1DO7S$1N-E5|?QuTrHjqWTTlCnD3nMClH zBTuaG`Pd4YoNLh;9m_||*pi&3*Vgx3;Ga;2G)co(d_0T# zPF*2~t|fJaSr<;@wyew+v+zdq`i(3;5z%r7v+uIN1_jOM%M|slxSKzQTGkdAW^eBE`QvnW~`dJo>QcLvLiWq%yR|_-FRT`W-Q${cV5i zJ&OXD6RH1%T6(si3+RET6Y{8iK54@LgTT^MRNuTWEr4|}!n9dZUi9$e`VpRs&nB*P z!-pG4LH2L^l|Sa!%-~pw~(^}+q6Bgd!Z4SbEuSLB=Q>&418CAlT;p|1o?tHIko-Z{C+P@jWjsm0& z`In~+qpZCHgT3|bE!FVE=ULy*BR`dRrQ+aRO5ozyE&F3O)M3xB)dNA8j7oag(4_`dO+91x-%C-?(uI-)}P`lCP?#pr)O96 zXu^DQB5nCYJ?$q7z_t@;Sf2ZmDWSXuTV*}c_fJEN?-sJ*{LgVqBx^W&ad>a5^Kh?y zzl*5Vp!^p;vqh@G_Flce)IWa?m_`X;eQfA+oLl#9qPK?vNU2fG_X(8%9lITTB~Ob0 zz5RdwZ2C1ki)EP8c+x?bvSiM&vgUMlH1V^-g0>^u+uAl69RCJ@f=u*&uhiNzy$EY7 zGZ=DDM59pb&i)A4V?8n=Tz-=FuL`$O6~NEIOxe|*{96$J5*ryD6sXz9OZb0L1ZgVv z8c=PyN)rBCQ2f%w5FJ!d_^C{H|JfwJgYz?3r99_V^ZQS@Qw#iOx&Hx~|1ft-;J>5p zzi9n!iqjYWQOWCreate WorkspaceCodeEdit DockerfileRestart Workspace - - + .d2-1840016246 .fill-N1{fill:#CDD6F4;} + .d2-1840016246 .fill-N2{fill:#BAC2DE;} + .d2-1840016246 .fill-N3{fill:#A6ADC8;} + .d2-1840016246 .fill-N4{fill:#585B70;} + .d2-1840016246 .fill-N5{fill:#45475A;} + .d2-1840016246 .fill-N6{fill:#313244;} + .d2-1840016246 .fill-N7{fill:#1E1E2E;} + .d2-1840016246 .fill-B1{fill:#CBA6f7;} + .d2-1840016246 .fill-B2{fill:#CBA6f7;} + .d2-1840016246 .fill-B3{fill:#6C7086;} + .d2-1840016246 .fill-B4{fill:#585B70;} + .d2-1840016246 .fill-B5{fill:#45475A;} + .d2-1840016246 .fill-B6{fill:#313244;} + .d2-1840016246 .fill-AA2{fill:#f38BA8;} + .d2-1840016246 .fill-AA4{fill:#45475A;} + .d2-1840016246 .fill-AA5{fill:#313244;} + .d2-1840016246 .fill-AB4{fill:#45475A;} + .d2-1840016246 .fill-AB5{fill:#313244;} + .d2-1840016246 .stroke-N1{stroke:#CDD6F4;} + .d2-1840016246 .stroke-N2{stroke:#BAC2DE;} + .d2-1840016246 .stroke-N3{stroke:#A6ADC8;} + .d2-1840016246 .stroke-N4{stroke:#585B70;} + .d2-1840016246 .stroke-N5{stroke:#45475A;} + .d2-1840016246 .stroke-N6{stroke:#313244;} + .d2-1840016246 .stroke-N7{stroke:#1E1E2E;} + .d2-1840016246 .stroke-B1{stroke:#CBA6f7;} + .d2-1840016246 .stroke-B2{stroke:#CBA6f7;} + .d2-1840016246 .stroke-B3{stroke:#6C7086;} + .d2-1840016246 .stroke-B4{stroke:#585B70;} + .d2-1840016246 .stroke-B5{stroke:#45475A;} + .d2-1840016246 .stroke-B6{stroke:#313244;} + .d2-1840016246 .stroke-AA2{stroke:#f38BA8;} + .d2-1840016246 .stroke-AA4{stroke:#45475A;} + .d2-1840016246 .stroke-AA5{stroke:#313244;} + .d2-1840016246 .stroke-AB4{stroke:#45475A;} + .d2-1840016246 .stroke-AB5{stroke:#313244;} + .d2-1840016246 .background-color-N1{background-color:#CDD6F4;} + .d2-1840016246 .background-color-N2{background-color:#BAC2DE;} + .d2-1840016246 .background-color-N3{background-color:#A6ADC8;} + .d2-1840016246 .background-color-N4{background-color:#585B70;} + .d2-1840016246 .background-color-N5{background-color:#45475A;} + .d2-1840016246 .background-color-N6{background-color:#313244;} + .d2-1840016246 .background-color-N7{background-color:#1E1E2E;} + .d2-1840016246 .background-color-B1{background-color:#CBA6f7;} + .d2-1840016246 .background-color-B2{background-color:#CBA6f7;} + .d2-1840016246 .background-color-B3{background-color:#6C7086;} + .d2-1840016246 .background-color-B4{background-color:#585B70;} + .d2-1840016246 .background-color-B5{background-color:#45475A;} + .d2-1840016246 .background-color-B6{background-color:#313244;} + .d2-1840016246 .background-color-AA2{background-color:#f38BA8;} + .d2-1840016246 .background-color-AA4{background-color:#45475A;} + .d2-1840016246 .background-color-AA5{background-color:#313244;} + .d2-1840016246 .background-color-AB4{background-color:#45475A;} + .d2-1840016246 .background-color-AB5{background-color:#313244;} + .d2-1840016246 .color-N1{color:#CDD6F4;} + .d2-1840016246 .color-N2{color:#BAC2DE;} + .d2-1840016246 .color-N3{color:#A6ADC8;} + .d2-1840016246 .color-N4{color:#585B70;} + .d2-1840016246 .color-N5{color:#45475A;} + .d2-1840016246 .color-N6{color:#313244;} + .d2-1840016246 .color-N7{color:#1E1E2E;} + .d2-1840016246 .color-B1{color:#CBA6f7;} + .d2-1840016246 .color-B2{color:#CBA6f7;} + .d2-1840016246 .color-B3{color:#6C7086;} + .d2-1840016246 .color-B4{color:#585B70;} + .d2-1840016246 .color-B5{color:#45475A;} + .d2-1840016246 .color-B6{color:#313244;} + .d2-1840016246 .color-AA2{color:#f38BA8;} + .d2-1840016246 .color-AA4{color:#45475A;} + .d2-1840016246 .color-AA5{color:#313244;} + .d2-1840016246 .color-AB4{color:#45475A;} + .d2-1840016246 .color-AB5{color:#313244;}.appendix text.text{fill:#CDD6F4}.md{--color-fg-default:#CDD6F4;--color-fg-muted:#BAC2DE;--color-fg-subtle:#A6ADC8;--color-canvas-default:#1E1E2E;--color-canvas-subtle:#313244;--color-border-default:#CBA6f7;--color-border-muted:#CBA6f7;--color-neutral-muted:#313244;--color-accent-fg:#CBA6f7;--color-accent-emphasis:#CBA6f7;--color-attention-subtle:#BAC2DE;--color-danger-fg:red;}.sketch-overlay-B1{fill:url(#streaks-normal);mix-blend-mode:color-burn}.sketch-overlay-B2{fill:url(#streaks-normal);mix-blend-mode:color-burn}.sketch-overlay-B3{fill:url(#streaks-dark);mix-blend-mode:overlay}.sketch-overlay-B4{fill:url(#streaks-dark);mix-blend-mode:overlay}.sketch-overlay-B5{fill:url(#streaks-darker);mix-blend-mode:lighten}.sketch-overlay-B6{fill:url(#streaks-darker);mix-blend-mode:lighten}.sketch-overlay-AA2{fill:url(#streaks-normal);mix-blend-mode:color-burn}.sketch-overlay-AA4{fill:url(#streaks-darker);mix-blend-mode:lighten}.sketch-overlay-AA5{fill:url(#streaks-darker);mix-blend-mode:lighten}.sketch-overlay-AB4{fill:url(#streaks-darker);mix-blend-mode:lighten}.sketch-overlay-AB5{fill:url(#streaks-darker);mix-blend-mode:lighten}.sketch-overlay-N1{fill:url(#streaks-normal);mix-blend-mode:color-burn}.sketch-overlay-N2{fill:url(#streaks-normal);mix-blend-mode:color-burn}.sketch-overlay-N3{fill:url(#streaks-normal);mix-blend-mode:color-burn}.sketch-overlay-N4{fill:url(#streaks-dark);mix-blend-mode:overlay}.sketch-overlay-N5{fill:url(#streaks-darker);mix-blend-mode:lighten}.sketch-overlay-N6{fill:url(#streaks-darker);mix-blend-mode:lighten}.sketch-overlay-N7{fill:url(#streaks-darker);mix-blend-mode:lighten}.light-code{display: none}.dark-code{display: block}]]>Create WorkspaceCodeEdit DockerfileRestart Workspace + + + + + diff --git a/scripts/diagram-light.png b/scripts/diagram-light.png new file mode 100644 index 0000000000000000000000000000000000000000..3018e3955aa1247c3e30df08a6d30428d38337d0 GIT binary patch literal 56922 zcmeFZWmuGL*ES3YN{E063Q`iH(k%@Jpp;0rNHZ`<4P6FGmvnayNOxFt$B+X^cf-)U z$9P?M-{13Y&vU=q_Wb;=AL1}`a-PSr*0I*w*S^*o-xmtfg!q*B7#J9Y&t#sw#K5>x zih+S?k9!%slV*Oo2>i!ttnti5ULNBCc#VsJg-L{g1zufx!VNAY{{0$#@m=r_gOi<~ zogYJ(Dl8xTf@$|s`VmG!7u6C51|7z;Cz2{odaKAQPAaNL&0CW(j3HQf*|@Oh#JBUS zBUSTP9+Z5F~W9}o((R(QL1_A|KYOw8_CIQ?WM#w{^NeUFmS98|9RV9 zUK}|0%#OL)us{6gizOuE`2Jr+DT|3eS}#W+dFMZ0{G!K%@E=kBw{n{LN0k5Zr2lg& z|3?x1W8nXzc>eNw|M<%P(Fy+(cK%24{0`jzMDYJMilIOF{|OkvyV_A%e?hEtWLSq!-K`!p8C(T$mQD$aEW_@h zTI@Z!Bgu;^uitC@Sc?6+kL>^Qj`<|X$(_$7FuarTUi4hkYWj5ZZ#c~T)axxQ(Xt!N#?~9DDD6OG4ypXlKdA`E&~U9A8_HJ#KQf-0j}}jDgSdtANr4| zmjnI?>Tm0nXaQLFqsUh|B5z`_lxvdLW3c|-zR`Pdy8s!U@Fy1n`g_%MWne|hE1hPz zbi@tdnuE?^tc&zO<|BB+KAY$6f0f~#o5{)4+t@s!*pKMIQm~&H;r|WpLrB2wc<4Lj zu3y~b?^Syb-~-{n1=8+#Nt}RdW&{igE>aOXYVbyXN>}uM%@E$}0FAVecI*=4dV{49 zGlt&3*s2uRVAm&Jo@W;~x!BiB7OYbRaq*D69=E=i3tGH`eOjz>uZ8s_yj@%}7r3kv(r+3!DI1ZKLPw}5_$yaHg4 zF5rMBd7ijkxb8+u?*I<71+qO|gLEu%Y;@%CYQVOzu<$5xnLCL&^Yb&ih?khyugO3h z$-A)zVz@3k1Tb}1Uff5*4ZK0%Z+hGCsHs^Bg7)&>Mg$sp)-3JwS3^`1R*M#C|AZL2 zEpRcZ_48l=Ml?XI_1#MsxQGjMn&2~kKMn!z3MPVRkKb{2l>HU8tG2OoE1A`XH`~8L zZyyuz6S~_0{uo$AAY28h`oaFJ#|4MGK6y0Q2C5*}% z=etKMj)UpGhN>@KOa%v!;Gi{ui6wr1lIL@ky1>sbJ1;={^o0`n5_9K+ZVaras2$LW~(=A)5wQt}xcU zN8{c1kEbHFH~RcJ>gHntW*%m#M|8VCq~b9m|8oEjfWSX;WOd+re?bF$ZK)WdzwzsP zRj?2L8016o!h7ep8A_bpOUB28Vjuo8*jubO&=|trc?7oa#{5E4Gb81@LVq8vr^Tnv z=PYHTBh|&W>-3X2auOGh!N5vQ?xY9cwgJ5rSD#Nr7sAnC01AqT2pOt!5}@ehY~t0h zsz4$rMQz`wCcYN^%^5RlPWf}xBVtVTS1EpK9Yc*ZTN@dP@umhp}y+x!r3(E?r;b+|j z`6RKr-In1ipA!epI2z!36))zVwTp$AVc>Yseq{Gm=M8f@HcXeHl!$ZREy0a#2p`}?nQLBl>vs9P8 zaglv*?#j8g2r?n#1LxfAY>OAeuXp748c!>(JfF)-^6afc1@%@o-{tc=elLFh>}I@Y z_5K8k6RuHI{L8^H=p&I><*)OklLaGrw`ue4{qKs@xnDoRtw`^EhGEM})=)cH`dv7z z^cP+{r8A9P9IW*UO**&hT75lfQh|iHg7nOOU(}=L?J2TdN#ksvT5&rsj-_p02RTtc zLUW<_78P$k9J3l7HI&Nd2%Sm2kDND3#gh&*#(4~8oGoj&q?YRqDxCOxXYX??jqk9d zEbSKWyyJBlNFeN{_L#*xT1-YrnRBl5JcsXX`z!K^ICf4#2h(N}H%qJ5N89;M1`LXPuE)Ya>Ky+3E;^qP>*pO(%C^@$r>8n3{Jd2sfk=U)=%Gj9L_fcC22&*{wDg#*u+ulZ(}KaSJD zA3Y`#dOqpKGuFRx=Oa}zSK9iv2x?G$a`_#RIMfMRA}`Q;cDzBVE&`oV_DL#K*H0Lo zhCwBddmir(dA~bLuG=5gr52j~A*ztekNa41>@@9)Bb*mi>H0(dXnsu4Wih2$M1EYv z{dn(~uv@7|ul=#4uxf*Mz#smkR1vtacn@k})bdTPk@eG3;Q{OAv%S6dTZRKghToNK z_Mh9KE!o=%r~VkKggh;USXfZ07TU&WGm_iedr&K5Py@{ovh!ugm;tpHBC|2$miuPS^&%Q)mqbg0>-r1on z&q%I|41Ejp^T4F|6EMxPKvd($nkOUg=4^60yLxnXyxtQe%&QYv8z@>rf!@AEJ{CeN z!Fm7Z(RRDFNZi9RSW+$-ll9M0A^?vsqH_8?!6?Ljy6&nV&f2iLDl5*KhU@XnR7#>N zYgTpgT9e>j=lx~vabZM=av%F5&f3Rjv4R!r%I}5_n_rJtQhF12*j^W>g9ZA&g~`eK zNL?6##3&lkap5>SyYnl5Y!Q%pkiGHxbg#yu??=&tduC3#fpvdYDI0l~4Wgs6s_KAh z6Wk;Fp8syRNksF%tMl&?a zy=gVMd3k#~-3=Ba#dAX2i=<_*f42F+MERH3tGf4HRvfGzhF4_pO%5WTvszk zN){H!G?|W?8Ay3eyK1G7tEK(dVo$~o$UQSmx?{44#13Tq$xqeu)C&_qHSafRsblJpY}>i$1?bBi%qvw2USF=*iyXC^DrjqKv#_OEckjwm zgP)$goUE96c(hx>!MoR4p{k`;06*Fr{PmGG-?)LjUlbMG8?JcP-aon3Oq(H2?fL|1 zSLb3?uvZO(YZLG^9E|oCWM{wl>F&ZOG8Fl?ye4zPX@{*d0hQo!Y8=DATJP-wEsEJc zKAoUhh$dYK_qkE(9)|nv&!N%+geWqIDQv9j;Iqt+6NYL66t}{Vj=Q=BU-#Kn z|7sEz`8j{2FfMC4fUJt&bsE)iT$`ixSpHHi8&DHP89Zk*5b8Nl zhZ^s*=%2?QUDy1&M}*>DT!?qnOdCcH#R<#0B37#$P*~@i--F`jv8K?l!h~n(Z!1$LZ80 z2$3z$#}$Ot6udnE@a;!F3D1S_Qh{%PnTR%gIS}VId0{V5fwYD1kK0wi_B;MUC|kHX zyHRXXN^`o~*;M%V z-gK)4iun&&&FHBeXB#R6x$5F%B(EuSu(8)XonLIOKVD5~ZVt9^T3#tMUGlmV9Mt}6 zU%POyCuPD)&-0Wm!QQXp#e`0lz{Hym3SvtK)lu5QkB<@B8@=z+L~i~HkL!-ocWJZp zg;lh;E*d$#ko4M~{{zP(&BZltUMXXCU2GJ5v3YlV~g4qIx@X~a-b#Lb*pjwfYMQNZgW z?c<1RqwWwvBx{&r)%85I#-KG3S*>U9g8Bp|JD{a8uJ@PqL|vDWj^aJt4JV_EdrtVH z)(Y<0j!3?jWD#WY(0UJ&yTvjS94C5rPA1RKyw>YH z<gXmb0q3GnV%Lclt1zivDU$Mtp=5!2;h;};UZ7n*8JGfhUZNI zE#`GvHOu1GCWp95+jdpko%n2?(kyvT_3u6jy=IB)TLs_G9?IA)u=k+xZLP<6(zT%k z&xWG1jJdp_AvUuyE!P0w=-WoKRcf^n@XrB;Mi3^!;wGijZtftyb635lP~BrYT~~%? zWO;qv9#osf5E8%_ff&X&RQ=VWb}I&-p{9>3vrj|W0Iqz;@IgQsnF6w*jI69olSN)l z{%N|=^ZWRQCQ_M`PNK4OMR3-R2pbNjZ&Ta16eW6$Kib%>I74bJ2I>W8 zC5sw1eLU(9Q5JQtd^%#?;L)&N6~ohj`1H zkvHdSw*CXXL@2SLOXZU#N&($sr`a&1L+4yMb#rg>yP&NTl6P?%D`(}IQ-XF?*KrNytS_K2N!`0w?}7CIl* z@Ala_e07F2*R1;p^TJkV2(1vO3_g`Nbt(DHc7>sPoz*{AdEXBf+<}+&Y1H(c6=Fb* z)!H;rFHJI}v`pZU+A+pbLB|{0pZMCT2H71gjWpG)jGT?<{EYji{a`p>oBjlmw*dYz zUoF*NIC*3m8Ni>r(N`Bg@B@!1PudZ6aEa(Q2vEVz*Qt8>*yq|1M*}P5R@dpH!oe;? zM1&e+q zCQ5R&=^75cl~6jM)@#nQ41Si`nvgFN2*1raYT}#?7@U^m~rM)$0{UD8?xI%(&=&M+0>@}@BYP2ac@Pjgq)o&l-_pFU&N6_ z$b}7h!|?1UY~P8@+|bSmHgdT=QXh773w1tpBcJEMK-Snn5zufB>{wA$QP8SAsG>o? zgH895L-$#r2X-A`wzp5-lXmZSw6c+V?k{OObOY!!U(zL{xh|TTM!QkF;~gaI;Z$*c)_{rdD-3I8IhgUo$AfhJT255O!tTozCc2u$=Y*881lnxmS*TZ|w91+aKC6D<(lUp; zo)5}YF=(@;S4U|EY!d0~Q+4kcTT)uu#UmjVgO?nphU>2Qz4j(W(3=Rso%E5*W zy!4{Ei`J`Vg2i?FC!M=@Ty`^ltmsGc5Pk1Gmy=u0NaBB_nr$mM0@enGTgTd(Hs6<8TlPVc((I zFg!72*gJMd1vr+Jmg!d{>naIl`5I=#%|9C?$+3%6AyqPvf1&`5w6tQJ6e5Ax73Vw9 zhmjUhmz2z&Sz7%V&1Y?u8b5dR>}s{~EGNm6LBx6@&x0$iis_qf41aQ{X%D^?XAQs4 zx!%w;+dk@M_}Lo1jcg!$`Q0wFvAVGJM$52UEAMI3XMQC4Fm?-c#C` ztD48R`ktrMX^V)EFuzO{5p9+Ce4*nz;ooS8TPIoBra~m`HN+q9a(ae7V`r@2U9n+-I*J26kl! z8Rs9h`>Vt6ZRx-}ko-UisN(k9urWGvn$tbM+q?j3Czki3lnXrXfGj-|AMugO)~hL? zIuGi{oM<03;bd3c{Hv`&FTJFDU$6c+udp6obG}<0{DDBMkj2cp{HK>xq3GT$!>Bu| zDXLko!axK(VRd`Sx-R?MH-$uKQvgNQZEHQAEGJrcLdsg!+x@Jh&H-AY+mstORz}ZP zMw&}^x`fNk$LuX{XoFso&@$7GMBVKj$PT1LcviSfK@!HuGU*$wKr&Jwm6;c@MlY?K zFQ;xezJNH%=PPs!RkE~xvxZkxW;H}!yU=b?@0)+$sa!o(fQh-zca&MXs7l* zet!1mMsx!FoTur@a?M7CaZ!_)TM~V!PgWQwx9;zv%M$`OVMvfq60APIOi6JpnB}?b zID=lVBG!d*rjQNuaujk}dzFiW)=Ue7QAAi^1HmocwVJLyc() zTZBWTlu=~$Q;V8v7jsqBuGcw6s;P!%ik^V((q37Mxec2O9Lv=`mbQu$+NxVcf zu4qWREp~JEEzAzpUBA*5x0_{nWvKJikHvamhBR^e;nmwN0nnN~>%;{;cZXaB3Xl2) zLDcmrTK7#Cq+QL1&5N{S-AVhI2%)7Pci`-&0Ng@Nv*JZm1a!{T6&sWjJ)bSauhb1y zx!yfW(yDr(-yB43-_Z+UjrISoh_m+7Kguj>a&04~#&rXgS9m0w*X1c<6yjH%e)xzS zE>%@7`H*1L27`AD#OU$y<$^NJa!90azNt~oXtk%jI?7bqq>k_Bn|@U(q_GtdMTr5u zR8S649&P@rh~X~0?5K}&p8`q_Z; znGWj1Cs5i2*~2pSc*(scMW;TVWW%*GRbwWBbas5H-xUBrF{|KXN|0_17KgE zTUsVwp)Sl8mZ$&=!?G8}1jeN;MJ6fvRKn|Euezf z2VT```knE=?jMPAEeYzv4$Eko9urZd_ol5|I<3bxAakXgMSq#wAvm@6n|w(pdCyE;9WY2qO{SU+ zhfO!EMg$D3bBb8Mr&pvTCumHcoOJRRs5R4sT8<$#N{Z;qKNguVQ1m}WK#h!CTFuga z1W_X6*vb%Y25{FsxdoP$#r6j)Zt#d5q&Vk{_usaJ3CcbRwX?=^H}_7Ty!^iRrMQ^h z#c1`?1NLh}AJ1n=`p|?zkB4_`tQNZ1(TS$Kc_6CF+S)ZU%Mm;EX6LxOfHq&c{%;+8 zx?G&>SbJCqi%2S0hdIA6(-+So8Z(`%5^Np$G)GnYHUb7EDEis%q-nbw2A=kbuCTmb z>#%5!(N9pVlQEt=j}19Br{O{)jiEwVoPjUpS61n6v3TY8$@jp*U# zqLc_!$>~ReKyEUfK97l8_OC4cWHOd#cJon3t0f$_$!b@wyy=dHb;lQYo@{=9xmLSEOBv`nZX>-I8CRgBpa4VG zHF4A~=1N)7xJ|(b#a5@P7m)nx?yW=at93)`)eDK*16J?$Q~~=K91xei)IZs;tQe#U zgTZun=J8t@Xloy%(O=*#u9pC3&Mj3t7P9S*=5@Dnw}#)cVY6mqZ=gTiVRVyJ#IttlBcQ%%-=I9EdV@ive(pF9)BUb2uggBkHWBQ) zE2*edlV7TH?&eYqLmEpH-uhyt1L+_NylDSp#424;c3d#*1O&_ZbbGNyH+H^J+m9<8A z)enz{xXTtYFUS9`74QW1##)PojT4=NV%|%IuF(bwg}zOD(CUCo2Vmt!y>>dOf{-j9 zAH-VN99|D+1-X5%5+4}uA(rz#`8Hi>)WQEXATj&#{a0&4WX4Qig=1+$jOU z)s|uYw@$mX`X_I}b5zQH8NeMFJ0rOCDf0@)+anN&<-9?b`L4Es?kk-)OSKz-@!tpF zr>1PKd<-hQwB#u^+GDx+g=ZJB>lioQAu8*Pd<1jTJ5$C);ghw(;@pzTr&LfU5fB zxY;$(4+4+U*ReA{08E$E73$(9D6+ZHyT+`flfSo*Nt`d9YeUcF^stq5dEJ2~{R)ni zxJjNVw7JWKAV#K;)zC$AB+<5ewS3vw$mQ-b%_JzSa_zHd?o>sbjtlMYWEzZI?R-4+)Y+v@0n{^wC0+qg3(7OwZrx+dXeQkl|+c zo^M8LKsE|kS*mW`1Fa945)jktm+jP^P}_U^^$C6z3fnMzgCcrgLO*h>1wrxkiLF%& zWDRyBhGU9j!MXVKBtC{LBb4I=Mzms|m6n4V)ZpNlP+7fEI+V@p7`%koMgm?9i3&pK zI>M`-I4TD($~sltc&rV$T#C+MQT~^8F3^eW7320O$hoAnfV1=V{B-omxYHV-*4}&b11CwyI-RUXQkc__4%ZbzwmRGZ-? z=C+|~Rofby$9kLs7|Sst@$L6_AD_+6X=-GRL@BL+h=}%)N_A zfTe`+ddrDVG=7atXM>kfgG2+=b5?0Y2Npva1{%bfZ)A8S-2t)HTL=@i?PZ$iUQ~aw z9e0^zMa-PSk{JvEX;mG7xL z=4^Hwb;>cPb{FU7)wc~?>fJhe+ECbpMlMaM$#rVR9nm(`Tk|!s^1j+m4z#_dkP0+e z8-zkSz6m!quQ9ANbl(|tg(p6{Z7n%g*KrV`t}`%QmyUFGOLAF0j1@SqQHxp_{#So>`6EuxF3b(R|!UpcB%0y?RB#p}XeivFkcDoW#+U(m*RM)$qT)!Uc9fH-i(&daCsJV?t(bt|}q98qC1h{3inNApuvg zFuc9+-lx2{&T0G2>XFx_Usn3$&dxvMMlEafe>~YQYZ4_G;}y#&DO-n+uRLhjklD~q zL#aoM5~YU~Blu_bGI0>Kh2?A8)AD<_d{9dcp3R1u!pG|+A8BYOD={8g9o1CN_BHSc zJsz?0JcfJrpArQP&X6K>)<@i_TsvL`R<30FoHxu{S1kA1C^|i={JZoW_)V*NS31G- z7nuWU0lG}$P7bxRv&aG3_Cv}l)SRzXZ0HHS>Zx}ym2BRA!?7khpJGEPewHLpjtoBT z|0T2#-ybW|E?j-CS@N9Mc7VYFH3(hiayv+J_^L;1E!iIqURV+nsJ;cIHnK z++m@1TeJzRrsCUHINj=zBm~N|c3hRFO-S!H^%%*ngV=BTHm}So2U$Oysj7N7zR;k5 z@YLT8H7+3KIC1Ee)d>m=c?8`kJ=Qr6Q>PdLx4-3CI6?2(< znOU4IColW{g;tU7zApO6+`l8Yi&Dxbg!(mSHcJJl(neiiAQfxOyMSJ` zElVpnoUh64m&Ss-W}UV1q_(P9;nvcJE&J!%!CtPLV+93OoJyt%TVlvvEIa>BeRomW z(0talgSfK?tjqDG5eF;9#<2p9-X5n8RYJ2c6%_P#c4cLht*vbszezqdfSE&N$~Vuy zJOM2-kQw>vJjFsi8`sb59Q`L7_5ESA9{DxWPThKO5dTgA`6Vjk^rq*{^W@@UvIqC> zz1WzD%aab~*E6D5Ys4)S?|%b z({NTT7b^1v)EiSe`H_`fqS-|IGm@P;$1`1>$W(Pk=3@bF6IX5h#YmU3EK`=wsKF}^ zAD|HNeAdEK4Xqc0iDF~bRvt=Qwyig+#wyCPJ3VUQN0#WUj#0|1`@Wtu=+@L!=h_R? z*Epf2DB16JW(us_$@GyG{5gR4SDG@)_NG_AAj~yVayLy4)|1;8rap!gqz3s(8}`G- zFm8*>7Lr~Z(QF(L6D%x^crO>NYuQ--11E>8R=atu>N6+^XYd)4Z z?j}x&61lCd@ZQLGqo>#SA%4T_bu5N18W1>_NoE-=wTmM;-g<30TT%(R-gd)hsL?t4! zAt^w77_xjev_)Pv5?oO)WQI;T7?siL(FM(6I#%+#3F^ZO4-M%PZ}YAdHE-A-!-p~+ zUiU$l)ws2%%7(I*s#mOc=rkZ%Dv=RXi6h({G~QFVZ@icuR^i*FyM47U(bkF9HoL}G z{E;H<-E=i`Mw!00c`g-C8TU&;RpZGT)Q@?R>zET+kHcg#y(K>33lCWIfY{;4fP*AaLRY2JLlP`G?T?3l)lEBAdZy`lQXz=6)>(=H>NOTZ~q!*O~Hye!r>A3^bfRz?JG4sxvZU67pN2Is=U$-tAxO*njxRz=3USD*t4&sz=|TyiUnv^Cb}q8H#!4_q0V^-cG) zF^mhEqCNUkCa%<&k)=ik(Ge0)!N+GJf>w>K#+b;||TT zeHKVZdpW{6$KvO>Bcp0_EgRt_w#H-o( zf4Yo6F8tGhDek<}w9~i^Df<~xZL_Kk`Vh|xGrE=GWTi|swbf0l+I6;Frfi`_w5XA{ z^i$VxnUO#Km zJquqgZR-nj=TUyI$v{kndF=}}v${bL@2WoM4!-LeeYksRdRpKBOQepLc}Bi_|E--c z-L=}`iFT7F?z>he_ZyE(#djX5;L?3jqgy?;v0W3!>`zHX!A3?jFTu+$8Fw^li0eP? z{?VrP++|XuZtfw>We|U-K&80NDqLT~Eg_()d@rU#5BPzJYI-)bWgFpd3jWNKl@yN2 zgGbcw_#Xv@+Pm17H0Zbm1&K@tn+N!%0Iu4*gS2vH17*>+!{N#dT@wWP6a$aqJBokb zLjZaY{+`Tf=VV2lL4nzU7lI|aRkzuP&( zm>L&u6oO_VBY0`_(Z99F?p&j`zY~dl0?@-}zWAlTbhkN)Qm6!> zrfFCXe~`dac*@Xo2{?|&UYA%~Uu1MU;e6Z1;VHCBLK=7ULhp>VSd#m{D?3taQV31E zA2G+YAzNHzDIM2AGUYC0XWJ=9f}1O$0fNgO^Li#We`ovJ=bF6Y)y7YJt}SK)tv8su za}QJbkGE!X`q4}Wu|L|fpMyqCO}=NA;eWbqXHzF5e^MTAx`WE$Gpd+^-<5$>oce=l z2)es!C3zT4F2d$9L?Jksp7?LPekM7$mS$s};PlVJ)-6}|CEbV4lcKsO29=LuK(688s=PAaHm)6Vl+Z)+o z5d~4@G2a!PaUKb+E6y#QRU06dX!HcIV@qFwn=94J{VQ)(!qK`py+ahKc-yTq7tMPR zD|kTW?WHAr=Iv6@${p|PY|>>#I^|w%1+@fzEQb_#*h9pDGhi0teJ$Y#nvjrDmp>Rw zUs39N`%9Uc(t_Mi!H-Q(t@dYZuJ=)a)yjN9OE(#G3D-7`4(lrD8BE4BndKd=m0XZc zL0gf`R#efQznMe?zKXi~%KPxHzHc8U1w_o@78VeQqYmIFZfe{?4}d&uoGx@Tvf!0G zcpDS0U;7vq=r|4!;XDQ!P%N^FIt8%)?wIqR!{xyO0E8zJB62$v6tz^k4PF(xN zal5=v#g4(ZkoOI%%iR?w(8r)$wiqq15L~p+AiHnI{92VMIrth*-La{$vHRo3jP4)4rh?5uk=7<#6XEs*a!<%F7808G2#? zoPgtPfjN+L?Z(36E&Ccy4^A8ea--ae9O6H^+B|l*MGYNmQn3%>rz6sn6S5;BygY3E zk95eCW-}KHdzNS{1+6>_hGQSYJvmjQJ-Y|GmHB%V{aD_^;MfKs z{UU+DXfXBHn)@F&X5YCkBPThAqp3X5)r!dK6`ACi{pN5-i(L)}6B~;TxCHAk@YN}h z-q@4W>GSXo>$LCaCiWwg^_DtD#Ur_tr$d=QdAX{eq3Aa zBX}}Zt9XT89@|$!iXh!nJhAsgPRtwY@Pm7E>*9|e_U5wivY7y19WbQBmAZ@W0|t`` zAPVw9%G&&KaoWLsMsu7M)Z_>lo6Kj_t+CEFJqif%dWJ-sHj`brnJbm&XtclO(=w1}PNNA6c2Cn_b9 z2c>bT&e?TggVvos4sb$u7rby^YWH-tuav}eaxvdt!ZwskLY%;PCpAQsX3VCS$?WRQ z;i82MA!Yzpk+MDa1gbZ~ymo_G>+@M3Q%_`jpf)5+Ce}oKNUi|%yuU&Yj|6A2#Gz-O zaPvt(e&omm1{;>FrSz1xR2WJW?VV7Qjvry;240E`cUjS?bUfBVqGOEMsb!|AtpXgG zW%0CC`&T?y$s@Xl=oCqz;2V|scEc-ZY}8oQOwrxf3ZDnX;K-ZWh%>W{ETcW_idNj6 zHiD*cr=)9AiJz`}%aO~Rd5DBN#JE1d0bRS=BO3vH{YD9`WOH`R738$XTZs2g)M(#S z0DKeZS8)8ml#6NM$u(0 zUv!>1^qHrWw4v!b!Vqe(mBi2#5WN28?|GuoFI*v9DJFG)!gb4_?-yy2~?)O^OZq;JLLf#ig&DwT}UG zQ4=gjBl+tLXc)kNv-TrDFMTgeT|;E#7SBT$uQ6zqn(KO%c3)Gv;co=cgcYT4lQ7}D z;J!f+M6IKww=JUg+p%2xlHQ!4RELEJynKSXiragmu8^_@>$R|H#Hb)*xS4)1j~1S- zQ5x+=H)RTro?tF(#&1Sg*EhTO?CCnmP#m|hV=yrb) z&44Te-@?WaO86DaBTNmPuH^wsYT{-K3JJXCeHspHeM}#0evhezjGzH8CCINLqavjw zhmFY;jASFa&jd>xqD8TmNC2pJ8K)Rz=|ONrqg$SNM&{)x66g(GN>?qA3Ej$wqF~j? z?gYte#JGOIu}ikzu%QpB!m>gJn-<*S>1M>-pNaWwH4la^JZ^wh39x{U#meVHzBz_U zTbP>Mg-^um+SwZhk&Z|zuSGl(pkdtL#w)aL>HZv5qr!I}`Z@Pl1UW@YjPBxMO9OBI zwN*8mA#jJlM9pJi`mvFHX2fL*L;B8<0o|86F33>l{M-&PmWX`(5(YbhEU=(o9W21g z9lbf{t~*a5y+sz;r=M4l_vJSO7==TF~!$( z6qj!5_HSG-DLEz}TT&*$-__01cHY!tsW-_jP?2_5!AV0DcP<0h2dwU4yT%y zfzi1+^|6fc;kaWs4euqo5PERbl(9QGL6&X;JYYpk5Fw9**QwKG#+du`US5SbASBBZ z;aKT&s9V2rPVjdO@4ad$_jm#nCm?}f$WuuX$4u%ydc+1>`On}M-o)R7Td%r4Ws~4$ z0yBLZQ}N{~@_@iZGiyAxYk?xBKX4!~<4_A?sxk60x9vevD7pIkPM<@%#$#RxFt`); zM(@|?(agU4jUFNiM*)H05i)KHDaq7}lkaN3P1gi+-IP+CjxSLd1g+nsi_Rn>XynW5 z?YeOro|Txm%4Q+ll>zdqjjN+NpKf|5131f+9Ydf#-mA|EXJk0eFe-<@A`$FGO(N?L z%M8)wd1`h-lXqU~AVXCH)45=N_tH;~ZsM&BWgrltJ&N1luqG$hty4~2U7I>pn+4%1 z!91Wmmq_&A#=vSvh`G=UdEvdhY?1J$PgARM^Vo+%BX2MWPVptz?JVC*!T$>h?q?tr ztcZf1%PXS9^_l5)>)*8oW%ij2_kuwi^G&8fAM*FUcZivQINCAM;*gLW+n}JYx5o!m zU6z<3I%I>og6^dorI!x0dsC7f&-3$JOq-%*+z;H3cKa9%XM3!*E6k4ROB zFW4Mp?go6X^l*8Z(Ym?x9~N_@9r{=H$1ijeu5c^0UqP5FnTOsI9TlUo&Yfm$M7I~8 zSZ}HcsDAFOx*N?yZ-M<5jVq^SA9REkxYE^TBA~Zh`(6#kQmC@b%MtC2$J_?PuA5@W zSmHtkuRkNO8>Yb3cywJW0^gc(7Y!TYjPXam>h5WRuRjqFz~@HtqNv*H3&};AIN(fv|#tdp46<&fw!PG-t-BQ5`4?I5QiY-iO~ou;}{oo<|oO z>mE3UUe6e2KjQ%fQ1D$sbEWd=gtq;Wq;O-yy?-A&QZ$?i6zIE{JHkx7lY&;)SK@>9 znx_t^{(_twbKK#&4ol;h@`&eZIELg;d?hx~mRlSDf$MesD<7`Bj{8c$Z!kX(^8j+w z1THKb#5n`vSK>52tlS_8z(sRvV0za1l;2OZ@XG3IT^%I#Lm>nwn{crA)f$F6gL zt!`!R-l72l{?*FF1N_P>^iJahx_Yk73++Ei0P*kptrFMW@V{_R$}fxAJJC$a=q%!d z?G=0wOmF6c*8e8bY-I>4M@wltvLTIG2>b1QpyyZfY?ewpI6RgV#$vuVKzC6%>LF;;bbW6}4T_Y@k^I0+IM{&4$x)ldzbd^8gH z9YH0Ev4?}1Wg|Vl(}AsD)ik;W4A=-bW-ueWePqSIKi>5->k0ilZ7(5ul&PTVsP9Er z)HC!E@E~*gHCeb=KDM#{-W2X$Tbw_O-Owbb^S_cXlzSOemCswEC3`t0VC`k2cHPKF zyt01{x$94*Ht1v!iJuJL&NjRPD!3vW=Zn6?{~u2AsvRsw+ZE(B&_cQ~U^B&lG-TF} zK6}Rs_SZ7P00Rf;9XgBEOi!lvQPmay*rg2(p|vrQ&4hecP7pHhOD|d;&5(^YY12VV zR-b@XyB<9jt$yf7rergE0UAbzzLHX5VO|HI!7Gex`o-Z}is@PIG8mv);dNtrocpQtK?zOGd zNoxHyDSZ|czetQ5g!WkBd7RR1H#;~@qpHe)%U;^YF9y?ljN0Hi#UGZ$TY$~Hix+(@ zjp5Dt+dDkPA2rYzTUr??HGuP@nR-z0rg2;>PQ- zvA0z{jJG)pjBWW}Juq;-r_jzLg-2HE_F%*3xG^R4w-(SK1+LBo#Gt=6U`pCMf<@h; zrRUetT$C~4FHQ3dHDPUawJzP+=4f9$Z#wj`l^Mae&IiZV8XBW{j8<#I9Z_H(wrI5u zMNPb818N;UWN-m$tw*a>5UBNp4>^0)R+wyqx|O>>Rev1xHu9MaAnzn_^`pC#PVR5E zX7M;o97Z|6EkV?4YpQYgB$1GkVSPnIz&EyJ^D;o2^Q+ty`}jbcdXnWpl16B4mUEt7 z(F9TsDUn_Uc5)f5%@Ux^t9jKif?!tocYo>8+(m!sC(};A(O_Vn+q;+kX+fYQXCXRCMH@0-YTnw_cV6O1)1ySDOLg8LkHsh<5&D2BSzly{N&1EqtM=>k-Ai-7Ot!K5zh8;2iDJY$5lmoo(}~98^u9sY zI5~44BPQ;RHr9#gdimpKA0EuRQ+Kcr9Q|N)nmzj=zSQ~xC)`&8KRlyM?dq@Bl`Vyg zi4g84*0f+pW01a~#sY(f-~CiqcrV9%xOK=#zbiXb6vk$Nwf@*kKx5oAUu5K^-Yb{w zMcc`+=U(Dj;mc$e}tn`5Ip!5mR#d~>_qWk$(n1T#N@Efmr2AkXsMM;I@P_TLW{-kg~y304Au~zg-iNH z9xTBb#)%15@G=U$f^AbtVTT1nzp_LzyRYBwwjSp*-bv^)J3DbMdaa4;=H~XY;~Ml6 zxW(NN1m1Y14Jmxo{^!m@H`~4UjTwudSnL-UHMJJLpPFLf-D$kdikI@!IouzEgEQQp zb!Y_-`@n9LR4Vis>^Ba>GuU7$}`IhuQ)kjfMQN+&YkB9 zQHWf0Hh-1PjACzzG_$}mPF&k>`FmpJsaZd1sr?xngnK9Rsq}k~^|QK}|HIZ>hDFt` z{llUHA|>4*f`l~EDGk!yAl=<9ARyf!UD7dhhe-EO0}Le%Lw7U(<$dq{9Pd7k_Zwee zEoRR8#d%%itbs{Xz=P=r(sgU)zY_cq;Q#T}4*1^3>Vko{)}#f#^(rAzTSIH?KgXvt zKPT+|dF->Nto$rQM$6(Fj}Q=GR**>;E^QNZIO4!%G2n=f|8vCt&qoyd_04nr*2_MJ zF8z8!%3X=Tv&s0p0o4h1`!tkh0)&+>PN35>H5;$My;7=?M`t92r%ZPwi7Md9ft%F8 zlZRwd_uW1I#@i9q;ML!FI|AbEHLye=6qab(1#ewfziA z88;Urnm?~!<`?9%IiAo7c%9|+WeRvX8_f)gRDB^|{PmLY<d`I-cY<0_ zILm6Ds>4#7hoNfsfi}(?rMHIb3~tj2?Y~^715N5wzdZ|3D1K#b{|)$tsNVn)LH%nZ zpIi}$NJ0PaEy1PYUzYEfv^vZSiY_UGy$fs?JXD6>*+bM(PL6R82di@r%RX|Z9iE+I zLeXW)O6|>|No=0i%Tq&3?2h}fi&}N=IF0tpR&|xXL5M~te>z`bd`LRD`lhtX6Vj@R>7~WQ}{W!b?iur_IAw)Eq9CJ|`7L9Q$~WX>G9B<-x#iDh+zf zC*wjw#w4F?Z*(2fEcJdHhzxZ%D+Aj+j;vsdO*=NY^w;buZ5jf?;~eGWy7@fK zIXYYH*K*f@SWgutJNwl3*PB~2@y>RyKRY}7yusVsf^5WfOD)dtG?@>H=WV$*=IXkv zymfOw3AH~=$)?X`3;Bhf6RiJ!ob1Wub^hA!Rb);qWGs-vtXsflbox83M7vJMI=#=X z?JWY3tYoc+Aj4F*3EHH=(ecX?Za9TD?NIji;~m^5ND+tx88~Ze=~B&MjlYO~^Vb zfV_XXy-4(YTOrwDqFgnx=+NmLLXkdqb-0jFcC@bzR;Z!xL+1_uC5%atue{+7osghB zl9CVzKyQ8m%Zxd4%Kdu(O3UGUobq$y$eUxYoZe`KmA4;=Cii_UDsY#R+*jr`Xx&k_!yB`E8`L$sc?@YF3q^x?r*u5{5SGoA z6c5dT3^6Z`WMZr~rb19tPem$`(&wya%VH!@&3SB9olQ+e_<2jb&9mB9bICNH;POi{4Z&(c{(|wQ`-IGvXF3hxwnC7>2P&WRvwWek1#m&%LZKhM-D7 z1lVoY3C6scZ#zbgx-$-_krU=l4rnRFZHPv7;jrMp#}%czE~d z0xGK=R4+{OB-xh5Wc)y;L}^Vw?2Mlu2Qbi!1BLrwq3_ZQ8v(yw(2AplMeeQLn=c~# zc2OU%r`Vn!b*?z8fij!_8b+n%kLwu)7{Gq>OqUIf8s2 zgBzt2)HL*}F&L~z;RJV~(S1}M%v-iU?WZwwPv_HE9*2`HA&0h~n+ z(_dioHF(hah)7;L>I&t+&%jGO%wwh01}t?3OGY>R_PB&4pKg_Om-3+jjq!uOy#UHf zBnF-z&|BIZs!IlbT7bH5?nnj*?$lE3 zu`>m($lBZ6ReU|Q!7*avsH3k7J%90`2|Id9TGu~^L!&V1yz^A27#)Z9zpvFt|{cpfR9tMR}-F6no zamT~$`TBgke)`-UI$@yOe&=L|6v$ApprzqycBzk0Sj2~f-}DIAS4|2hoU?CrANZ$u zoSW?8u1V^}27($F70fJ%$m3=wR_{HeRon|Dg@uJH^gCVwtLwF@#8{%a66NE~{(^d% zhPTeg`75*yry4`tYg=Z&kGsN*wky2OnAVZdVfl)ns&sbTu)DqK;*>c$NMmd)CY^nM zNMx$h!wtQ-&=Wym$aAz1M`t(xi+TE`_FH6fA>VJd3kJW-LEDnIwa35b0zK*BPb483 zP4;X3(YmiNyTW&OX$NzJDPC_*MCMCGzV*9vCHBb?^a%}v3;gc+{>pWKMR7xg$LC6u z)%-$9_K8vZLF8SN^4<-jzJlT5cdoaH7r)lVtr z#S>`VMe@3C=HT}EI-k0}$NN~}$EX#*P|Gv7qZy=19WJ(mp49;1-ta`>;ZrUOp@_JJ zorzS;;|@(e1A;AAlZWFlCxcW&Kh9WEsU++cep-Z+SmC3$QZbq*G99retJ83$CWFjR zinq_7pP5_^BNDeAt-?{SpV~on>S?Ml?0fnSVq=t39L)vqur|e*oFPK7iIB^hI`Me^3oZJmfiO|0S19KN#JLg0Lzl zg8t>Vr%Zxzz#A9i=Rnjq(;-kg5}C6x-Z|~k5OBti89%32__aKz+xXHBTRKO`hvv9p z;q#^N_m1+_vR4T+H)^Nx$>h-A=s5Sb`BW&1By?R;h4*Z6~re_gvM@7U*N@5B9S>$J~}- zk-$1&dgqXd?H!{Cg{R1!g1%2_!h2KhlyPZ`iEOF4qn!P+4+iaY<`bDEI4@;*7MtL2 zt<^vkyvhq87|k7#U^;(dr;G?T8!ytruoUUlB1X}pTerg5fTG=D-xh}j3R=X0ZJ=NwhTvu?`5 z1HQa7?|wjS!q$Jd!lrZ+E2ZP-sy&S|7gHd1t?xJv(v8$@G+4v~A8t1-R&5so=~XRk z&g9ci5QkIeT7?RQMzcGf@p~L{RwT}I+d!T%WpJ}c-A}cf(T*8?e|q%J|D1eA$Yz|$ zSY0-qGFJZ9EA8MlH1^#)aH~A`SZ1F0ivPa6Tm~P+7}*Q|hoDbq6zl=a0K6>z%SL90pYwnqo@#0Ff^cE9w@44J$kB+T`ZT)(+S#xUs{b_+!oO?qyp z!zod&A@7F#VFqu?C{DX>$Hrf+EN7ZBu+y)tz&bgi^~a^~gX7DO=3~>RD#P6N7brse zlS5R-%Zomk@2}4jX#+%8{d+jZUVW#?rwGxIKWc38r-{RUNs3o*Vpnqx{#nKR`w( zZfj$MiQ?;r^Ko6z$IIr`4_Ez&Jxr_QV*(2MZyFa6FpCp*V6fo)!3`$T$6tz%Gi1 zU{YSgc$Q8zkOmRa$Gbk+%&U703f@2xZu@n~8>SnJ;cxj8s0D-jpdlOPLV2C(N$Bxf z&#NeMK_X;-p8Qgeg(=>%5Q5J_D=T!$qQ_s>_ogkH-33YKstg8_tIL}`t~-|`l^iLP z7;ljb{P*6A>B66ODD>L!J$Mh3Z5NbsKN;|{^qoIF-HY8^&Ycft(&NA&l_8AUwSRH~ z$k(TlDmlYyhRWw3;!0Dv4E%536lZP61NJ*AcppnwW{;vU{GAP*10aN8zh9~5H&??-?r=YBtzZE6RSzgD zzsi?23VE(nriNlV7`KwA_^$Pv=LuN(w&QfCqbrWV1LO7`{+L5SX933r}}Mo*x(I_$FNpWDnJseM&G$K(IWV=^cW2-n}vWHMOw4&PSAK`X&_5#J-?QH*4`M}p7|-`mw9`uImTq^ExC2&f8FpP#?nawt+UqYG4gL{o zt;rt?rciv0(rJjoTvHAd^uhQtlbVR8STXGqfz0iHQ0*CF$49hUJi$uJ=d5S(BDu@| zF!Uj-TmfiF5xQLKk>OkppImpr*i>QH9*e=sKznP5#kB`mwZA0Ezlr_!AAmBFom-hp z<&-Y2w(ojZd>sEVtgp26;l~x;&ZvHr*Le!D_vE`{9g_hx#N`vN&ut#ZsV3qi%gjX& z-Q6LZBeGi~soi5I!b**zOkt8guI2-HUBC&c;@#>tT*l*>5~1!YlC{Y%|9HDyt_ex` zv0+8xgKB3qi{0f%(*zMr`W0r|6^L}2?1Q#O4v)!H45aKkn`~#*X;(mcSV3j#9|o@u zmhWFc8{wmIk#3KWbsjQ+MOil2U-(&(=Wz%Pi^D{n za4%7cWZX>iXccpo6Ab1zvs3J_XvHkJ;6q6*B1D9IwS=ST^c%eO#}F<9U-%4Xo9sR7 zi1*-u<`DAa7Uvaf@v(~3OlVd?zqnLQwP&;WF|IJ4hK>o;cd^*!y=U3i)Q+Q~^>^dHvBJ`T2qS|rgw~X44WZuFpDl{t&;20!SXcY&)>THUDf2Z~X z^XE85sh@+<17&#UBFx!|2{7LoLb!lh8#u&QrOYPv3Z)DpT~eEy5}|L=U=q8lyt5b1ll^WXE ziYhVok2HVG0iTN{gFc7zKbX<=dAn#Kv~?`%FG9Pm1)&M1&Xb&-Zdm*&kL$at9h%r< zRnOwPTU-lRY6XZY-M;;xi4p`Vk&Iq%?8C!CkznsUsV0o8+LlchfH~Q$u2%lHQr}xLYrDDl^vkfY8Z7T3a)GAy4MI8 z(Zhd4I!>Lkbi%Rgn;ceynU^nCRO~nfoO_?S9)MNO=kJi%P~w4OooG@R^()KGPjZC_ zg^gOp9-YO(TsmSgM1n8vbhwijzdMYT<6rHu-ekOIusX1wsb(&cpEVHw3?ud)i0Rl} zY5*0HK?{mHT=qGf9;7)OR(@6G`W?*2TF(5;suBoaqmE#(8U(Xx)i`&E9n88+S#~a# zf5Vt%$rRRidE<9*WAzFp{5-)16mz&*B{Ia_kcv`@ww!+qu4y^~w=4TW*uZ5;f4^}BWjAPF} zGzF(J8wN{6YG_4F+u34sCX3THKq{k`sX^CPsL}H}6_!|xKZJV{xJ_4#D|9-PEnc?} zqrCn=-FC{=+9KZ+wbxja0L*YhhwcM0l^b~Vn4~;WMpVWQD&?w!nyQ@kUbf4ow6V`R zJ3E_!W1=9Ha7l6>=ceWJy73g)I!=A*F=*QlPx$GkZmnH;yHaIg#ZUOl3;_MT;|osM;lRZio^%F=#Q6=OzrYtSp(0WV}DB> z8VGwT+A%omx)9(3%|*8Ve(50noMyy@Ws7JldE|%O$?wP=8W|;$>IX7i%x~c;w)dtrX;5PKj-JRx(qjy zli*=L=id%Z>|PRhy?5gsrj>bf0n(GaW%fJusEBtazct))5OLrSZcp)L$sW3-ztn{# z49KW!F2NbxmFP*_w1>41@)so1$eIkjjZ(|1=07Ao(O>m#C%(O*A;YPG(-7T^MPx z@pz4Mbq~&#r$5ak_(3m{w)o!}`Jaj*ZS@}tD#jBxJil*b?_?_4qQOJCz;m>*|C{~Z z(Lw{ttzq+CQ6z&yN}1_7eq}3pAtTckj*Hm+*|AZH0hN2~LcI7PnQ$Wsi&?arp~BGM z`^L&v>9b8rV-9z6uxB{44VtKGvAwEU~K{`$5LaYM8>z~ z7jqe@y39TACBa3Z71tji6$?=a!IHeDFU&sZHrCU~F?L91zghvQujXBcHk4$8TUH8X zb2zN4XOPE?@4-8ptC(Mm_ zUwjZ`t|&P~a0K3y)Le95=B%lz3{ltBlQE^K(ays4k$KQ;GtUvYmAyCyEfcP`n7m;B zIq}t+QwTD1D$~KYd+lXQu_vXI6`Qep_i01C_zK|2-VmJUpNm z*M!jLQ#|IeE{RNR+d>L=o_SMmJuMMtHuGJib?m&&11|m;TFK#MWuP?b{yt``M=G}3 zWPqH*R*3*0DC;&G-dmRMyzFDQn{I52iM=#>O}vxcft;r``TK0licc=@C*a#LUb<-} zU$ZdC9#E?s`h;~u9g*MsH=UA)VIq!m%7nWPFJ;KgxpS&~1vxFc2r1P2tNn8l| zZt@_ZLt1%PJMFTH$fZ_SGZhv8l($4yn^$rz1+r$`_fUt3eW{?95gCm}`KkUqg2Xv7 z)%DZZAB`K{CQ;}0UPViAN#(e?8A5j4IANBM=I~tABSQXD^i+#o!45#7#OU*{3#1+_ zdc3iNS_RaV?1%f}TB#mrph9tb$~{p!=yHSu3^?qFNuD;#LHIWKl@R+g>?!!n=m_fQ z|EYWb#hTOvf%R`Kr}@g@U*nzUY4%;u`?W{N=P|=#@+2agQJFrku$!&)4^YgCYt6ec z>NiTNN+kAQvYMS)-ZqGZ#TID0$k!`^p(UCV%-;bnd7-XLn;D5ohgA#%SRyFgoo>S*YCNZiEb zCJrw04i0`XS?oa|m&He*JgH2~4T)g0=AO>+RUsjRHhuRPCJs;9WcMCRN37NC`SaU# zj*2Uu!6stN_h?CC#O7SAD9`=yT7;6&$CpcKIt8lVR0jbAeX%z}&nuC^`7S^=$Q_uU zPOO48(w~%&2km(J0~+J9o39!X^i9vp~uI+&*yi`T)r#4FF>+&66= z_;mkbYq*19f&p5b+l5IwU&iZss=U0-1$<4QKA0^()BPEz1fTs4B94>RE)vU^#Z+G&__x5;y@c~+YSi~S(6H{sOe$nPv zO0Ir~x0x@Sa~$*0dLk`6@9y5|C^7l2$xc5~j_9LK`u*Mo5pPi~-{E4N1VN<8W2Rn^ z;hmsbryEdsHI@%;u+bJ9qWKY0-iTz_`vpK`@6F1aaWxmKP4_;Jp<51r1QcK&!Y$nm z$UDvjS=wcx5_LnWT5^GEGI;s2^f|PG2q%t8%ALci$ctnF0a`kUlvTb zIv_*h%khFut}l*37HJmka_jZL8RDVY$ei)~iBq~a z9mY~CXC!0SLX=*x_G*wcUD(Xv1dy7!<97c@>i?IT+z5)e^;YuON^iS-*+Q8!Nd>J* z?Unqfe(-p}rU>{wj%ME;uXZ7Om$q}o+&N?8GwE0ji5c$)KGvLmjqX*4@;{wLHb1ID z_xb(qG&yWE47uweHWcLR@uX&=MZINq|gD{ov`u6Bm_5)DkSjB67w7d6JHR^YFe ziUMJd?a}7lFg%zvgc(aN@zw^ySc^{mVOpq`5ARElTsAre3N#?yAO%P z>D}vZij5_JRIVBit(L0VL=i$T0nz6Rw7wpTENq@_^eYZ5#l3H>sl)#!W1G$EgW&B4L76R6pV52CsJj->s$`BU z+a~fjAL-m^uQgYqOjK7C8y1i<>MVpkC!dgN<`8Vq=D5Z#R8rXSP3HW(x8|79XJSsss<-pcNj|nGzo@*|aVEuzG)1sef5%DsfYfFEP~N4nQ|>+r%6B zOP3wF?hJ?ih%8CJx7U}l_!E;!HEz3Km(r)v-9JkPx%)EjlR3#NUQXHVIB0raLqdbDr?=fdx|#@Qo5Kt2GPu{xpuNyOnqaGk_%zKO#R^K*chtd1;Gpid zcMsc4sL@&?B#P815`PLe``ksff4ce`d@15^vXA{A3g-PQKqDR*+-jy;+D+L@kspcYHauAQ zd0F|TFeXQl0(^i<&g=c$4r<+@|LDJ$#7GkssxOvYrsI&RhS}$8ztWy}5X#q?q$IM~ z%G@gBbGnYF*L=g@xmaiDL6v2n9?)TxR``QS#Ao72o=!)t&7um|IVD&Aa<*z(&0~`0 zVA#rbz8Nt%tart`Se&`n&$pd3tTegv0w(w}qw5KhFn4#-474eLRND8PgR!x#>$bYm z^QP4E#m9Bo>>!C9_N27A>a22GDsivq4oU`HjUVnS=YQLZp&}QI`c#9xiy)B2uH zllds6-3{q=piCa_UOrWyUYGM$8I9~}=Q6m-L-V#Ft!~HU%(T+r6Da5L@(sY~7J9DY zNFw8VUmpORFrW4CtOHo`V`_o2b7+~>_-(i#%!5S3>1fHzb?>UBVAkd=zWmVnIJu_Xe8jD; za(JkLtL2A=3J~Xb3l!A$J{8{p=cYa9PEEJ>^LOn-q=hqA;BFK2Gy zfG73}t%h+GiOTfZEzjvhnu#g3Hj{gA(=lwHcM~O;kgDF~-;hS|zV$T>_+S9fg7VMr zr-K0O5%ZWVbpAUG!iqyyq|M zdh-5?MKwUu>RH1nlGO8Tv#LY|&reU|y@9tcPL%(qMwJe&vP-qZdFU=w!pBEg8PADw zG?P>DDX&UF^m`}k28|q{!=|&|C-D~vM7VS$qE7^(`9BuT+i!HiE_;1-1YVSk0jjwh z^X=G=vTl7sIPV$_wj7k^ZFv2;~ZYWFr0+;bPs-x+TD8Clpa0>5BF2U=hp znlOe(#v2>Mg|9d%1!LL7*!$WE8DL_znJ#-R5AcMuCQ`WG_SHKKzO4cQ+^iJXS@x zpn5H(bq@~ro7Ajbr9Po+w6!I>$eD}mf~UZ*M%LjxH4YAxy&9(%UdObGRfd`{$G_GO z5@@(vn|`gD8847pOuBr?oMoDjb}qX+J-H0>e6MJQ1H~M!{_UX4`$yeV9Y@NvTjlfr zdI98#e1LK#7!%bI?|CHMGBeuq|2yI$co;eWD2qnfX)S0kMYpW?3Q&>h3{L~Xf44s_ za?;(JJGK1P1wXo8%TEmYd0yKeiloL&6;Atv$VKJK-}?X}Szl|z>O2}rp=*TrLyp2x z-NmG|W1(Od%0@#T{j5?Zy=A$65rZ8mZ#xbpr7TBN!q5xCWpLH1l27Sje}mzs)}pn? zj4OY@Bea<{b6iU-(eCg3BM0|~<7W^|kUH%F2LNTVVYs3$(FX-fv)>{OUVRbx z-49;Vb}PniKN%#0%#6iBfRBKHSxRc^P0tMCTIjGp#1@;|n&Il>O5=B^8Owl1 ztVQCF|K67{NQAqpfg86{^m%hdT*}-QkNCLvK4-WFXDhU6(@KD8t~c>M>;>3PWbt`?CDF^Dj*fcklkZNr`Jy#H51>}tv2^*Al@#(g zYs0;%r32Y(s*_@5ZcDN_MS(URIGMJ$x!HwL25+=4@u<)=oCDs%2C1)$7XK0W{v|Ik z{*o7;O8_o;b4AdQ%I|KzdFFGzZXI zfrX;oKyF_pV<|I0tF0!~4WQCkojn7H0)c*`mI{ngY@k%eDFY39WG!={) zRq}$Hg<#~%6iknK*+yfBMFuJ6C3=pGHtN6JJTBnuWOqqi9{*b5|0G24wtF2p(>|w4 zhYhd^nJcR7n+(V&D+X)RUPc>85lffk+HLA&+=(~E92UaV87S)lXp|}rVuh`$a4J>- z70eFGk!VtNFyBG|%zv|LX}BUj$DI_cut=2jyWKC{VHEyvKnt$ku?nZW38LFfklxiI z_Tvs+Vp@eKQ~FNxOHtgx9L#I@1L`GgppS-aShvcSNY3A>HX{3Y5&l9lX6h<%-9WV+ z(=A^Y+^}zH!a99EMxyyvvud7@b%@?l^>H((P!XiXP1}gw0)pRV9L_h|rRI$s)5ybg zI|s?|>Z7;O$?)Hbq))*(N!Yllk7nL>NSTK(2~v2D4ub+b$JaAP{H>aSGOap&`I-(V4i<1_KEbq5Ei@rr-* zj4+;@%yW7gNPCOpDE1r$r@rB+FS>Of@OL#XjhAu@k`kHoeNO^?LeYm*>T2xi%@MWm zf^-U>YgE1JJ`_I6mD7GaMlWcF{2RAFSs;-nQo}#p$$|Ol>XqaD|IqV8B7c1tKMA0@a^44}e=Nqk1sG_kXA$ym*xCZN4xIN$LtD$P!wATuQ3cQ{mxL1P#i7tdf+*$IeHeR1GZl-;=c+Ake=pjkUhIuag{V9^GehxQ^E(3jU#or z?B?1d(jhBgV8xzhzl=V)#DvnwB=iULoftie92F*}``_&mT);j`YofR0lJgqBWjDW= z7HA8Av!GG%VwipI3UNPN|26}Z&BmoHkWR}p#207$GnJ79ECpi42I=jt?ACy{jx|m0 zY#*Q@<{Z8ictt9-a(NujQ={&{OkL|Td;j2#i2m8n=%#EtG=sKX0|q`72A=YfILEu? zD)|&4Xk_tHk-Dnu0U?3z`PA0xWM}Y7)>VRpi&C>=Tp_)413;=~l0Th1Hl?VVjN{fL zW9&wUCz|h`lsJ4Vr%0$SiHVnQD$=abi%l+v*ua!fCxKiByxX(O)m3Vxr2Vnck2Mm) ztubW6g;o~*7fznVu6hT`K0#brU{wm0+~L9UUv~|%uEp||d4vrB#Afk*a9Y=6T*yd# z5Qur!@+9ni@DnLq`+JJwJ5fzG+P_Y2fVPibiX^#Q2I?uDz{{3bZ9Y5MMLva}lYceo zYSLOZM=#C!aaz0s07za@HjAWU|L+h64iKjUwigCFm|2n&uUl2BrFS_>XD*|FARi-=0r|0!7`H?+K&mWC8h=se_7f z5EwK+gw2+3+c>Ut`Vp9|OdGs=XXH7rbf3p8T&9nsT??E0L_N=}+lisuc*H^4`Y!=p zu9Xe^Dd8+=m|lxBY#2~X@&4K-yt3>IF-WEqWBhy{`{mUtexj@HS_d^MKbSk?RpJW| z;c>TY7KuAG>}fP5RGd!%lPPfX0>BWfDLSJMG4^u?yHNQMX9Q7uR4tCQFi*6p(s3 zA3hWKcvaV9xP);m9JrCg2c()>82l#rLu>2{xxYnM0FaMP?T*de%ug__trDom#?5Cx z;IRSLxZ;a%TsAVJ9bWsG`hFsTh?DtVCNB{QL`}u1{ckTbAGQjbPsllqb!K9s9snBR5I~>!V)4IgstV}iOMb>M$b<0wnO*@vG(KnbZi5R6 zgE(Y=u7}gFt}{>mk#cSy68;hrD?HV@RK$YnEbMZBMJ9js8gDp=D=3*~_9D(+<#4_M zS)O=y4rSvMeI*%OZaLkUF~&%pp5R&bbB(Hh4N$13Rbam(e4Mv9VPezfWUeL(2Ssw^ z!KsGhuOk>%3U;0?NAI+{3HN^WcQS6SdwIwri7pxqQ?K?M`V8DYg@+;?XHX*`-g#N^ z^9A5Qn*Uy7xmxu;ovYM=SMCkj<4aDVl47G`_ZE9<%V`7zVt#MK$$O)PGfu5m+vu1s z@6INh)fcG&n6sN{9BZgaEbuDd2*=s<=pWn;;Z+!y=!d}7K?A#NiEdtlkN1FwYs={^ zw-gDtLtkvMe}6?94kNQ|lmO!3aVY^8UGfxW5kRC;()6R9>HVU&dg*?+62@jyZ21A} z4M_n-2-jzwF}-w@W+Vh30LlZE-TMzv*=J}Aen2kxvOXV8mq#)mM6UDMLz5=cC4R&+ zuqku)iHOc(9E_n6e6K|nTAgw{IxQ88M82q* zhW)jncb3!sF0a!?D204t zY+IYBA~ZQ|%(1XyiCo{Fry5;oRnF;Q zPw$LoTRK>`%(?*CgF%1_wdeH9a;N|~C>hX`jlW3PD$#1RWidL0NeupAE0`vxKCRUc zBM&c&Rd@upW8_viDs>f)4aCBEZPmPd4?7#FRt9z?OG!lO|KNpDzP`Yt$hPtd+8Rza>awv4sjYtt{q7kPgZoK zA0i6kVl(Jqha|shi3f&_=y7$A`>RUuQOJGDU%6xiqR+^c(BWzlwUi6L18%@`-Sdz) zWYDbo>SSWPkxYmJz3LCnRwY@rHL$uh;`~S51w|{1-bUbhpD_v-#8=spm z=2<&2iCd9sYUg~5D#Zu$F&JX)R|V823^=w?WvPi$aoV+vDRFfr!`T?1Mm z{#P*TcqM_OmK)4Lb~c`Yd(2dQ?*Gfye*2p)2PjZiqi-);3@4t7x9F^??YjZ^BN~p~$8m{4B80Kn6cC0*bvJVFkbuiDMk?t2Iy%GJ z1Zl2RALm?jompT8;EyaU3PN#7ypm`-YbVFdt;JBNBnWhcj)kA|yB|!>Z%2_p)wofP zA>ojpW9cv!?}d!WtTz9uI#qr1kvRsXR@VbNy{TtqUGVU+53@%@$#Q@5N)}85s@|Vt z`ke@W5(|l-fMKx4Dai&^CIEMG(!>0q&uNBG3#2gnbqm*j$H`}41zAnkLx0~m^-R`E z9QY9du8!>`f&!$$(?u3IdFVD1|9<-r)&M~M47+%&J_TD&Ym{G?Y^WyEer-JT{xh6h zp6;Z)3dZPiLft~U5-TCY{!=Q%rzA@7wMg#v^*)4IG+Ur$^KO^*l~;S60j;f>q7SsWn=NRrt0(ME(@4K?d+e31&9c zo;8rZ;?{OPjD`i<6nzEVw#XRFmAMR!K|`LHBEV4_wBATqcxn<>pi+32BK5EPw{dxw zbiPa8r7i8X3I!4-Su~YY48L{4gV}V2wwS!X1u9gnrE+lq_ha_Dum@mDzougARFEU& zevZeGl2}Kr0dDNvdKyb1`@!zk*mIB+Wi66naDQ5yGYL?;VK7SNtfhE{{Q3^3rmC5E z&rJ-X1f-9X@5)?Kg?Iwi5{j<33Dx}1ZX)ciUs=WBc`S6G0ErsYO27^KyL3S3m+y!# z78)aS@<&CkY?+_?D;Gt!z%3@;hdBrn&!M_?hq5tlxm;I}1Pu_Rw))<(52hZgFSYrsuER2ZOWQ5h4!_em24qZEDlHl6 z`D6xNVgL~hWcvy4LYx45qj=brQy!1cC3(=8VL*9ZA|&Xqi9Pg!=o5D%J<;-ZQG=eN zvWPwE`0HN z!JX5NOe`JE5kd^r4&rKApkc8|N?|fY<$IQ=z!54Hdm!4FYM5Y1&hJkwDw?8G8~64J zqWk_07FPK2l`n{7Cu?!8Ks-F(*`G>8x0J5a4IU@r2d>oR=Vb_AnYFR(_%XH5(Nt$T zgyua#_`V@meqtjsME*R1gYv!hvEe*loZ{98HBZIP7XPayugH>4>sep5Qrxz%W9D!` z(WmMI?xP%CGOeEGmIZ%8NuY^yE(pB@ms_NoS~4|$#-6k6fj|!R+Vybpe?y+T8a zGgJWFXuo0X*omop`2LOm5R>nZ)JR*PG^vY?*nQMRKd@jzzy$l(yWQ?3m9fqf9`57r z(D(q(yBPxjw~N-BM*O@(^!1zY=`GbjNvrA5t-&mQW+c>Q>ni2xo1~1pt2#IR&0)XM zW_pavK)t;&KgqLZz$F_9-Xnx<)ntMt4?D8Y_ zAm+hpnbm+LX)@a5-Q}m_Y;TqdfJX#MxIw!kjWwi#CPVK2_pFU?e&lIYFFJSYEXC^5 zTZ8*+e-kG}@|;||a`G~h>+xt{e}c=Dwgt>`Zjavc0_}9cn8|Xr!v|BnY}W1OG0!u$ zfJ>UCOnCz*%r3X*(v|+xVg7PjPTG(%0iXkv0!zrGb))Doxm5ZSbe6=uDC%xqHtvAV zc4tx}iX0vfw4upl~Q!#^~3+6AKTk(&zg!zJy5ZG%pQK5jKT;e`S$UeD0@BTxh2| zXU&yt6`ZmD)&bfszq6e_ocL_v&vfi$u95z-d3{Gs-4-MhwsgNd*SaprmPM}$s0D1k zS^*hj$`tNL&@gn8POf$=1(W4=ZL#*pMxjnrbr>4JNmzEc-g?v@zac2s-TSr65|*no z`#JXoBA{TUE?1Q;J2Tww{*nN;?Y5>3V5ZcTs(za^{v!~w^!ClxPSpVjmx}mK$e&_dtdTtfEMT`XbZ%D`|d;Jz{eC4Jjvux; zkJ{VZC!@K($yG0>p}RlgZ+(GswM{7;b|BeocYowe>iixhaAXooGh*67%e?ocKsd0~ zE4>b&YNx;axm(Wey>oZaRx6=WV}H()`@|0}jk<18uj`{ngq%xaQx7(2`wM*LOe?}v znE4HbGkml=lg&fg~k0^Svef z{DiVhqtZt8%g^BF1QQe2TCN0R~7E&~Jj44br6k!Q_D?AoRT zwdyFoFj+Djce|7M@NPQUkM2@hPPdj8vbd_Yn+zE6ovwaI8FP8PtN|=JgsntY#r>_) z1;2tC8AgYbnz{$(gl{sb^QxyfvQ~6zExDdouD_)%wz{DDbCl(it7;t|A8V$GD-sy; zXMZOnb8L8h6X4wbtQAPb$ze;Aj%G0w)2w`tN5ECRk@V!TU+s+4Jl_-nv|VhbC)Gvn z9*4EHf>#Ub9st}Z`1I6%3h~$(7hD3Q<_APziI}0SHg9q5D9L{1?u9EXqW^v?^`EUd z_edeIr+%CF3rq?j(JNBWTg`FOvme^4G{rTn*!4hEe`hnqKwQ>>KVCDO5vn}6-%*Mj z3;Ki`dmFa-i@9$othcZ0^c(ON?8IZmI_<{_Z}w0Fx7H74uwb5Tj&FhWimso969fo) zzU}gGHl2uR2W^XlqV3_ByiWCUTCY{#uX@!*MEe>gu$ayKd{L1i%qp0Njg96H!ppyl zncs^ie1yH6A6q}RZU9(T5{>R^4#t^LZM1^-v16b+B!lLsfm~^kw&u~9_J@gW3)ms)CwE+-JBpQ~yWnnR4@F{izX;)mxIabJj=~X7L z3;t%mp3FFQeW?(%>uABJnv`UT?$P(|nKMsyv3yvkA&UkiZnpQ7YO>wyh5RK@(vs1D z)!Di58rW|c?Z4HZ%Q?)BBg{%T?M`ejOl*MF2J#%7$?#}Zw59UdN0#z-J>fS7^ zbIWJ%v!(#-_G3!Lx=n|@zeEgS{iR-YNQ8XNES;qMM1XW;!ZsldE(&uc4ALu6mFA*y zKyqyhn6I9|B2|-~Q$NdYb3FYFICLkKaEo8u0<(EfagP5>!&S7ZGpzkMGGN`guQ#Ys zY`VXW9oTjrkE2#)aRUwZakW6Vfm!j>)GJ^*#ru$i_y}I_eOZAr`hzGaI-{OB;TzG5 z0xFQ4u@1mB&(r?pH%>3(*1eYRHX427j`ji+<7pL{<03)dJg4$9o@s*gdw?7oM&2!n z$;}m&P91l05ZIKA62JPp?$q#G1UORQhvj72x^-BH{y>K?JgLuXqF&E>x&Uw%{y3wqaJ!&=--u7a{Rro&Yugr z;J3({JyjdWT@>&1LoWu?9`Rlx;>m&2xe9I7p-%n8e1&#JS4lM(9oRI6_(?lz*-K}I zJOx0bnMQ~I4hh~Ady~o%GJMYV|Jr-&zbcpSeOR`jAkrWrok|NxqaZ1zf~0IhTIudm zkd*FJ6p`)**@Wb#rMsmYq`$LKJotIedH#a;dA-grxbL}V)wQl!GqdhHr4*KPO{OK% zg?(5URer>iCE(7updUV()HY8~4aTOv!`@huPoN`JqD}|-BC=GiE;Xi=HaXcvpxBXG{5%F`fRfFTch2La3(Nr^xq09e1*Y!w`Z}* zsygPMt85d4$o5sUgVU{5(iO-7$Sc3u^GiZM|J`^1rI+ho7yCc{KEuXHk&#s0f%2#+ z>-&?9Q01)Xpm^Suq(VuuxKbs$2~}j3&fUxnne4Hr0lVkhAF@8E8~ERnKZ+DRd>Ph? znjM8^e+jf4*>Hcj?as0e>A8F*q~iC*f?xK2PYqjQPQ;giaiD=h|AXn4LkuRviVkC& z{FJXJ4~JA3ut?Bla<#XC_n76blbEAGKpzn=IW6M9N{R8Z?jXGAT6Wlw`YAdy zRc{}W*(&P5(`PB~$kPmPUe{%0Lbk7d$WNrw`S`LKrsnDU=}?-g)YZlK!LX<4y1w^1 zomQ#N+Wc^$qEdF2Nt4G{n@j2h`@_9<&d&nyUWrH&_-fh2nW@Vd?e2td$-P^wPl_T5 zqZ9e^M_B*d!vfdpO(2zHQp3|7RbV@oVh3&A^)mWUnjreKPRKaZ!^^af@V?7(y;8xd zars_5$IEkVYRO2Hp3z|Ac*uJJlu3hm5Z320g|?I-gNy=AgcAQ{eIx9N zok*jA%W5?M0Ea;1MvS4ndEz0mBaKHFDD;Bf_;B|~Ef%o^~f${o5|4f9RH^|7iGCG44 z+#!j3jZxOY$w{XT_$>7wCcAWSHdLf3G2C8u)mTTROhNA z1Aal12<%LgNE&5LhNPjz7ew3tcNKsS5$t?AHIWG2Z~@J}u^rieTkQQ6%lY`S!K2W$ zsB$Ouib4thIgI;-jU&vz1jUVJ<3J|nHtD^28-DX9se@jq(Q~*G8`2)4VbUsxQlc?E z>)rbf)@5cFVEa)8cvpmNW1>?#LRp}<{1wptLWhp-BLNXRiZZ*y{F$?PO}cDs-Zr`BEu zU&<-zo_KE+1yI$T9#T0D$fvaIBLm`T@L^%u&`&|%cQDEO$ctVn+1WmWlK=9*dJ!!c zEQAMa$$FPxZ>+Vj3jR%gnUOHrGN2Ag1IaR4a+u*7a72t9`T<=WuLRi z@S8c;E2r-x=_H?jSe5_1cZ9V_Yv+ZS=)}RFU5o1kzjiGwU6uxg=_Ve4rd!`!8d}1b zXHh=i4r1ARcTun?gkY-!RviezFn$SU1Q)%nHn~I`t}Nxf$ki zG1FJdhp^x!(D3LY80Dj$gAqF~LPiOMjE$j2QUVAWfw$`ZC8Ih*#?ibNtq2*3&tz-| zGK$Pqf#Yo!lA&YI?123w?TCa9rMYLkMX%zTy_I6MXiRCd_ zu9pm~3U&R8ztg*@)aR|}GEYcVS1;1aZZ0GZvy_#M`7y9X8GHa<2U-+;Px5+P3_C0F zbR*joy}Q*Xl9Q+&KZ2|L`yTiP&N<#RQ~k!9X5e=d&IpKi{tVTrMYOA$gqI@!g*Quh z0B@FpaMVCYHg^L&5ECmuH!)1Olhzj4QNB*h2jD8f+>m27Ni*%=A}?YC*C^0P<&Z+%QH1(|j2ww7JY($|`-MaMW(O$&y_2 zbiyjsvsQ|`vZ^Xix62s$$_?wa#JrueDCedEZ4)>OUyTw>a+|4U-he#OfzOpBJ-|F{ zdskljnH{Nw3ZR22CzQ;{SpG&d5j!$6Y`DU`*^|K?pMnO5;$gZrGUH+wfkuVHGSgHb zP~Les!*)N&bn3S^^37uR*&U=yQrGQCP>e19dk+YuscHBSzex@m-2^jv)`$p z`4EFNeOVxe`mYF(ffXTUQq=GU0J;pt3XZvMH6^p$i-*Iq68s{cC#2|DNfhEV!38;Z z5;tjRecj^cx)yyzleD*g&M^@bc%7YN>cgn0qvN=sl*}DSkFmH~7|pd{bLNLsII+mf zG_+q6es}x(M;jD42pmN<_Jqg6y2Szy|PPaWf5G*M@PA@ zzB$i;o5Zv5iAbOkV9|IXQC*tnh0C5-u}Z0U2XyYdI?sDZUZlMu;DP{*|90n$g5 z&zg}FnwKhHRu>+V2>WD#I^xdq{LJ4A0U4MB0^|iEp?HX}qF@7ztxsMDd_JMz4ngZV zQ+tJ%u9W7(_R;%uvf{|SvmdVD3=}z$Hv}XkFO<3u+&sQhC^z)6=Dt284S~1v-uxx$ zv|5EQcrYaeEPU*cP^cwTLBpA!AM~3OKn!r!^#S;&kT!4#Mz-&z*AagOvjOkV2Nysb ztQZH4Hvi#hw^}c7wB6mZ-{$n8HD4QZDKzXQl9Q9Oc!azvDi(e1aujt8LLB#JOyEhsrC7nXpZ42`^~3m9NK zW`orAh)ksgX_khKxr%HSuR0cOVVHo)C5T-_SFEl1R)2z4k%{T)=_~iO@Xnq|d`Q8x zd~2w9Ein3KL5hg*gr#+T#MGrk2XWRi!ix~{%FTl)xR=#`Z^yHthc1XBRJRg}2kAhH zf9oj{O5~kP^^KC^UTqk!)%aNio_mgbBuJ+6k=!K>XX%kn%dMKygiYt+@J{3Kd5uat zy`+RYk}38b4E8$P>^>q#T*6z&hJd5~A{l+^IIdffDH6gHF9QaTc6%CMR69K4E+dwK zWg(DX4I`@Y)>0FML*(;&20GRSAbabCr?%RPw zYGFDy6PiLtq7ZI44-T_Aeo)5{OgM6E>)RV%!BFC#oe z96=>9n3dx%LC{pS_-;Mn2bn0t=J~G7!ZC+3SB-*t=%-i0KDeRGZZjK$Eaq3StA!>4 z?oJXGU>ltHqoQA?!0ps)KWpoGjt;?B5YTYW4x{LTgO9I7t@?xo03%CMBxuSCDG9nI zp$7-6r44s2G?pi=*==9XzFBJ9BO<@!w?TyXW|`z6p;ZBtV8M|z>O7a;o3HwXl2 z;Q3_3h+7WZZoUcaZX`A^w!jkxCv#A<8snXL?# zPZOvHznAK-dVVK>^7lo?xCmE=S)nZtRbhb+8(r2*e`@8Ej#v|dnwNGoqH5z z_WM6ks%B@sLPKdHke^<|m68&oGbp_6-iTYTM`#-X zGQq%lHc;&}IGV!=S1N*OQaduA_spUlv=p8~^DHjrC$+S)x7*mh9L;P*i2N3W6(Pu3 zBJ$PLx}v+&Hg)WeoaSEJpjuLZSIbxBrWM?5?D#CSUM<+mbblO*mR6^sfdq|s^y6(Q zljKw=Z-aHSt}E^>0-LSks{A9wD*KNd6llooD0AyxX;d=F=`jo;Q{BJHTfm&{e_`@3JuZ_<1-5k<%t=?!aaY!u7X+`l~v6yr3!MmUm%h#iP4`?>ajwD1EW)--#j6^ z&wFJgaB4?M`;jHl8C}^zrLn25afmDxmqk=aczyy@N(!UFSAuWs?ly#q72w=d>EvVr zHKjAY>kcb0=;%zhQT80K)LY2Hw!fvrReii|F<|x*Y>tJXl@cJ9@)?txDmn>yuPTrEQD-Hd7HY@4jx3OKSGD4 zh48u1p1o-Hq=MLE2t7oTKbeUqw5}r-z7|DMOG!Dx^obaIic4nB+$vxJFp3avcc)Xj zR17FL0y2Lm(NJMnBkPy&D8jK&nsDS#cC=_~H0cj0SJ5T;uD^J&F3{lobFjMK)#ekK zh8eH0@77tMg@UiqavKf;{@`rY_96ECB8$!3pL!1&nKy+c-I&XzOq0cpJ+hJK|3?bz5)qy5PAshzDwFxw}S6F^01e}l%!>)WhGJ&5GC?cXhsdsLWlHQtk zi4Tc<2R|*S-#JVu72PnRDt?=G8()rw6Xfyn(F;3c*w?H-uN@?0+CAr+ z_K+P`?P*Wyp!xZA&9Q&WDRIZ457o%UAd3LU&GhxEk39?fu3or{qVN32>^;4DD;wW= z%qN&D{`T6hcpH3$Xv!`)tTv=)!}#6{BA;RMv(`pr3cE#^%?{vNU?UP z$o)j{I#@sSG0Kvf6@B<}1w8jy9k&1Wn=NGJ*?3gsNIpN;$w&E?#v3;dHlw~`lOH#E zM?uxjhUc;Hh%?)lIyxe()&RG&D;@PMr%v#4v}M5f0|WMk)YwNY$wL9aONygE);;=1E+U?^Hn z3x0Nh|8wJs>#Y*k6=Hv`2h`-tr17PeE`9^AAQ=)cw8&l#Wg~}An0J+bESr{_8#P@KO&M!0w5haVf8Gb?Ds%{G=Vsy}Ff6GKsxtj%YIk&$Pf6x`N2gI~EN?pV=h)@F z`p>d)8YNW(zzc-llX|!uCI)3>m$7_KO${Q1+uJ(km+O|D)RrZLWC*(pQBNTIA8XN~ z^1z^I1?I1G26&2%D!Zz^(NyuqrNOjiI=30R_9hs_BTMCe{2>-qb&`+*4!f3?T3ZuS0NwD@>9r(#AO~0l( zus&=za%0VLp8s)q3$hrRApveQ87kRC)<=TACb;i($14Ax*CAXPdzmfIy*AQDYdM^h8xo%5l)!z z&~7*5fx996z0IU!O9=0^K{H|ckIJVKRj2E^r(Qm(8qe8uxUqi)SuHY`Y1w+{V{4W+ z1{P+bD7|p$kM(-A+FRXkV_Eg;&8S+cxshzu`>XE@?98|^W$9OBfnZxoyM+igB}A}k zLsr5@-$a(W+AaNLrfumbj4a#q3u=6qRPE{;!Fe0uEMYi5S1*qTRjiiV$%VlytGIbB zdYZrIu^>SV9s}>>FvyvohG5=Z!DvJlmpEOmER#}Z#`)hmVDKmeje%pK5pFM{8?PMOj}nyj{BuaWdoB293U-|g@b zrY4!3`g5tWLQyCexkgmA+N*`>+l0yccB5ugo8#2e+zvEuJxePN~?G&gY@A^5+?JYHtV$@=V-?Q`v1>L;gmp6E?SD5vP|fjKg&DOdurNYUo3{Sv zq*=hid<7Xc743rp?D-4JuXJo|1zQJcubf^NUgKLVIORg_pu=X(d%oawd4sXn_*HLZMNtw~9YPP?pt(~9cs-#WY zsK#ia6ySTwV}}N~_tfAG!=$%WZn|6F)^hkZ3cS?ST?bPv_SDA%5NXrDBXFS*5 zH6^X}Yqe&siM8?1F0T?dVp7s~hieu5wyOdLgHHWO2(kc2mw7DvO;Op6Z+Fej0!i}m zoH2B4NtUGgl;C-Fpr6k{{t&0ccP>RXStD%Yc539o&m-2dK{(Ui*Gs^IHPUr-56C-I!> z?J}=ioKe6FCeL1KwlCu*$XnCnCjnFH3xoam5jP&4fYCodrm)-(`KMbrHbp{?;zq;v^1Z!$4h?Fmk^)67`@V!Wo1Qtx~u-6n3UMYc%S&ydN+9v=MiQwcrhGekLW(uPE@d zv|ac-MQFA(4dL-;Cd^TfzQCabdOuWTx&=c6z-9*Gk17uz=m7EC!|jo`&` zF>kqbF!ylLOM}?#y_0J;zGczN+5c^OdwXNjK!E?gsygn5s8x|aInZ_*?8m@QYPlA@ z4C|k7t8SpMlP4s`mpU_>#+p#&g74k0@!9F%`H|D>B&6S~`PoNzGe>GL)W(>emfK*a zuiH5ht1hmeIi-GJf>S%Q?6tFOO#JtT8C~7&{BVutA7ag86UB?PmZ5ARJ9Cs#)RymUn6&E7P$1~Eq`MX)G7Mq}EpPxBySoZqc=a+SlU?D+T zneMt_a@g(0ne%1P`H&%h$B|VTtz_kon{b?*#*cD%^k$Q2kWn?!&!>v0jGH~lFLReH z&1!9=6OQmqj~=$boe}TptQ~>F;)F{c1h*%gMNkW)Ugj(ZzB|ni-%RwHt#)ZN*~qs0 zsuSa=5DAlg>-g*+bY7PQorwwkq%%%3&9I|ij4^PJfF zV80dH);sei_{g3o(P8^Ae?!y}Y6k8zXa_wsPpo3<@(+lSmomeSk20*y*58-W-IemC zQ&|&Bi<{-x=~|OcbFzt?JAC*>qyJcg_tyKU_7DwDLXIs;$5cYLF62$CrMAJ{Y<&+a zF1rc?lNGx~lY|=b`I%w{J~e6IESs5P`Nlq`vMkY}B2m*;pDbE!wa9!&T1FjpK*IXmFbZ&pla72WkO_PFFxgkWPwOn_d$ zFk3Yxtxgf$Kd9|i*4Wr6-Pr196)}^-MQ}bNH0M2!wMKQj(TX2Szepz@=Naie<3)^8 zTCH)m0dNenH2&Vm$GuSI#8LV5g4keZY@ev|GOvTkrU6_Wwd8v_TS?OQ#<@#W$eD~@ zGYCp!H#IHn=L*DOS8ucvV>Wy@D4boOKjXkku8qt$Z53J%ht_k*maZDeUW+f_-yr?Vc54H?#6_wlmqbH0->tW=jk#Xgnhw$+29>X}vB{Aq^rRyPMn=S4)=%v3eC5p#i0t+>>Ut+%PWv&Awt`m5&Ll2t zB#U^&v8>EepcN|GuJ zyYgbZKW>3d&*%2RJBrWDinHU^tg`Pd^=`Eu(CSN*dQ(fOON~N#$64j}<^vUu@u*%wP+Q0CrN{C9s81P#wi?PGm)vG0neB-zOhNve97oUMFq^8p`f@+T+DTi^`&J-$s0A5}}9 ziERm{7`HH#w~`h;EkQ#grp)SK}S%i_MP z&-Zo00*;>P9;Jv5lG7&nGr2W49TVwke&5ktg3lXJC;3P1t*Vt(D~gN%QhFvAl80>t z2iFK`#m~KzdfLVoObm=~ReK{i6}IXFrccrlg$0<=HF#UwH_oI=%9HJF4)nU^L09!^ zr`^s(!S^6H{<0o0vI24gw>Zf{VBoIAVI6VBU`I_vip}VPe#B@y#c~|har(n1?$a{9 zTM+$7(_YV6+H%^~3C{d7+$z-)>yK#dL2un9-qb3y+~A4mdL`lx^F^JJj!8jFO7agq zq?#@Z7*LMbg!b)dIjC!ogNwt^`uZ7Pg+k0jsg=)kL3ZWq%FfV1;`#hRULl~qi*F&x%|%W$l2}I8ZrWEzEn(InfZCN& zzNk+?y;WmhM1gM>8L>JJHF?kt{wI%&`l$S2-Ur*|8@$9L{*w@9_ScRm_9-n<5~h^v zV^K|SVl1{@c%xOT6~ss~IWTvZHsuXUN#fH|u^PT7X$JUAmvHhopJYA$twkPnL_mF# z;LHB@R2TNy4H0lGWy|hz`YKhb_R){w?DR&;|w@UX`OajB4!)2d4fXr)rvbbL&%7T zsal@W4+;dC@)TSbn=uc+8OS6u`{OWx9ZJO&<5|QQ9K(?!%383UT$XT@w?~^wJ^yCm zArI+n3jAP#;(c~Y&GI~YO1j4C{7B6ppN3yl5D9cBYq&oMhQs#0fGTtZ^{$-gVwiOr4+74ad zxZBZ=wCbCGOoaqt6rG4~%PZn1Ii8#}t>%d5<&P&$pUljdgrBUhKjjQpy|NZJFJurw zolrCH$djJ`0j-%@r!@UaE0<{G1Gld`K$w!}(ge)?)W~tDV2fk{qwv$HDj4rc|o-cO|4EH6nkAv0O`ZieXdF zhl{2GRpM63_g9rORLMsz$nQ9T=5Y~^7Zj}$G|cSF3B$VZgjin=ank*=Y+s(cP=#0X z8c6bA!Z-80QobIgu+~cmZcg4vXMBioqT3ASTg;liNvBi&{z^FVimaiBlWN0DoZj65VA?SG;8*0MZcq2#>g;iL@?Ch$LpWPL^qO#fH$t)%s= z_(n73n-<2PulG+f{rfFi-%tV5xl1sbx1kv+KUR=iU$R!pJhszr$Mhi`F)W2S`^Can zZ0~vDONMy~k-Ps@6O7VA?_K=1x<>h2rF}PdyxPFWe5Wm@=W_(V`Nf{0U>fS5jQ{1O z{@7SteCdM1;>)v=%-=Ko^@dCmB#&Ks__w!h6=QvA>E6=5A9UKrH|MsdzpjlC)z#Id zK9$2e(=P$|YQ%0sYWA_r_Tdg=8^^To*vx8)MEywG_wR52b++JrXLbuq@}FEC_*TtI z%fj1dZD@FBwQ4?aplGQ!Eax){0vY2&VIw0@=vFiF)RQNITT+Mc=#~DR3InW_{W^Mr zUZ4LQT@H9fSbX5>K=!b7nkjT(`R!89yNm}G-z1gt`4A&BiT#78Qo5ui+n;U6i^I!@ z{#(NmK*P9JZ29|Bd&&tv2W#n~G;Lpwxb$;>#1Fsy@TDSRP|1nO@e`H@hh=euy-je< z9lG0N6`yKv;r*xCq=c?H)%IA_6nEuKN_?z$Xp9XA3~V^w8ztqpOSGz0%zT8fhBRJu ze5_wa_KEfK`03PZVcwVm3x^zTi=&fi=_PR^gcbhn4O%F`)t;E+mj7PCPA>jg!J;YU zB8rr#g_Qrqn^bR%a)-?0k#TVs&VL^eQCFY_g&&Wyn?%IbHSQcI@$Z3w z+?N81>NcONz-~vYe2deKZ19Z=Q-8avEz%`R@4PNBVII7T@m8L5xZx2aQbEp#r_+5K zp`(ABckWU!_7qspV8~wlT@Ejl8u$phq+-`YlJPJ38NHOWD*=$~l2eE&ZbfhZNB7HaCbR_IiivZj<3$jR`E*QbPsRM&_!t(UPp(54XoN8sQ zxVQRsDNgQMw%)+i2*l=3#x;SX;qh+6W%%*-GH!8zaFg*Sfa2^1(gbah- zj36m`aB$_nH^ivFoCldmT{rW;jtHbMzk<@|@_uwQ zf5|N}{Xo;e9Mk%&&XhwL95cQw`}hb;=-;CgAqBq5p2zNwzKzQ`vza@*$Hj=P3Pu%* zt*(V;-&W;yRE&53E(u6Y`oNaI3NWq9zY2Zvo4k+G=>!qJtogwz||l46FAtCd%-+~oN$p*#IxEjYyX)|f4xSC7{oaje((Q9tN-J{9(^dtLs?-AXSwkIXM!dmW`>_C{Kd2X z<>&uX-2csJ|5M!mJnny?dY;<;hjHhcIA3i3FFBlR;yhFRFRPww;yjo9FRPww;yjm_ cpdJZoXRQu-1ZrNt1pbp0dn}qQto8c;0h-ozQ~&?~ literal 0 HcmV?d00001 diff --git a/scripts/diagram-light.svg b/scripts/diagram-light.svg index 200b3d22..1652a32e 100644 --- a/scripts/diagram-light.svg +++ b/scripts/diagram-light.svg @@ -1,10 +1,10 @@ -Create WorkspaceCodeEdit DockerfileRestart Workspace - - + .d2-1840016246 .fill-N1{fill:#0A0F25;} + .d2-1840016246 .fill-N2{fill:#676C7E;} + .d2-1840016246 .fill-N3{fill:#9499AB;} + .d2-1840016246 .fill-N4{fill:#CFD2DD;} + .d2-1840016246 .fill-N5{fill:#DEE1EB;} + .d2-1840016246 .fill-N6{fill:#EEF1F8;} + .d2-1840016246 .fill-N7{fill:#FFFFFF;} + .d2-1840016246 .fill-B1{fill:#0A0F25;} + .d2-1840016246 .fill-B2{fill:#676C7E;} + .d2-1840016246 .fill-B3{fill:#9499AB;} + .d2-1840016246 .fill-B4{fill:#CFD2DD;} + .d2-1840016246 .fill-B5{fill:#DEE1EB;} + .d2-1840016246 .fill-B6{fill:#EEF1F8;} + .d2-1840016246 .fill-AA2{fill:#676C7E;} + .d2-1840016246 .fill-AA4{fill:#CFD2DD;} + .d2-1840016246 .fill-AA5{fill:#DEE1EB;} + .d2-1840016246 .fill-AB4{fill:#CFD2DD;} + .d2-1840016246 .fill-AB5{fill:#DEE1EB;} + .d2-1840016246 .stroke-N1{stroke:#0A0F25;} + .d2-1840016246 .stroke-N2{stroke:#676C7E;} + .d2-1840016246 .stroke-N3{stroke:#9499AB;} + .d2-1840016246 .stroke-N4{stroke:#CFD2DD;} + .d2-1840016246 .stroke-N5{stroke:#DEE1EB;} + .d2-1840016246 .stroke-N6{stroke:#EEF1F8;} + .d2-1840016246 .stroke-N7{stroke:#FFFFFF;} + .d2-1840016246 .stroke-B1{stroke:#0A0F25;} + .d2-1840016246 .stroke-B2{stroke:#676C7E;} + .d2-1840016246 .stroke-B3{stroke:#9499AB;} + .d2-1840016246 .stroke-B4{stroke:#CFD2DD;} + .d2-1840016246 .stroke-B5{stroke:#DEE1EB;} + .d2-1840016246 .stroke-B6{stroke:#EEF1F8;} + .d2-1840016246 .stroke-AA2{stroke:#676C7E;} + .d2-1840016246 .stroke-AA4{stroke:#CFD2DD;} + .d2-1840016246 .stroke-AA5{stroke:#DEE1EB;} + .d2-1840016246 .stroke-AB4{stroke:#CFD2DD;} + .d2-1840016246 .stroke-AB5{stroke:#DEE1EB;} + .d2-1840016246 .background-color-N1{background-color:#0A0F25;} + .d2-1840016246 .background-color-N2{background-color:#676C7E;} + .d2-1840016246 .background-color-N3{background-color:#9499AB;} + .d2-1840016246 .background-color-N4{background-color:#CFD2DD;} + .d2-1840016246 .background-color-N5{background-color:#DEE1EB;} + .d2-1840016246 .background-color-N6{background-color:#EEF1F8;} + .d2-1840016246 .background-color-N7{background-color:#FFFFFF;} + .d2-1840016246 .background-color-B1{background-color:#0A0F25;} + .d2-1840016246 .background-color-B2{background-color:#676C7E;} + .d2-1840016246 .background-color-B3{background-color:#9499AB;} + .d2-1840016246 .background-color-B4{background-color:#CFD2DD;} + .d2-1840016246 .background-color-B5{background-color:#DEE1EB;} + .d2-1840016246 .background-color-B6{background-color:#EEF1F8;} + .d2-1840016246 .background-color-AA2{background-color:#676C7E;} + .d2-1840016246 .background-color-AA4{background-color:#CFD2DD;} + .d2-1840016246 .background-color-AA5{background-color:#DEE1EB;} + .d2-1840016246 .background-color-AB4{background-color:#CFD2DD;} + .d2-1840016246 .background-color-AB5{background-color:#DEE1EB;} + .d2-1840016246 .color-N1{color:#0A0F25;} + .d2-1840016246 .color-N2{color:#676C7E;} + .d2-1840016246 .color-N3{color:#9499AB;} + .d2-1840016246 .color-N4{color:#CFD2DD;} + .d2-1840016246 .color-N5{color:#DEE1EB;} + .d2-1840016246 .color-N6{color:#EEF1F8;} + .d2-1840016246 .color-N7{color:#FFFFFF;} + .d2-1840016246 .color-B1{color:#0A0F25;} + .d2-1840016246 .color-B2{color:#676C7E;} + .d2-1840016246 .color-B3{color:#9499AB;} + .d2-1840016246 .color-B4{color:#CFD2DD;} + .d2-1840016246 .color-B5{color:#DEE1EB;} + .d2-1840016246 .color-B6{color:#EEF1F8;} + .d2-1840016246 .color-AA2{color:#676C7E;} + .d2-1840016246 .color-AA4{color:#CFD2DD;} + .d2-1840016246 .color-AA5{color:#DEE1EB;} + .d2-1840016246 .color-AB4{color:#CFD2DD;} + .d2-1840016246 .color-AB5{color:#DEE1EB;}.appendix text.text{fill:#0A0F25}.md{--color-fg-default:#0A0F25;--color-fg-muted:#676C7E;--color-fg-subtle:#9499AB;--color-canvas-default:#FFFFFF;--color-canvas-subtle:#EEF1F8;--color-border-default:#0A0F25;--color-border-muted:#676C7E;--color-neutral-muted:#EEF1F8;--color-accent-fg:#676C7E;--color-accent-emphasis:#676C7E;--color-attention-subtle:#676C7E;--color-danger-fg:red;}.sketch-overlay-B1{fill:url(#streaks-darker);mix-blend-mode:lighten}.sketch-overlay-B2{fill:url(#streaks-dark);mix-blend-mode:overlay}.sketch-overlay-B3{fill:url(#streaks-normal);mix-blend-mode:color-burn}.sketch-overlay-B4{fill:url(#streaks-normal);mix-blend-mode:color-burn}.sketch-overlay-B5{fill:url(#streaks-bright);mix-blend-mode:darken}.sketch-overlay-B6{fill:url(#streaks-bright);mix-blend-mode:darken}.sketch-overlay-AA2{fill:url(#streaks-dark);mix-blend-mode:overlay}.sketch-overlay-AA4{fill:url(#streaks-normal);mix-blend-mode:color-burn}.sketch-overlay-AA5{fill:url(#streaks-bright);mix-blend-mode:darken}.sketch-overlay-AB4{fill:url(#streaks-normal);mix-blend-mode:color-burn}.sketch-overlay-AB5{fill:url(#streaks-bright);mix-blend-mode:darken}.sketch-overlay-N1{fill:url(#streaks-darker);mix-blend-mode:lighten}.sketch-overlay-N2{fill:url(#streaks-dark);mix-blend-mode:overlay}.sketch-overlay-N3{fill:url(#streaks-normal);mix-blend-mode:color-burn}.sketch-overlay-N4{fill:url(#streaks-normal);mix-blend-mode:color-burn}.sketch-overlay-N5{fill:url(#streaks-bright);mix-blend-mode:darken}.sketch-overlay-N6{fill:url(#streaks-bright);mix-blend-mode:darken}.sketch-overlay-N7{fill:url(#streaks-bright);mix-blend-mode:darken}.light-code{display: block}.dark-code{display: none}]]>Create WorkspaceCodeEdit DockerfileRestart Workspace + + + + + diff --git a/scripts/diagram.sh b/scripts/diagram.sh index b6fe5da2..a4c0f1f2 100755 --- a/scripts/diagram.sh +++ b/scripts/diagram.sh @@ -3,5 +3,8 @@ cd "$(dirname "${BASH_SOURCE[0]}")" set -euxo pipefail -d2 ./diagram.d2 --pad=32 -t 1 ./diagram-light.svg -d2 ./diagram.d2 --pad=32 -t 200 ./diagram-dark.svg \ No newline at end of file +formats=( svg png ) +for format in "${formats[@]}"; do + d2 ./diagram.d2 --pad=32 -t 1 "./diagram-light.${format}" + d2 ./diagram.d2 --pad=32 -t 200 "./diagram-dark.${format}" +done From ea779ad2158f8e220a9a609311b5f8925d84ccc5 Mon Sep 17 00:00:00 2001 From: Bruno Quaresma Date: Thu, 26 Sep 2024 14:38:50 -0300 Subject: [PATCH 61/81] docs: improve readme (#357) Co-authored-by: Cian Johnston Co-authored-by: Muhammad Atif Ali (cherry picked from commit 63a383bd696534f12d219b839a85b1bd6af3c5e8) --- .github/workflows/ci.yaml | 4 +- Makefile | 2 +- README.md | 432 +++++--------------------------- docs/caching.md | 65 +++++ docs/container-registry-auth.md | 77 ++++++ docs/custom-certificates.md | 5 + docs/env-variables.md | 42 ++++ docs/git-auth.md | 66 +++++ docs/images/dark-logo.svg | 1 + docs/images/light-logo.svg | 1 + docs/usage-with-coder.md | 27 ++ docs/using-local-files.md | 34 +++ scripts/docsgen/main.go | 29 +-- 13 files changed, 384 insertions(+), 401 deletions(-) create mode 100644 docs/caching.md create mode 100644 docs/container-registry-auth.md create mode 100644 docs/custom-certificates.md create mode 100644 docs/env-variables.md create mode 100644 docs/git-auth.md create mode 100644 docs/images/dark-logo.svg create mode 100644 docs/images/light-logo.svg create mode 100644 docs/usage-with-coder.md create mode 100644 docs/using-local-files.md diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 457b3117..b40f1cf3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -57,8 +57,8 @@ jobs: with: go-version: "~1.22" - - name: Generate docs - run: make docs + - name: Generate env vars docs + run: make docs/env-variables.md - name: Check for unstaged files run: git diff --exit-code diff --git a/Makefile b/Makefile index 8bd3f6b5..ca4c0e6d 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ update-golden-files: .gen-golden go test ./options -update @touch "$@" -docs: options/options.go options/options_test.go +docs/env-variables.md: options/options.go options/options_test.go go run ./scripts/docsgen/main.go .PHONY: test diff --git a/README.md b/README.md index 28bfe098..981208c9 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,20 @@ -# envbuilder +

+ + + + + + +

+ +

+ + + + +

-[![discord](https://img.shields.io/discord/747933592273027093?label=discord)](https://discord.gg/coder) -[![release](https://img.shields.io/github/v/tag/coder/envbuilder)](https://github.com/coder/envbuilder/pkgs/container/envbuilder) -[![godoc](https://pkg.go.dev/badge/github.com/coder/envbuilder.svg)](https://pkg.go.dev/github.com/coder/envbuilder) -[![license](https://img.shields.io/github/license/coder/envbuilder)](./LICENSE) +# Envbuilder Build development environments from a Dockerfile on Docker, Kubernetes, and OpenShift. Allow developers to modify their environment in a tight feedback loop. @@ -11,26 +22,17 @@ Build development environments from a Dockerfile on Docker, Kubernetes, and Open - Cache image layers with registries for speedy builds - Runs on Kubernetes, Docker, and OpenShift - - -## Quickstart +## Getting Started -The easiest way to get started is to run the `envbuilder` Docker container that clones a repository, builds the image from a Dockerfile, and runs the `$ENVBUILDER_INIT_SCRIPT` in the freshly built container. +The easiest way to get started is by running the `envbuilder` Docker container that clones a repository, builds the image from a Dockerfile, and runs the `$ENVBUILDER_INIT_SCRIPT` in the freshly built container. -> `/tmp/envbuilder` directory persists demo data between commands. You can choose a different directory. +> **Note**: The `/tmp/envbuilder` directory persists demo data between commands. You can choose a different directory if needed. ```bash -docker run -it --rm \ - -v /tmp/envbuilder:/workspaces \ - -e ENVBUILDER_GIT_URL=https://github.com/coder/envbuilder-starter-devcontainer \ - -e ENVBUILDER_INIT_SCRIPT=bash \ +docker run -it --rm + -v /tmp/envbuilder:/workspaces + -e ENVBUILDER_GIT_URL=https://github.com/coder/envbuilder-starter-devcontainer + -e ENVBUILDER_INIT_SCRIPT=bash ghcr.io/coder/envbuilder ``` @@ -45,303 +47,31 @@ vim .devcontainer/Dockerfile + RUN apt-get install vim sudo htop -y ``` -Exit the container, and re-run the `docker run` command... after the build completes, `htop` should exist in the container! 🥳 - -> [!NOTE] -> Envbuilder performs destructive filesystem operations! To guard against accidental data -> loss, it will refuse to run if it detects that KANIKO_DIR is not set to a specific value. -> If you need to bypass this behavior for any reason, you can bypass this safety check by setting -> `ENVBUILDER_FORCE_SAFE=true`. - -If you don't have a remote Git repo or you want to quickly iterate with some -local files, simply omit `ENVBUILDER_GIT_URL` and instead mount the directory -containing your code to `/workspaces/empty` inside the Envbuilder container. - -For example: - -```shell -# Create a sample Devcontainer and Dockerfile in the current directory -printf '{"build": { "dockerfile": "Dockerfile"}}' > devcontainer.json -printf 'FROM debian:bookworm\nRUN apt-get update && apt-get install -y cowsay' > Dockerfile - -# Run envbuilder with the current directory mounted into `/workspaces/empty`. -# The instructions to add /usr/games to $PATH have been omitted for brevity. -docker run -it --rm -e ENVBUILDER_INIT_SCRIPT='bash' -v $PWD:/workspaces/empty ghcr.io/coder/envbuilder:latest -``` - -Alternatively, if you prefer to mount your project files elsewhere, tell -Envbuilder where to find them by specifying `ENVBUILDER_WORKSPACE_FOLDER`: - -```shell -docker run -it --rm -e ENVBUILDER_INIT_SCRIPT='bash ' -e ENVBUILDER_WORKSPACE_FOLDER=/src -v $PWD:/src ghcr.io/coder/envbuilder:latest -``` - -By default, Envbuilder will look for a `devcontainer.json` or `Dockerfile` in -both `${ENVBUILDER_WORKSPACE_FOLDER}` and `${ENVBUILDER_WORKSPACE_FOLDER}/.devcontainer`. -You can control where it looks with `ENVBUILDER_DEVCONTAINER_DIR` if needed. - -```shell -ls build/ -Dockerfile devcontainer.json -docker run -it --rm -e ENVBUILDER_INIT_SCRIPT='bash' -e ENVBUILDER_DEVCONTAINER_DIR=build -v $PWD:/src ghcr.io/coder/envbuilder:latest -``` - -## Usage with Coder - -Coder provides sample -[Docker](https://github.com/coder/coder/tree/main/examples/templates/devcontainer-docker) -and -[Kubernetes](https://github.com/coder/coder/tree/main/examples/templates/devcontainer-kubernetes) -templates for use with Envbuilder. You can import these templates and modify them to fit -your specific requirements. - -Below are some specific points to be aware of when using Envbuilder with a Coder -deployment: - -- The `ENVBUILDER_INIT_SCRIPT` should execute `coder_agent.main.init_script` in - order for you to be able to connect to your workspace. -- In order for the Agent init script to be able to fetch the agent binary from - your Coder deployment, the resulting Devcontainer must contain a download tool - such as `curl`, `wget`, or `busybox`. -- `CODER_AGENT_TOKEN` should be included in the environment variables for the - Envbuilder container. You can also set `CODER_AGENT_URL` if required. - - -### Git Branch Selection - -Choose a branch using `ENVBUILDER_GIT_URL` with a _ref/heads_ reference. For instance: - -``` -ENVBUILDER_GIT_URL=https://github.com/coder/envbuilder-starter-devcontainer/#refs/heads/my-feature-branch -``` - -## Container Registry Authentication - -envbuilder uses Kaniko to build containers. You should [follow their instructions](https://github.com/GoogleContainerTools/kaniko#pushing-to-different-registries) to create an authentication configuration. - -After you have a configuration that resembles the following: - -```json -{ - "auths": { - "https://index.docker.io/v1/": { - "auth": "base64-encoded-username-and-password" - } - } -} -``` - -`base64` encode the JSON and provide it to envbuilder as the `ENVBUILDER_DOCKER_CONFIG_BASE64` environment variable. - -Alternatively, if running `envbuilder` in Kubernetes, you can create an `ImagePullSecret` and -pass it into the pod as a volume mount. This example will work for all registries. - -```shell -# Artifactory example -kubectl create secret docker-registry regcred \ - --docker-server=my-artifactory.jfrog.io \ - --docker-username=read-only \ - --docker-password=secret-pass \ - --docker-email=me@example.com \ - -n coder -``` - -```hcl -resource "kubernetes_deployment" "example" { - metadata { - namespace = coder - } - spec { - spec { - container { - # Define the volumeMount with the pull credentials - volume_mount { - name = "docker-config-volume" - mount_path = "/.envbuilder/config.json" - sub_path = ".dockerconfigjson" - } - } - # Define the volume which maps to the pull credentials - volume { - name = "docker-config-volume" - secret { - secret_name = "regcred" - } - } - } - } -} -``` - -### Docker Hub - -Authenticate with `docker login` to generate `~/.docker/config.json`. Encode this file using the `base64` command: - -```bash -$ base64 -w0 ~/.docker/config.json -ewoJImF1dGhzIjogewoJCSJodHRwczovL2luZGV4LmRvY2tlci5pby92MS8iOiB7CgkJCSJhdXRoIjogImJhc2U2NCBlbmNvZGVkIHRva2VuIgoJCX0KCX0KfQo= -``` - -Provide the encoded JSON config to envbuilder: - -```env -ENVBUILDER_DOCKER_CONFIG_BASE64=ewoJImF1dGhzIjogewoJCSJodHRwczovL2luZGV4LmRvY2tlci5pby92MS8iOiB7CgkJCSJhdXRoIjogImJhc2U2NCBlbmNvZGVkIHRva2VuIgoJCX0KCX0KfQo= -``` - -### Docker-in-Docker +Exit the container and re-run the `docker run` command. After the build completes, `htop` should be available in the container! 🥳 -See [here](./docs/docker.md) for instructions on running Docker containers inside -environments built by Envbuilder. +To explore more examples, tips, and advanced usage, check out the following guides: -## Git Authentication - -Two methods of authentication are supported: - -### HTTP Authentication - -If `ENVBUILDER_GIT_URL` starts with `http://` or `https://`, envbuilder will -authenticate with `ENVBUILDER_GIT_USERNAME` and `ENVBUILDER_GIT_PASSWORD`, if set. - -For access token-based authentication, follow the following schema (if empty, there's no need to provide the field): - -| Provider | `ENVBUILDER_GIT_USERNAME` | `ENVBUILDER_GIT_PASSWORD` | -| ------------ | ------------------------- | ------------------------- | -| GitHub | [access-token] | | -| GitLab | oauth2 | [access-token] | -| BitBucket | x-token-auth | [access-token] | -| Azure DevOps | [access-token] | | - -If using envbuilder inside of [Coder](https://github.com/coder/coder), you can use the `coder_external_auth` Terraform resource to automatically provide this token on workspace creation: - -```hcl -data "coder_external_auth" "github" { - id = "github" -} - -resource "docker_container" "dev" { - env = [ - ENVBUILDER_GIT_USERNAME = data.coder_external_auth.github.access_token, - ] -} -``` - -### SSH Authentication - -If `ENVBUILDER_GIT_URL` does not start with `http://` or `https://`, -envbuilder will assume SSH authentication. You have the following options: - -1. Public/Private key authentication: set `ENVBUILDER_GIT_SSH_PRIVATE_KEY_PATH` to the path of an - SSH private key mounted inside the container. Envbuilder will use this SSH - key to authenticate. Example: - - ```bash - docker run -it --rm \ - -v /tmp/envbuilder:/workspaces \ - -e ENVBUILDER_GIT_URL=git@example.com:path/to/private/repo.git \ - -e ENVBUILDER_INIT_SCRIPT=bash \ - -e ENVBUILDER_GIT_SSH_PRIVATE_KEY_PATH=/.ssh/id_rsa \ - -v /home/user/id_rsa:/.ssh/id_rsa \ - ghcr.io/coder/envbuilder - ``` - -1. Agent-based authentication: set `SSH_AUTH_SOCK` and mount in your agent socket, for example: - - ```bash - docker run -it --rm \ - -v /tmp/envbuilder:/workspaces \ - -e ENVBUILDER_GIT_URL=git@example.com:path/to/private/repo.git \ - -e ENVBUILDER_INIT_SCRIPT=bash \ - -e SSH_AUTH_SOCK=/tmp/ssh-auth-sock \ - -v $SSH_AUTH_SOCK:/tmp/ssh-auth-sock \ - ghcr.io/coder/envbuilder - ``` - -> Note: by default, envbuilder will accept and log all host keys. If you need -> strict host key checking, set `SSH_KNOWN_HOSTS` and mount in a `known_hosts` -> file. - - -## Layer Caching - -Cache layers in a container registry to speed up builds. To enable caching, [authenticate with your registry](#container-registry-authentication) and set the `ENVBUILDER_CACHE_REPO` environment variable. - -```bash -CACHE_REPO=ghcr.io/coder/repo-cache -``` - -To experiment without setting up a registry, use `ENVBUILDER_LAYER_CACHE_DIR`: - -```bash -docker run -it --rm \ - -v /tmp/envbuilder-cache:/cache \ - -e ENVBUILDER_LAYER_CACHE_DIR=/cache - ... -``` - -Each layer is stored in the registry as a separate image. The image tag is the hash of the layer's contents. The image digest is the hash of the image tag. The image digest is used to pull the layer from the registry. - -The performance improvement of builds depends on the complexity of your -Dockerfile. For -[`coder/coder`](https://github.com/coder/coder/blob/main/.devcontainer/Dockerfile), -uncached builds take 36m while cached builds take 40s (~98% improvement). - -## Pushing the built image - -Set `ENVBUILDER_PUSH_IMAGE=1` to push the entire image to the cache repo -in addition to individual layers. `ENVBUILDER_CACHE_REPO` **must** be set in -order for this to work. - -> **Note:** this option forces Envbuilder to perform a "reproducible" build. -> This will force timestamps for all newly added files to be set to the start of the UNIX epoch. - -## Probe Layer Cache - -To check for the presence of a pre-built image, set -`ENVBUILDER_GET_CACHED_IMAGE=1`. Instead of building the image, this will -perform a "dry-run" build of the image, consulting `ENVBUILDER_CACHE_REPO` for -each layer. - -If any layer is found not to be present in the cache repo, envbuilder -will exit with an error. Otherwise, the image will be emitted in the log output prefixed with the string -`ENVBUILDER_CACHED_IMAGE=...`. - -## Image Caching - -When the base container is large, it can take a long time to pull the image from the registry. You can pre-pull the image into a read-only volume and mount it into the container to speed up builds. - -```bash -# Pull your base image from the registry to a local directory. -docker run --rm \ - -v /tmp/kaniko-cache:/cache \ - gcr.io/kaniko-project/warmer:latest \ - --cache-dir=/cache \ - --image= - -# Run envbuilder with the local image cache. -docker run -it --rm \ - -v /tmp/kaniko-cache:/image-cache:ro \ - -e ENVBUILDER_BASE_IMAGE_CACHE_DIR=/image-cache -``` - -In Kubernetes, you can pre-populate a persistent volume with the same warmer image, then mount it into many workspaces with the [`ReadOnlyMany` access mode](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes). - -A sample script to pre-fetch a number of images can be viewed [here](./examples/kaniko-cache-warmer.sh). This can be run, for example, as a cron job to periodically fetch the latest versions of a number of base images. +- [Using Local Files](./docs/using-local-files.md) +- [Usage with Coder](./docs/usage-with-coder.md) +- [Container Registry Authentication](./docs/container-registry-auth.md) +- [Git Authentication](./docs/git-auth.md) +- [Caching](./docs/caching.md) +- [Custom Certificates](./docs/custom-certificates.md) ## Setup Script The `ENVBUILDER_SETUP_SCRIPT` environment variable dynamically configures the user and init command (PID 1) after the container build process. -> [!NOTE] -> `TARGET_USER` is passed to the setup script to specify who will execute `ENVBUILDER_INIT_COMMAND` (e.g., `code`). +> **Note**: `TARGET_USER` is passed to the setup script to specify who will execute `ENVBUILDER_INIT_COMMAND` (e.g., `code`). Write the following to `$ENVBUILDER_ENV` to shape the container's init process: -- `TARGET_USER`: Identifies the `ENVBUILDER_INIT_COMMAND` executor (e.g.`root`). +- `TARGET_USER`: Identifies the `ENVBUILDER_INIT_COMMAND` executor (e.g., `root`). - `ENVBUILDER_INIT_COMMAND`: Defines the command executed by `TARGET_USER` (e.g. `/bin/bash`). -- `ENVBUILDER_INIT_ARGS`: Arguments provided to `ENVBUILDER_INIT_COMMAND` (e.g. `-c 'sleep infinity'`). +- `ENVBUILDER_INIT_ARGS`: Arguments provided to `ENVBUILDER_INIT_COMMAND` (e.g., `-c 'sleep infinity'`). ```bash -# init.sh - change the init if systemd exists +# init.sh - Change the init if systemd exists if command -v systemd >/dev/null; then echo "Hey 👋 $TARGET_USER" echo ENVBUILDER_INIT_COMMAND=systemd >> $ENVBUILDER_ENV @@ -349,44 +79,42 @@ else echo ENVBUILDER_INIT_COMMAND=bash >> $ENVBUILDER_ENV fi -# run envbuilder with the setup script -docker run -it --rm \ - -v ./:/some-dir \ - -e ENVBUILDER_SETUP_SCRIPT=/some-dir/init.sh \ +# Run envbuilder with the setup script +docker run -it --rm + -v ./:/some-dir + -e ENVBUILDER_SETUP_SCRIPT=/some-dir/init.sh ... ``` -## Custom Certificates +## Environment Variables -- [`ENVBUILDER_SSL_CERT_FILE`](https://go.dev/src/crypto/x509/root_unix.go#L19): Specifies the path to an SSL certificate. -- [`ENVBUILDER_SSL_CERT_DIR`](https://go.dev/src/crypto/x509/root_unix.go#L25): Identifies which directory to check for SSL certificate files. -- `ENVBUILDER_SSL_CERT_BASE64`: Specifies a base64-encoded SSL certificate that will be added to the global certificate pool on start. +You can see all the supported environment variables in [this document](./docs/env-variables.md). -## Unsupported features +## Unsupported Features ### Development Containers -The table keeps track of features we would love to implement. Feel free to [create a new issue](https://github.com/coder/envbuilder/issues/new) if you want Envbuilder to support it. +The table below keeps track of features we plan to implement. Feel free to [create a new issue](https://github.com/coder/envbuilder/issues/new) if you'd like Envbuilder to support a particular feature. -| Name | Description | Known issues | -| ------------------------ | ------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------ | -| Volume mounts | Volumes are used to persist data and share directories between the host and container. | [#220](https://github.com/coder/envbuilder/issues/220) | -| Port forwarding | Port forwarding allows exposing container ports to the host, making services accessible. | [#48](https://github.com/coder/envbuilder/issues/48) | -| Script init & Entrypoint | `init` adds a tiny init process to the container and `entrypoint` sets a script to run at container startup. | [#221](https://github.com/coder/envbuilder/issues/221) | -| Customizations | Product specific properties, for instance: _VS Code_ `settings` and `extensions`. | [#43](https://github.com/coder/envbuilder/issues/43) | -| Composefile | Define multiple containers and services for more complex development environments. | [#236](https://github.com/coder/envbuilder/issues/236) | +| Name | Description | Known Issues | +| ------------------------ | ------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------ | +| Volume mounts | Volumes are used to persist data and share directories between the host and container. | [#220](https://github.com/coder/envbuilder/issues/220) | +| Port forwarding | Port forwarding allows exposing container ports to the host, making services accessible. | [#48](https://github.com/coder/envbuilder/issues/48) | +| Script init & Entrypoint | `init` adds a tiny init process to the container, and `entrypoint` sets a script to run at container startup. | [#221](https://github.com/coder/envbuilder/issues/221) | +| Customizations | Product-specific properties, e.g., _VS Code_ settings and extensions. | [#43](https://github.com/coder/envbuilder/issues/43) | +| Composefile | Define multiple containers and services for more complex development environments. | [#236](https://github.com/coder/envbuilder/issues/236) | ### Devfile -> [Devfiles](https://devfile.io/) automate and simplify development process by adopting the existing devfiles that are available in the [public community registry](https://registry.devfile.io/viewer). +> [Devfiles](https://devfile.io/) automate and simplify development by adopting existing devfiles available in the [public community registry](https://registry.devfile.io/viewer). Issue: [#113](https://github.com/coder/envbuilder/issues/113) -# Local Development +## Contributing Building `envbuilder` currently **requires** a Linux system. -On MacOS or Windows systems, we recommend either using a VM or the provided `.devcontainer` for development. +On macOS or Windows systems, we recommend using a VM or the provided `.devcontainer` for development. **Additional Requirements:** @@ -396,52 +124,8 @@ On MacOS or Windows systems, we recommend either using a VM or the provided `.de **Makefile targets:** -- `build`: builds and tags `envbuilder:latest` for your current architecture. -- `develop`: runs `envbuilder:latest` against a sample Git repository. -- `test`: run tests. -- `test-registry`: stands up a local registry for caching images used in tests. - - - -## Environment Variables - -| Flag | Environment variable | Default | Description | -| - | - | - | - | -| `--setup-script` | `ENVBUILDER_SETUP_SCRIPT` | | The script to run before the init script. It runs as the root user regardless of the user specified in the devcontainer.json file. SetupScript is ran as the root user prior to the init script. It is used to configure envbuilder dynamically during the runtime. e.g. specifying whether to start systemd or tiny init for PID 1. | -| `--init-script` | `ENVBUILDER_INIT_SCRIPT` | | The script to run to initialize the workspace. Default: `sleep infinity`. | -| `--init-command` | `ENVBUILDER_INIT_COMMAND` | | The command to run to initialize the workspace. Default: `/bin/sh`. | -| `--init-args` | `ENVBUILDER_INIT_ARGS` | | The arguments to pass to the init command. They are split according to /bin/sh rules with https://github.com/kballard/go-shellquote. | -| `--cache-repo` | `ENVBUILDER_CACHE_REPO` | | The name of the container registry to push the cache image to. If this is empty, the cache will not be pushed. | -| `--base-image-cache-dir` | `ENVBUILDER_BASE_IMAGE_CACHE_DIR` | | The path to a directory where the base image can be found. This should be a read-only directory solely mounted for the purpose of caching the base image. | -| `--layer-cache-dir` | `ENVBUILDER_LAYER_CACHE_DIR` | | The path to a directory where built layers will be stored. This spawns an in-memory registry to serve the layers from. | -| `--devcontainer-dir` | `ENVBUILDER_DEVCONTAINER_DIR` | | The path to the folder containing the devcontainer.json file that will be used to build the workspace and can either be an absolute path or a path relative to the workspace folder. If not provided, defaults to `.devcontainer`. | -| `--devcontainer-json-path` | `ENVBUILDER_DEVCONTAINER_JSON_PATH` | | The path to a devcontainer.json file that is either an absolute path or a path relative to DevcontainerDir. This can be used in cases where one wants to substitute an edited devcontainer.json file for the one that exists in the repo. | -| `--dockerfile-path` | `ENVBUILDER_DOCKERFILE_PATH` | | The relative path to the Dockerfile that will be used to build the workspace. This is an alternative to using a devcontainer that some might find simpler. | -| `--build-context-path` | `ENVBUILDER_BUILD_CONTEXT_PATH` | | Can be specified when a DockerfilePath is specified outside the base WorkspaceFolder. This path MUST be relative to the WorkspaceFolder path into which the repo is cloned. | -| `--cache-ttl-days` | `ENVBUILDER_CACHE_TTL_DAYS` | | The number of days to use cached layers before expiring them. Defaults to 7 days. | -| `--docker-config-base64` | `ENVBUILDER_DOCKER_CONFIG_BASE64` | | The base64 encoded Docker config file that will be used to pull images from private container registries. | -| `--fallback-image` | `ENVBUILDER_FALLBACK_IMAGE` | | Specifies an alternative image to use when neither an image is declared in the devcontainer.json file nor a Dockerfile is present. If there's a build failure (from a faulty Dockerfile) or a misconfiguration, this image will be the substitute. Set ExitOnBuildFailure to true to halt the container if the build faces an issue. | -| `--exit-on-build-failure` | `ENVBUILDER_EXIT_ON_BUILD_FAILURE` | | Terminates the container upon a build failure. This is handy when preferring the FALLBACK_IMAGE in cases where no devcontainer.json or image is provided. However, it ensures that the container stops if the build process encounters an error. | -| `--force-safe` | `ENVBUILDER_FORCE_SAFE` | | Ignores any filesystem safety checks. This could cause serious harm to your system! This is used in cases where bypass is needed to unblock customers. | -| `--insecure` | `ENVBUILDER_INSECURE` | | Bypass TLS verification when cloning and pulling from container registries. | -| `--ignore-paths` | `ENVBUILDER_IGNORE_PATHS` | | The comma separated list of paths to ignore when building the workspace. | -| `--skip-rebuild` | `ENVBUILDER_SKIP_REBUILD` | | Skip building if the MagicFile exists. This is used to skip building when a container is restarting. e.g. docker stop -> docker start This value can always be set to true - even if the container is being started for the first time. | -| `--git-url` | `ENVBUILDER_GIT_URL` | | The URL of a Git repository containing a Devcontainer or Docker image to clone. This is optional. | -| `--git-clone-depth` | `ENVBUILDER_GIT_CLONE_DEPTH` | | The depth to use when cloning the Git repository. | -| `--git-clone-single-branch` | `ENVBUILDER_GIT_CLONE_SINGLE_BRANCH` | | Clone only a single branch of the Git repository. | -| `--git-username` | `ENVBUILDER_GIT_USERNAME` | | The username to use for Git authentication. This is optional. | -| `--git-password` | `ENVBUILDER_GIT_PASSWORD` | | The password to use for Git authentication. This is optional. | -| `--git-ssh-private-key-path` | `ENVBUILDER_GIT_SSH_PRIVATE_KEY_PATH` | | Path to an SSH private key to be used for Git authentication. | -| `--git-http-proxy-url` | `ENVBUILDER_GIT_HTTP_PROXY_URL` | | The URL for the HTTP proxy. This is optional. | -| `--workspace-folder` | `ENVBUILDER_WORKSPACE_FOLDER` | | The path to the workspace folder that will be built. This is optional. | -| `--ssl-cert-base64` | `ENVBUILDER_SSL_CERT_BASE64` | | The content of an SSL cert file. This is useful for self-signed certificates. | -| `--export-env-file` | `ENVBUILDER_EXPORT_ENV_FILE` | | Optional file path to a .env file where envbuilder will dump environment variables from devcontainer.json and the built container image. | -| `--post-start-script-path` | `ENVBUILDER_POST_START_SCRIPT_PATH` | | The path to a script that will be created by envbuilder based on the postStartCommand in devcontainer.json, if any is specified (otherwise the script is not created). If this is set, the specified InitCommand should check for the presence of this script and execute it after successful startup. | -| `--coder-agent-url` | `CODER_AGENT_URL` | | URL of the Coder deployment. If CODER_AGENT_TOKEN is also set, logs from envbuilder will be forwarded here and will be visible in the workspace build logs. | -| `--coder-agent-token` | `CODER_AGENT_TOKEN` | | Authentication token for a Coder agent. If this is set, then CODER_AGENT_URL must also be set. | -| `--coder-agent-subsystem` | `CODER_AGENT_SUBSYSTEM` | | Coder agent subsystems to report when forwarding logs. The envbuilder subsystem is always included. | -| `--push-image` | `ENVBUILDER_PUSH_IMAGE` | | Push the built image to a remote registry. This option forces a reproducible build. | -| `--get-cached-image` | `ENVBUILDER_GET_CACHED_IMAGE` | | Print the digest of the cached image, if available. Exits with an error if not found. | -| `--remote-repo-build-mode` | `ENVBUILDER_REMOTE_REPO_BUILD_MODE` | `false` | Use the remote repository as the source of truth when building the image. Enabling this option ignores user changes to local files and they will not be reflected in the image. This can be used to improving cache utilization when multiple users are building working on the same repository. | -| `--verbose` | `ENVBUILDER_VERBOSE` | | Enable verbose logging. | - +- `build`: Builds and tags `envbuilder:latest` for your current architecture. +- `develop`: Runs `envbuilder:latest` against a sample Git repository. +- `test`: Runs tests. +- `test-registry`: Stands up a local registry for caching images used in tests. +- `docs/env-variables.md`: Updated the [environment variables documentation](./docs/env-variables.md). diff --git a/docs/caching.md b/docs/caching.md new file mode 100644 index 00000000..5963083e --- /dev/null +++ b/docs/caching.md @@ -0,0 +1,65 @@ +# Layer Caching + +Cache layers in a container registry to speed up builds. To enable caching, [authenticate with your registry](#container-registry-authentication) and set the `ENVBUILDER_CACHE_REPO` environment variable. + +```bash +ENVBUILDER_CACHE_REPO=ghcr.io/coder/repo-cache +``` + +To experiment without setting up a registry, use `ENVBUILDER_LAYER_CACHE_DIR`: + +```bash +docker run -it --rm \ + -v /tmp/envbuilder-cache:/cache \ + -e ENVBUILDER_LAYER_CACHE_DIR=/cache + ... +``` + +Each layer is stored in the registry as a separate image. The image tag is the hash of the layer's contents. The image digest is the hash of the image tag. The image digest is used to pull the layer from the registry. + +The performance improvement of builds depends on the complexity of your +Dockerfile. For +[`coder/coder`](https://github.com/coder/coder/blob/main/dogfood/contents/Dockerfile), +uncached builds take 36m while cached builds take 40s (~98% improvement). + +# Pushing the built image + +Set `ENVBUILDER_PUSH_IMAGE=1` to push the entire image to the cache repo +in addition to individual layers. `ENVBUILDER_CACHE_REPO` **must** be set in +order for this to work. + +> **Note:** this option forces Envbuilder to perform a "reproducible" build. +> This will force timestamps for all newly added files to be set to the start of the UNIX epoch. + +# Probe Layer Cache + +To check for the presence of a pre-built image, set +`ENVBUILDER_GET_CACHED_IMAGE=1`. Instead of building the image, this will +perform a "dry-run" build of the image, consulting `ENVBUILDER_CACHE_REPO` for +each layer. + +If any layer is found not to be present in the cache repo, envbuilder +will exit with an error. Otherwise, the image will be emitted in the log output prefixed with the string +`ENVBUILDER_CACHED_IMAGE=...`. + +# Image Caching + +When the base container is large, it can take a long time to pull the image from the registry. You can pre-pull the image into a read-only volume and mount it into the container to speed up builds. + +```bash +# Pull your base image from the registry to a local directory. +docker run --rm \ + -v /tmp/kaniko-cache:/cache \ + gcr.io/kaniko-project/warmer:latest \ + --cache-dir=/cache \ + --image= + +# Run envbuilder with the local image cache. +docker run -it --rm \ + -v /tmp/kaniko-cache:/image-cache:ro \ + -e ENVBUILDER_BASE_IMAGE_CACHE_DIR=/image-cache +``` + +In Kubernetes, you can pre-populate a persistent volume with the same warmer image, then mount it into many workspaces with the [`ReadOnlyMany` access mode](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes). + +A sample script to pre-fetch a number of images can be viewed [here](./examples/kaniko-cache-warmer.sh). This can be run, for example, as a cron job to periodically fetch the latest versions of a number of base images. diff --git a/docs/container-registry-auth.md b/docs/container-registry-auth.md new file mode 100644 index 00000000..e0d7663e --- /dev/null +++ b/docs/container-registry-auth.md @@ -0,0 +1,77 @@ +# Container Registry Authentication + +envbuilder uses Kaniko to build containers. You should [follow their instructions](https://github.com/GoogleContainerTools/kaniko#pushing-to-different-registries) to create an authentication configuration. + +After you have a configuration that resembles the following: + +```json +{ + "auths": { + "https://index.docker.io/v1/": { + "auth": "base64-encoded-username-and-password" + } + } +} +``` + +`base64` encode the JSON and provide it to envbuilder as the `ENVBUILDER_DOCKER_CONFIG_BASE64` environment variable. + +Alternatively, if running `envbuilder` in Kubernetes, you can create an `ImagePullSecret` and +pass it into the pod as a volume mount. This example will work for all registries. + +```shell +# Artifactory example +kubectl create secret docker-registry regcred \ + --docker-server=my-artifactory.jfrog.io \ + --docker-username=read-only \ + --docker-password=secret-pass \ + --docker-email=me@example.com \ + -n coder +``` + +```hcl +resource "kubernetes_deployment" "example" { + metadata { + namespace = coder + } + spec { + spec { + container { + # Define the volumeMount with the pull credentials + volume_mount { + name = "docker-config-volume" + mount_path = "/.envbuilder/config.json" + sub_path = ".dockerconfigjson" + } + } + # Define the volume which maps to the pull credentials + volume { + name = "docker-config-volume" + secret { + secret_name = "regcred" + } + } + } + } +} +``` + +## Docker Hub + +Authenticate with `docker login` to generate `~/.docker/config.json`. Encode this file using the `base64` command: + +```bash +$ base64 -w0 ~/.docker/config.json +ewoJImF1dGhzIjogewoJCSJodHRwczovL2luZGV4LmRvY2tlci5pby92MS8iOiB7CgkJCSJhdXRoIjogImJhc2U2NCBlbmNvZGVkIHRva2VuIgoJCX0KCX0KfQo= +``` + +Provide the encoded JSON config to envbuilder: + +```env +ENVBUILDER_DOCKER_CONFIG_BASE64=ewoJImF1dGhzIjogewoJCSJodHRwczovL2luZGV4LmRvY2tlci5pby92MS8iOiB7CgkJCSJhdXRoIjogImJhc2U2NCBlbmNvZGVkIHRva2VuIgoJCX0KCX0KfQo= +``` + +## Docker-in-Docker + +See [here](./docs/docker.md) for instructions on running Docker containers inside +environments built by Envbuilder. diff --git a/docs/custom-certificates.md b/docs/custom-certificates.md new file mode 100644 index 00000000..dd33192f --- /dev/null +++ b/docs/custom-certificates.md @@ -0,0 +1,5 @@ +# Custom Certificates + +- [`ENVBUILDER_SSL_CERT_FILE`](https://go.dev/src/crypto/x509/root_unix.go#L19): Specifies the path to an SSL certificate. +- [`ENVBUILDER_SSL_CERT_DIR`](https://go.dev/src/crypto/x509/root_unix.go#L25): Identifies which directory to check for SSL certificate files. +- `ENVBUILDER_SSL_CERT_BASE64`: Specifies a base64-encoded SSL certificate that will be added to the global certificate pool on start. diff --git a/docs/env-variables.md b/docs/env-variables.md new file mode 100644 index 00000000..1c80f4fc --- /dev/null +++ b/docs/env-variables.md @@ -0,0 +1,42 @@ + +# Environment Variables + +| Flag | Environment variable | Default | Description | +| - | - | - | - | +| `--setup-script` | `ENVBUILDER_SETUP_SCRIPT` | | The script to run before the init script. It runs as the root user regardless of the user specified in the devcontainer.json file. SetupScript is ran as the root user prior to the init script. It is used to configure envbuilder dynamically during the runtime. e.g. specifying whether to start systemd or tiny init for PID 1. | +| `--init-script` | `ENVBUILDER_INIT_SCRIPT` | | The script to run to initialize the workspace. Default: `sleep infinity`. | +| `--init-command` | `ENVBUILDER_INIT_COMMAND` | | The command to run to initialize the workspace. Default: `/bin/sh`. | +| `--init-args` | `ENVBUILDER_INIT_ARGS` | | The arguments to pass to the init command. They are split according to /bin/sh rules with https://github.com/kballard/go-shellquote. | +| `--cache-repo` | `ENVBUILDER_CACHE_REPO` | | The name of the container registry to push the cache image to. If this is empty, the cache will not be pushed. | +| `--base-image-cache-dir` | `ENVBUILDER_BASE_IMAGE_CACHE_DIR` | | The path to a directory where the base image can be found. This should be a read-only directory solely mounted for the purpose of caching the base image. | +| `--layer-cache-dir` | `ENVBUILDER_LAYER_CACHE_DIR` | | The path to a directory where built layers will be stored. This spawns an in-memory registry to serve the layers from. | +| `--devcontainer-dir` | `ENVBUILDER_DEVCONTAINER_DIR` | | The path to the folder containing the devcontainer.json file that will be used to build the workspace and can either be an absolute path or a path relative to the workspace folder. If not provided, defaults to `.devcontainer`. | +| `--devcontainer-json-path` | `ENVBUILDER_DEVCONTAINER_JSON_PATH` | | The path to a devcontainer.json file that is either an absolute path or a path relative to DevcontainerDir. This can be used in cases where one wants to substitute an edited devcontainer.json file for the one that exists in the repo. | +| `--dockerfile-path` | `ENVBUILDER_DOCKERFILE_PATH` | | The relative path to the Dockerfile that will be used to build the workspace. This is an alternative to using a devcontainer that some might find simpler. | +| `--build-context-path` | `ENVBUILDER_BUILD_CONTEXT_PATH` | | Can be specified when a DockerfilePath is specified outside the base WorkspaceFolder. This path MUST be relative to the WorkspaceFolder path into which the repo is cloned. | +| `--cache-ttl-days` | `ENVBUILDER_CACHE_TTL_DAYS` | | The number of days to use cached layers before expiring them. Defaults to 7 days. | +| `--docker-config-base64` | `ENVBUILDER_DOCKER_CONFIG_BASE64` | | The base64 encoded Docker config file that will be used to pull images from private container registries. | +| `--fallback-image` | `ENVBUILDER_FALLBACK_IMAGE` | | Specifies an alternative image to use when neither an image is declared in the devcontainer.json file nor a Dockerfile is present. If there's a build failure (from a faulty Dockerfile) or a misconfiguration, this image will be the substitute. Set ExitOnBuildFailure to true to halt the container if the build faces an issue. | +| `--exit-on-build-failure` | `ENVBUILDER_EXIT_ON_BUILD_FAILURE` | | Terminates the container upon a build failure. This is handy when preferring the FALLBACK_IMAGE in cases where no devcontainer.json or image is provided. However, it ensures that the container stops if the build process encounters an error. | +| `--force-safe` | `ENVBUILDER_FORCE_SAFE` | | Ignores any filesystem safety checks. This could cause serious harm to your system! This is used in cases where bypass is needed to unblock customers. | +| `--insecure` | `ENVBUILDER_INSECURE` | | Bypass TLS verification when cloning and pulling from container registries. | +| `--ignore-paths` | `ENVBUILDER_IGNORE_PATHS` | | The comma separated list of paths to ignore when building the workspace. | +| `--skip-rebuild` | `ENVBUILDER_SKIP_REBUILD` | | Skip building if the MagicFile exists. This is used to skip building when a container is restarting. e.g. docker stop -> docker start This value can always be set to true - even if the container is being started for the first time. | +| `--git-url` | `ENVBUILDER_GIT_URL` | | The URL of a Git repository containing a Devcontainer or Docker image to clone. This is optional. | +| `--git-clone-depth` | `ENVBUILDER_GIT_CLONE_DEPTH` | | The depth to use when cloning the Git repository. | +| `--git-clone-single-branch` | `ENVBUILDER_GIT_CLONE_SINGLE_BRANCH` | | Clone only a single branch of the Git repository. | +| `--git-username` | `ENVBUILDER_GIT_USERNAME` | | The username to use for Git authentication. This is optional. | +| `--git-password` | `ENVBUILDER_GIT_PASSWORD` | | The password to use for Git authentication. This is optional. | +| `--git-ssh-private-key-path` | `ENVBUILDER_GIT_SSH_PRIVATE_KEY_PATH` | | Path to an SSH private key to be used for Git authentication. | +| `--git-http-proxy-url` | `ENVBUILDER_GIT_HTTP_PROXY_URL` | | The URL for the HTTP proxy. This is optional. | +| `--workspace-folder` | `ENVBUILDER_WORKSPACE_FOLDER` | | The path to the workspace folder that will be built. This is optional. | +| `--ssl-cert-base64` | `ENVBUILDER_SSL_CERT_BASE64` | | The content of an SSL cert file. This is useful for self-signed certificates. | +| `--export-env-file` | `ENVBUILDER_EXPORT_ENV_FILE` | | Optional file path to a .env file where envbuilder will dump environment variables from devcontainer.json and the built container image. | +| `--post-start-script-path` | `ENVBUILDER_POST_START_SCRIPT_PATH` | | The path to a script that will be created by envbuilder based on the postStartCommand in devcontainer.json, if any is specified (otherwise the script is not created). If this is set, the specified InitCommand should check for the presence of this script and execute it after successful startup. | +| `--coder-agent-url` | `CODER_AGENT_URL` | | URL of the Coder deployment. If CODER_AGENT_TOKEN is also set, logs from envbuilder will be forwarded here and will be visible in the workspace build logs. | +| `--coder-agent-token` | `CODER_AGENT_TOKEN` | | Authentication token for a Coder agent. If this is set, then CODER_AGENT_URL must also be set. | +| `--coder-agent-subsystem` | `CODER_AGENT_SUBSYSTEM` | | Coder agent subsystems to report when forwarding logs. The envbuilder subsystem is always included. | +| `--push-image` | `ENVBUILDER_PUSH_IMAGE` | | Push the built image to a remote registry. This option forces a reproducible build. | +| `--get-cached-image` | `ENVBUILDER_GET_CACHED_IMAGE` | | Print the digest of the cached image, if available. Exits with an error if not found. | +| `--remote-repo-build-mode` | `ENVBUILDER_REMOTE_REPO_BUILD_MODE` | `false` | Use the remote repository as the source of truth when building the image. Enabling this option ignores user changes to local files and they will not be reflected in the image. This can be used to improving cache utilization when multiple users are building working on the same repository. | +| `--verbose` | `ENVBUILDER_VERBOSE` | | Enable verbose logging. | diff --git a/docs/git-auth.md b/docs/git-auth.md new file mode 100644 index 00000000..5f0acb0b --- /dev/null +++ b/docs/git-auth.md @@ -0,0 +1,66 @@ +# Git Authentication + +Two methods of authentication are supported: + +## HTTP Authentication + +If `ENVBUILDER_GIT_URL` starts with `http://` or `https://`, envbuilder will +authenticate with `ENVBUILDER_GIT_USERNAME` and `ENVBUILDER_GIT_PASSWORD`, if set. + +For access token-based authentication, follow the following schema (if empty, there's no need to provide the field): + +| Provider | `ENVBUILDER_GIT_USERNAME` | `ENVBUILDER_GIT_PASSWORD` | +| ------------ | ------------------------- | ------------------------- | +| GitHub | [access-token] | | +| GitLab | oauth2 | [access-token] | +| BitBucket | x-token-auth | [access-token] | +| Azure DevOps | [access-token] | | + +If using envbuilder inside of [Coder](https://github.com/coder/coder), you can use the `coder_external_auth` Terraform resource to automatically provide this token on workspace creation: + +```hcl +data "coder_external_auth" "github" { + id = "github" +} + +resource "docker_container" "dev" { + env = [ + ENVBUILDER_GIT_USERNAME = data.coder_external_auth.github.access_token, + ] +} +``` + +## SSH Authentication + +If `ENVBUILDER_GIT_URL` does not start with `http://` or `https://`, +envbuilder will assume SSH authentication. You have the following options: + +1. Public/Private key authentication: set `ENVBUILDER_GIT_SSH_PRIVATE_KEY_PATH` to the path of an + SSH private key mounted inside the container. Envbuilder will use this SSH + key to authenticate. Example: + + ```bash + docker run -it --rm \ + -v /tmp/envbuilder:/workspaces \ + -e ENVBUILDER_GIT_URL=git@example.com:path/to/private/repo.git \ + -e ENVBUILDER_INIT_SCRIPT=bash \ + -e ENVBUILDER_GIT_SSH_PRIVATE_KEY_PATH=/.ssh/id_rsa \ + -v /home/user/id_rsa:/.ssh/id_rsa \ + ghcr.io/coder/envbuilder + ``` + +1. Agent-based authentication: set `SSH_AUTH_SOCK` and mount in your agent socket, for example: + +```bash + docker run -it --rm \ + -v /tmp/envbuilder:/workspaces \ + -e ENVBUILDER_GIT_URL=git@example.com:path/to/private/repo.git \ + -e ENVBUILDER_INIT_SCRIPT=bash \ + -e SSH_AUTH_SOCK=/tmp/ssh-auth-sock \ + -v $SSH_AUTH_SOCK:/tmp/ssh-auth-sock \ + ghcr.io/coder/envbuilder +``` + +> Note: by default, envbuilder will accept and log all host keys. If you need +> strict host key checking, set `SSH_KNOWN_HOSTS` and mount in a `known_hosts` +> file. diff --git a/docs/images/dark-logo.svg b/docs/images/dark-logo.svg new file mode 100644 index 00000000..17081204 --- /dev/null +++ b/docs/images/dark-logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/images/light-logo.svg b/docs/images/light-logo.svg new file mode 100644 index 00000000..ecba8ae3 --- /dev/null +++ b/docs/images/light-logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/usage-with-coder.md b/docs/usage-with-coder.md new file mode 100644 index 00000000..cb0e58cb --- /dev/null +++ b/docs/usage-with-coder.md @@ -0,0 +1,27 @@ +# Usage with Coder + +Coder provides sample +[Docker](https://github.com/coder/coder/tree/main/examples/templates/devcontainer-docker) +and +[Kubernetes](https://github.com/coder/coder/tree/main/examples/templates/devcontainer-kubernetes) +templates for use with Envbuilder. You can import these templates and modify them to fit +your specific requirements. + +Below are some specific points to be aware of when using Envbuilder with a Coder +deployment: + +- The `ENVBUILDER_INIT_SCRIPT` should execute `coder_agent.main.init_script` in + order for you to be able to connect to your workspace. +- In order for the Agent init script to be able to fetch the agent binary from + your Coder deployment, the resulting Devcontainer must contain a download tool + such as `curl`, `wget`, or `busybox`. +- `CODER_AGENT_TOKEN` should be included in the environment variables for the + Envbuilder container. You can also set `CODER_AGENT_URL` if required. + +## Git Branch Selection + +Choose a branch using `ENVBUILDER_GIT_URL` with a _ref/heads_ reference. For instance: + +``` +ENVBUILDER_GIT_URL=https://github.com/coder/envbuilder-starter-devcontainer/#refs/heads/my-feature-branch +``` diff --git a/docs/using-local-files.md b/docs/using-local-files.md new file mode 100644 index 00000000..3c4f9b24 --- /dev/null +++ b/docs/using-local-files.md @@ -0,0 +1,34 @@ +# Using local files + +If you don't have a remote Git repo or you want to quickly iterate with some +local files, simply omit `ENVBUILDER_GIT_URL` and instead mount the directory +containing your code to `/workspaces/empty` inside the Envbuilder container. + +For example: + +```shell +# Create a sample Devcontainer and Dockerfile in the current directory +printf '{"build": { "dockerfile": "Dockerfile"}}' > devcontainer.json +printf 'FROM debian:bookworm\nRUN apt-get update && apt-get install -y cowsay' > Dockerfile + +# Run envbuilder with the current directory mounted into `/workspaces/empty`. +# The instructions to add /usr/games to $PATH have been omitted for brevity. +docker run -it --rm -e ENVBUILDER_INIT_SCRIPT='bash' -v $PWD:/workspaces/empty ghcr.io/coder/envbuilder:latest +``` + +Alternatively, if you prefer to mount your project files elsewhere, tell +Envbuilder where to find them by specifying `ENVBUILDER_WORKSPACE_FOLDER`: + +```shell +docker run -it --rm -e ENVBUILDER_INIT_SCRIPT='bash ' -e ENVBUILDER_WORKSPACE_FOLDER=/src -v $PWD:/src ghcr.io/coder/envbuilder:latest +``` + +By default, Envbuilder will look for a `devcontainer.json` or `Dockerfile` in +both `${ENVBUILDER_WORKSPACE_FOLDER}` and `${ENVBUILDER_WORKSPACE_FOLDER}/.devcontainer`. +You can control where it looks with `ENVBUILDER_DEVCONTAINER_DIR` if needed. + +```shell +ls build/ +Dockerfile devcontainer.json +docker run -it --rm -e ENVBUILDER_INIT_SCRIPT='bash' -e ENVBUILDER_DEVCONTAINER_DIR=build -v $PWD:/src ghcr.io/coder/envbuilder:latest +``` diff --git a/scripts/docsgen/main.go b/scripts/docsgen/main.go index 83d992c4..b61de096 100644 --- a/scripts/docsgen/main.go +++ b/scripts/docsgen/main.go @@ -3,37 +3,18 @@ package main import ( "fmt" "os" - "strings" + "path/filepath" "github.com/coder/envbuilder/options" ) -const ( - startSection = "" - endSection = "" -) - func main() { - readmePath := "README.md" - readmeFile, err := os.ReadFile(readmePath) - if err != nil { - panic("error reading " + readmePath + " file") - } - readmeContent := string(readmeFile) - startIndex := strings.Index(readmeContent, startSection) - endIndex := strings.Index(readmeContent, endSection) - if startIndex == -1 || endIndex == -1 { - panic("start or end section comments not found in the file.") - } - + path := filepath.Join("docs", "env-variables.md") var options options.Options - mkd := "\n## Environment Variables\n\n" + options.Markdown() - modifiedContent := readmeContent[:startIndex+len(startSection)] + mkd + readmeContent[endIndex:] - - err = os.WriteFile(readmePath, []byte(modifiedContent), 0o644) + mkd := "\n# Environment Variables\n\n" + options.Markdown() + err := os.WriteFile(path, []byte(mkd), 0o644) if err != nil { panic(err) } - - fmt.Println("README updated successfully with the latest flags!") + fmt.Printf("%s updated successfully with the latest flags!", path) } From 647ced3e21ba93b60f62bcc060cee04741971c96 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 26 Sep 2024 21:30:17 +0300 Subject: [PATCH 62/81] docs: fix link in readme (#363) (cherry picked from commit 4f3e9cde3acf0ed45f3079293ba43be23889a47d) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 981208c9..22208805 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- + From e7bf77b161da2d8b157e5532cae6d478d361f64e Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 30 Sep 2024 14:00:23 +0100 Subject: [PATCH 63/81] chore(README.md): remove logo (#362) Co-authored-by: Marcin Tojek (cherry picked from commit 845f11292a0a12c31b020cc3f5b87a7ac3c2bb1c) --- README.md | 9 --------- docs/images/dark-logo.svg | 1 - docs/images/light-logo.svg | 1 - 3 files changed, 11 deletions(-) delete mode 100644 docs/images/dark-logo.svg delete mode 100644 docs/images/light-logo.svg diff --git a/README.md b/README.md index 22208805..af5323de 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,3 @@ -

- - - - - - -

-

diff --git a/docs/images/dark-logo.svg b/docs/images/dark-logo.svg deleted file mode 100644 index 17081204..00000000 --- a/docs/images/dark-logo.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/docs/images/light-logo.svg b/docs/images/light-logo.svg deleted file mode 100644 index ecba8ae3..00000000 --- a/docs/images/light-logo.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file From b33c64c6e1c4f9fca1f65f364d1e4bafba8467f1 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Mon, 30 Sep 2024 16:52:14 +0300 Subject: [PATCH 64/81] fix: refactor coder logger to allow flush without deadlock (#375) (cherry picked from commit 817032347e524b6bff0fc99993f611a0761d858c) --- cmd/envbuilder/main.go | 16 ++- envbuilder.go | 7 +- go.mod | 2 +- integration/integration_test.go | 67 ++++++++++ log/coder.go | 102 +++++++++------- log/coder_internal_test.go | 208 ++++++++++++++++++++++++++------ 6 files changed, 316 insertions(+), 86 deletions(-) diff --git a/cmd/envbuilder/main.go b/cmd/envbuilder/main.go index 410e0897..9159f84a 100644 --- a/cmd/envbuilder/main.go +++ b/cmd/envbuilder/main.go @@ -37,6 +37,15 @@ func envbuilderCmd() serpent.Command { Options: o.CLI(), Handler: func(inv *serpent.Invocation) error { o.SetDefaults() + var preExecs []func() + preExec := func() { + for _, fn := range preExecs { + fn() + } + preExecs = nil + } + defer preExec() // Ensure cleanup in case of error. + o.Logger = log.New(os.Stderr, o.Verbose) if o.CoderAgentURL != "" { if o.CoderAgentToken == "" { @@ -49,7 +58,10 @@ func envbuilderCmd() serpent.Command { coderLog, closeLogs, err := log.Coder(inv.Context(), u, o.CoderAgentToken) if err == nil { o.Logger = log.Wrap(o.Logger, coderLog) - defer closeLogs() + preExecs = append(preExecs, func() { + o.Logger(log.LevelInfo, "Closing logs") + closeLogs() + }) // This adds the envbuilder subsystem. // If telemetry is enabled in a Coder deployment, // this will be reported and help us understand @@ -78,7 +90,7 @@ func envbuilderCmd() serpent.Command { return nil } - err := envbuilder.Run(inv.Context(), o) + err := envbuilder.Run(inv.Context(), o, preExec) if err != nil { o.Logger(log.LevelError, "error: %s", err) } diff --git a/envbuilder.go b/envbuilder.go index 683f6a54..94998165 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -84,7 +84,9 @@ type execArgsInfo struct { // Logger is the logf to use for all operations. // Filesystem is the filesystem to use for all operations. // Defaults to the host filesystem. -func Run(ctx context.Context, opts options.Options) error { +// preExec are any functions that should be called before exec'ing the init +// command. This is useful for ensuring that defers get run. +func Run(ctx context.Context, opts options.Options, preExec ...func()) error { var args execArgsInfo // Run in a separate function to ensure all defers run before we // setuid or exec. @@ -103,6 +105,9 @@ func Run(ctx context.Context, opts options.Options) error { } opts.Logger(log.LevelInfo, "=== Running the init command %s %+v as the %q user...", opts.InitCommand, args.InitArgs, args.UserInfo.user.Username) + for _, fn := range preExec { + fn() + } err = syscall.Exec(args.InitCommand, append([]string{args.InitCommand}, args.InitArgs...), args.Environ) if err != nil { diff --git a/go.mod b/go.mod index b3fa7843..9fa1d696 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/gliderlabs/ssh v0.3.7 github.com/go-git/go-billy/v5 v5.5.0 github.com/go-git/go-git/v5 v5.12.0 + github.com/google/go-cmp v0.6.0 github.com/google/go-containerregistry v0.20.1 github.com/google/uuid v1.6.0 github.com/hashicorp/go-multierror v1.1.1 @@ -149,7 +150,6 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.2 // indirect - github.com/google/go-cmp v0.6.0 // indirect github.com/google/nftables v0.2.0 // indirect github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect github.com/gorilla/handlers v1.5.1 // indirect diff --git a/integration/integration_test.go b/integration/integration_test.go index 66dfe846..79b678d5 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -23,6 +23,8 @@ import ( "testing" "time" + "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/codersdk/agentsdk" "github.com/coder/envbuilder" "github.com/coder/envbuilder/devcontainer/features" "github.com/coder/envbuilder/internal/magicdir" @@ -58,6 +60,71 @@ const ( testImageUbuntu = "localhost:5000/envbuilder-test-ubuntu:latest" ) +func TestLogs(t *testing.T) { + t.Parallel() + + token := uuid.NewString() + logsDone := make(chan struct{}) + + logHandler := func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v2/buildinfo": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"version": "v2.8.9"}`)) + return + case "/api/v2/workspaceagents/me/logs": + w.WriteHeader(http.StatusOK) + tokHdr := r.Header.Get(codersdk.SessionTokenHeader) + assert.Equal(t, token, tokHdr) + var req agentsdk.PatchLogs + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + for _, log := range req.Logs { + t.Logf("got log: %+v", log) + if strings.Contains(log.Output, "Closing logs") { + close(logsDone) + return + } + } + return + default: + t.Errorf("unexpected request to %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + return + } + } + logSrv := httptest.NewServer(http.HandlerFunc(logHandler)) + defer logSrv.Close() + + // Ensures that a Git repository with a devcontainer.json is cloned and built. + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ + "devcontainer.json": `{ + "build": { + "dockerfile": "Dockerfile" + }, + }`, + "Dockerfile": fmt.Sprintf(`FROM %s`, testImageUbuntu), + }, + }) + _, err := runEnvbuilder(t, runOpts{env: []string{ + envbuilderEnv("GIT_URL", srv.URL), + "CODER_AGENT_URL=" + logSrv.URL, + "CODER_AGENT_TOKEN=" + token, + }}) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + select { + case <-ctx.Done(): + t.Fatal("timed out waiting for logs") + case <-logsDone: + } +} + func TestInitScriptInitCommand(t *testing.T) { t.Parallel() diff --git a/log/coder.go b/log/coder.go index d8b4fe0d..b551140d 100644 --- a/log/coder.go +++ b/log/coder.go @@ -6,6 +6,7 @@ import ( "fmt" "net/url" "os" + "sync" "time" "cdr.dev/slog" @@ -27,13 +28,14 @@ var ( minAgentAPIV2 = "v2.9" ) -// Coder establishes a connection to the Coder instance located at -// coderURL and authenticates using token. It then establishes a -// dRPC connection to the Agent API and begins sending logs. -// If the version of Coder does not support the Agent API, it will -// fall back to using the PatchLogs endpoint. -// The returned function is used to block until all logs are sent. -func Coder(ctx context.Context, coderURL *url.URL, token string) (Func, func(), error) { +// Coder establishes a connection to the Coder instance located at coderURL and +// authenticates using token. It then establishes a dRPC connection to the Agent +// API and begins sending logs. If the version of Coder does not support the +// Agent API, it will fall back to using the PatchLogs endpoint. The closer is +// used to close the logger and to wait at most logSendGracePeriod for logs to +// be sent. Cancelling the context will close the logs immediately without +// waiting for logs to be sent. +func Coder(ctx context.Context, coderURL *url.URL, token string) (logger Func, closer func(), err error) { // To troubleshoot issues, we need some way of logging. metaLogger := slog.Make(sloghuman.Sink(os.Stderr)) defer metaLogger.Sync() @@ -44,9 +46,11 @@ func Coder(ctx context.Context, coderURL *url.URL, token string) (Func, func(), } if semver.Compare(semver.MajorMinor(bi.Version), minAgentAPIV2) < 0 { metaLogger.Warn(ctx, "Detected Coder version incompatible with AgentAPI v2, falling back to deprecated API", slog.F("coder_version", bi.Version)) - sendLogs, flushLogs := sendLogsV1(ctx, client, metaLogger.Named("send_logs_v1")) - return sendLogs, flushLogs, nil + logger, closer = sendLogsV1(ctx, client, metaLogger.Named("send_logs_v1")) + return logger, closer, nil } + // Note that ctx passed to initRPC will be inherited by the + // underlying connection, nothing we can do about that here. dac, err := initRPC(ctx, client, metaLogger.Named("init_rpc")) if err != nil { // Logged externally @@ -54,8 +58,14 @@ func Coder(ctx context.Context, coderURL *url.URL, token string) (Func, func(), } ls := agentsdk.NewLogSender(metaLogger.Named("coder_log_sender")) metaLogger.Warn(ctx, "Sending logs via AgentAPI v2", slog.F("coder_version", bi.Version)) - sendLogs, doneFunc := sendLogsV2(ctx, dac, ls, metaLogger.Named("send_logs_v2")) - return sendLogs, doneFunc, nil + logger, closer = sendLogsV2(ctx, dac, ls, metaLogger.Named("send_logs_v2")) + var closeOnce sync.Once + return logger, func() { + closer() + closeOnce.Do(func() { + _ = dac.DRPCConn().Close() + }) + }, nil } type coderLogSender interface { @@ -74,7 +84,7 @@ func initClient(coderURL *url.URL, token string) *agentsdk.Client { func initRPC(ctx context.Context, client *agentsdk.Client, l slog.Logger) (proto.DRPCAgentClient20, error) { var c proto.DRPCAgentClient20 var err error - retryCtx, retryCancel := context.WithTimeout(context.Background(), rpcConnectTimeout) + retryCtx, retryCancel := context.WithTimeout(ctx, rpcConnectTimeout) defer retryCancel() attempts := 0 for r := retry.New(100*time.Millisecond, time.Second); r.Wait(retryCtx); { @@ -95,65 +105,67 @@ func initRPC(ctx context.Context, client *agentsdk.Client, l slog.Logger) (proto // sendLogsV1 uses the PatchLogs endpoint to send logs. // This is deprecated, but required for backward compatibility with older versions of Coder. -func sendLogsV1(ctx context.Context, client *agentsdk.Client, l slog.Logger) (Func, func()) { +func sendLogsV1(ctx context.Context, client *agentsdk.Client, l slog.Logger) (logger Func, closer func()) { // nolint: staticcheck // required for backwards compatibility - sendLogs, flushLogs := agentsdk.LogsSender(agentsdk.ExternalLogSourceID, client.PatchLogs, slog.Logger{}) + sendLog, flushAndClose := agentsdk.LogsSender(agentsdk.ExternalLogSourceID, client.PatchLogs, slog.Logger{}) + var mu sync.Mutex return func(lvl Level, msg string, args ...any) { log := agentsdk.Log{ CreatedAt: time.Now(), Output: fmt.Sprintf(msg, args...), Level: codersdk.LogLevel(lvl), } - if err := sendLogs(ctx, log); err != nil { + mu.Lock() + defer mu.Unlock() + if err := sendLog(ctx, log); err != nil { l.Warn(ctx, "failed to send logs to Coder", slog.Error(err)) } }, func() { - if err := flushLogs(ctx); err != nil { + ctx, cancel := context.WithTimeout(ctx, logSendGracePeriod) + defer cancel() + if err := flushAndClose(ctx); err != nil { l.Warn(ctx, "failed to flush logs", slog.Error(err)) } } } // sendLogsV2 uses the v2 agent API to send logs. Only compatibile with coder versions >= 2.9. -func sendLogsV2(ctx context.Context, dest agentsdk.LogDest, ls coderLogSender, l slog.Logger) (Func, func()) { +func sendLogsV2(ctx context.Context, dest agentsdk.LogDest, ls coderLogSender, l slog.Logger) (logger Func, closer func()) { + sendCtx, sendCancel := context.WithCancel(ctx) done := make(chan struct{}) uid := uuid.New() go func() { defer close(done) - if err := ls.SendLoop(ctx, dest); err != nil { + if err := ls.SendLoop(sendCtx, dest); err != nil { if !errors.Is(err, context.Canceled) { l.Warn(ctx, "failed to send logs to Coder", slog.Error(err)) } } - - // Wait for up to 10 seconds for logs to finish sending. - sendCtx, sendCancel := context.WithTimeout(context.Background(), logSendGracePeriod) - defer sendCancel() - // Try once more to send any pending logs - if err := ls.SendLoop(sendCtx, dest); err != nil { - if !errors.Is(err, context.DeadlineExceeded) { - l.Warn(ctx, "failed to send remaining logs to Coder", slog.Error(err)) - } - } - ls.Flush(uid) - if err := ls.WaitUntilEmpty(sendCtx); err != nil { - if !errors.Is(err, context.DeadlineExceeded) { - l.Warn(ctx, "log sender did not empty", slog.Error(err)) - } - } }() - logFunc := func(l Level, msg string, args ...any) { - ls.Enqueue(uid, agentsdk.Log{ - CreatedAt: time.Now(), - Output: fmt.Sprintf(msg, args...), - Level: codersdk.LogLevel(l), - }) - } + var closeOnce sync.Once + return func(l Level, msg string, args ...any) { + ls.Enqueue(uid, agentsdk.Log{ + CreatedAt: time.Now(), + Output: fmt.Sprintf(msg, args...), + Level: codersdk.LogLevel(l), + }) + }, func() { + closeOnce.Do(func() { + // Trigger a flush and wait for logs to be sent. + ls.Flush(uid) + ctx, cancel := context.WithTimeout(ctx, logSendGracePeriod) + defer cancel() + err := ls.WaitUntilEmpty(ctx) + if err != nil { + l.Warn(ctx, "log sender did not empty", slog.Error(err)) + } - doneFunc := func() { - <-done - } + // Stop the send loop. + sendCancel() + }) - return logFunc, doneFunc + // Wait for the send loop to finish. + <-done + } } diff --git a/log/coder_internal_test.go b/log/coder_internal_test.go index 4895150e..8b8bb632 100644 --- a/log/coder_internal_test.go +++ b/log/coder_internal_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "math/rand" "net/http" "net/http/httptest" "net/url" @@ -38,10 +39,8 @@ func TestCoder(t *testing.T) { defer closeOnce.Do(func() { close(gotLogs) }) tokHdr := r.Header.Get(codersdk.SessionTokenHeader) assert.Equal(t, token, tokHdr) - var req agentsdk.PatchLogs - err := json.NewDecoder(r.Body).Decode(&req) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) + req, ok := decodeV1Logs(t, w, r) + if !ok { return } if assert.Len(t, req.Logs, 1) { @@ -54,15 +53,44 @@ func TestCoder(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - u, err := url.Parse(srv.URL) - require.NoError(t, err) - log, closeLog, err := Coder(ctx, u, token) - require.NoError(t, err) - defer closeLog() - log(LevelInfo, "hello %s", "world") + + logger, _ := newCoderLogger(ctx, t, srv.URL, token) + logger(LevelInfo, "hello %s", "world") <-gotLogs }) + t.Run("V1/Close", func(t *testing.T) { + t.Parallel() + + var got []agentsdk.Log + handler := func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v2/buildinfo" { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"version": "v2.8.9"}`)) + return + } + req, ok := decodeV1Logs(t, w, r) + if !ok { + return + } + got = append(got, req.Logs...) + } + srv := httptest.NewServer(http.HandlerFunc(handler)) + defer srv.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + logger, closer := newCoderLogger(ctx, t, srv.URL, uuid.NewString()) + logger(LevelInfo, "1") + logger(LevelInfo, "2") + closer() + logger(LevelInfo, "3") + require.Len(t, got, 2) + assert.Equal(t, "1", got[0].Output) + assert.Equal(t, "2", got[1].Output) + }) + t.Run("V1/ErrUnauthorized", func(t *testing.T) { t.Parallel() @@ -140,42 +168,31 @@ func TestCoder(t *testing.T) { require.Len(t, ld.logs, 10) }) - // In this test, we just stand up an endpoint that does not - // do dRPC. We'll try to connect, fail to websocket upgrade - // and eventually give up. - t.Run("V2/Err", func(t *testing.T) { + // In this test, we just fake out the DRPC server. + t.Run("V2/Close", func(t *testing.T) { t.Parallel() - token := uuid.NewString() - handlerDone := make(chan struct{}) - var closeOnce sync.Once - handler := func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/api/v2/buildinfo" { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"version": "v2.9.0"}`)) - return - } - defer closeOnce.Do(func() { close(handlerDone) }) - w.WriteHeader(http.StatusOK) - } - srv := httptest.NewServer(http.HandlerFunc(handler)) - defer srv.Close() - - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() - u, err := url.Parse(srv.URL) - require.NoError(t, err) - _, _, err = Coder(ctx, u, token) - require.ErrorContains(t, err, "failed to WebSocket dial") - require.ErrorIs(t, err, context.DeadlineExceeded) - <-handlerDone + + ld := &fakeLogDest{t: t} + ls := agentsdk.NewLogSender(slogtest.Make(t, nil)) + logger, closer := sendLogsV2(ctx, ld, ls, slogtest.Make(t, nil)) + defer closer() + + logger(LevelInfo, "1") + logger(LevelInfo, "2") + closer() + logger(LevelInfo, "3") + + require.Len(t, ld.logs, 2) }) // In this test, we validate that a 401 error on the initial connect // results in a retry. When envbuilder initially attempts to connect // using the Coder agent token, the workspace build may not yet have // completed. - t.Run("V2Retry", func(t *testing.T) { + t.Run("V2/Retry", func(t *testing.T) { t.Parallel() ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -221,6 +238,99 @@ func TestCoder(t *testing.T) { }) } +//nolint:paralleltest // We need to replace a global timeout. +func TestCoderRPCTimeout(t *testing.T) { + // This timeout is picked with the current subtests in mind, it + // should not be changed without good reason. + testReplaceTimeout(t, &rpcConnectTimeout, 500*time.Millisecond) + + // In this test, we just stand up an endpoint that does not + // do dRPC. We'll try to connect, fail to websocket upgrade + // and eventually give up after rpcConnectTimeout. + t.Run("V2/Err", func(t *testing.T) { + t.Parallel() + + token := uuid.NewString() + handlerDone := make(chan struct{}) + handlerWait := make(chan struct{}) + var closeOnce sync.Once + handler := func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v2/buildinfo" { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"version": "v2.9.0"}`)) + return + } + defer closeOnce.Do(func() { close(handlerDone) }) + <-handlerWait + w.WriteHeader(http.StatusOK) + } + srv := httptest.NewServer(http.HandlerFunc(handler)) + defer srv.Close() + + ctx, cancel := context.WithTimeout(context.Background(), rpcConnectTimeout/2) + defer cancel() + u, err := url.Parse(srv.URL) + require.NoError(t, err) + _, _, err = Coder(ctx, u, token) + require.ErrorContains(t, err, "failed to WebSocket dial") + require.ErrorIs(t, err, context.DeadlineExceeded) + close(handlerWait) + <-handlerDone + }) + + t.Run("V2/Timeout", func(t *testing.T) { + t.Parallel() + + token := uuid.NewString() + handlerDone := make(chan struct{}) + handlerWait := make(chan struct{}) + var closeOnce sync.Once + handler := func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v2/buildinfo" { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"version": "v2.9.0"}`)) + return + } + defer closeOnce.Do(func() { close(handlerDone) }) + <-handlerWait + w.WriteHeader(http.StatusOK) + } + srv := httptest.NewServer(http.HandlerFunc(handler)) + defer srv.Close() + + ctx, cancel := context.WithTimeout(context.Background(), rpcConnectTimeout*2) + defer cancel() + u, err := url.Parse(srv.URL) + require.NoError(t, err) + _, _, err = Coder(ctx, u, token) + require.ErrorContains(t, err, "failed to WebSocket dial") + require.ErrorIs(t, err, context.DeadlineExceeded) + close(handlerWait) + <-handlerDone + }) +} + +func decodeV1Logs(t *testing.T, w http.ResponseWriter, r *http.Request) (agentsdk.PatchLogs, bool) { + t.Helper() + var req agentsdk.PatchLogs + err := json.NewDecoder(r.Body).Decode(&req) + if !assert.NoError(t, err) { + http.Error(w, err.Error(), http.StatusBadRequest) + return req, false + } + return req, true +} + +func newCoderLogger(ctx context.Context, t *testing.T, us string, token string) (Func, func()) { + t.Helper() + u, err := url.Parse(us) + require.NoError(t, err) + logger, closer, err := Coder(ctx, u, token) + require.NoError(t, err) + t.Cleanup(closer) + return logger, closer +} + type fakeLogDest struct { t testing.TB logs []*proto.Log @@ -231,3 +341,27 @@ func (d *fakeLogDest) BatchCreateLogs(ctx context.Context, request *proto.BatchC d.logs = append(d.logs, request.Logs...) return &proto.BatchCreateLogsResponse{}, nil } + +func testReplaceTimeout(t *testing.T, v *time.Duration, d time.Duration) { + t.Helper() + if isParallel(t) { + t.Fatal("cannot replace timeout in parallel test") + } + old := *v + *v = d + t.Cleanup(func() { *v = old }) +} + +func isParallel(t *testing.T) (ret bool) { + t.Helper() + // This is a hack to determine if the test is running in parallel + // via property of t.Setenv. + defer func() { + if r := recover(); r != nil { + ret = true + } + }() + // Random variable name to avoid collisions. + t.Setenv(fmt.Sprintf("__TEST_CHECK_IS_PARALLEL_%d", rand.Int()), "1") + return false +} From e9f46cc84317718df9b7918113fb717a8ecc4b43 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Mon, 30 Sep 2024 19:27:33 +0300 Subject: [PATCH 65/81] revert: "fix: refactor coder logger to allow flush without deadlock (#375)" (#376) (cherry picked from commit a1e8f3c3b70841e7245de3eea709e6abdde2cc10) --- cmd/envbuilder/main.go | 16 +-- envbuilder.go | 7 +- go.mod | 2 +- integration/integration_test.go | 67 ---------- log/coder.go | 102 +++++++--------- log/coder_internal_test.go | 208 ++++++-------------------------- 6 files changed, 86 insertions(+), 316 deletions(-) diff --git a/cmd/envbuilder/main.go b/cmd/envbuilder/main.go index 9159f84a..410e0897 100644 --- a/cmd/envbuilder/main.go +++ b/cmd/envbuilder/main.go @@ -37,15 +37,6 @@ func envbuilderCmd() serpent.Command { Options: o.CLI(), Handler: func(inv *serpent.Invocation) error { o.SetDefaults() - var preExecs []func() - preExec := func() { - for _, fn := range preExecs { - fn() - } - preExecs = nil - } - defer preExec() // Ensure cleanup in case of error. - o.Logger = log.New(os.Stderr, o.Verbose) if o.CoderAgentURL != "" { if o.CoderAgentToken == "" { @@ -58,10 +49,7 @@ func envbuilderCmd() serpent.Command { coderLog, closeLogs, err := log.Coder(inv.Context(), u, o.CoderAgentToken) if err == nil { o.Logger = log.Wrap(o.Logger, coderLog) - preExecs = append(preExecs, func() { - o.Logger(log.LevelInfo, "Closing logs") - closeLogs() - }) + defer closeLogs() // This adds the envbuilder subsystem. // If telemetry is enabled in a Coder deployment, // this will be reported and help us understand @@ -90,7 +78,7 @@ func envbuilderCmd() serpent.Command { return nil } - err := envbuilder.Run(inv.Context(), o, preExec) + err := envbuilder.Run(inv.Context(), o) if err != nil { o.Logger(log.LevelError, "error: %s", err) } diff --git a/envbuilder.go b/envbuilder.go index 94998165..683f6a54 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -84,9 +84,7 @@ type execArgsInfo struct { // Logger is the logf to use for all operations. // Filesystem is the filesystem to use for all operations. // Defaults to the host filesystem. -// preExec are any functions that should be called before exec'ing the init -// command. This is useful for ensuring that defers get run. -func Run(ctx context.Context, opts options.Options, preExec ...func()) error { +func Run(ctx context.Context, opts options.Options) error { var args execArgsInfo // Run in a separate function to ensure all defers run before we // setuid or exec. @@ -105,9 +103,6 @@ func Run(ctx context.Context, opts options.Options, preExec ...func()) error { } opts.Logger(log.LevelInfo, "=== Running the init command %s %+v as the %q user...", opts.InitCommand, args.InitArgs, args.UserInfo.user.Username) - for _, fn := range preExec { - fn() - } err = syscall.Exec(args.InitCommand, append([]string{args.InitCommand}, args.InitArgs...), args.Environ) if err != nil { diff --git a/go.mod b/go.mod index 9fa1d696..b3fa7843 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,6 @@ require ( github.com/gliderlabs/ssh v0.3.7 github.com/go-git/go-billy/v5 v5.5.0 github.com/go-git/go-git/v5 v5.12.0 - github.com/google/go-cmp v0.6.0 github.com/google/go-containerregistry v0.20.1 github.com/google/uuid v1.6.0 github.com/hashicorp/go-multierror v1.1.1 @@ -150,6 +149,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.2 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/nftables v0.2.0 // indirect github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect github.com/gorilla/handlers v1.5.1 // indirect diff --git a/integration/integration_test.go b/integration/integration_test.go index 79b678d5..66dfe846 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -23,8 +23,6 @@ import ( "testing" "time" - "github.com/coder/coder/v2/codersdk" - "github.com/coder/coder/v2/codersdk/agentsdk" "github.com/coder/envbuilder" "github.com/coder/envbuilder/devcontainer/features" "github.com/coder/envbuilder/internal/magicdir" @@ -60,71 +58,6 @@ const ( testImageUbuntu = "localhost:5000/envbuilder-test-ubuntu:latest" ) -func TestLogs(t *testing.T) { - t.Parallel() - - token := uuid.NewString() - logsDone := make(chan struct{}) - - logHandler := func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/api/v2/buildinfo": - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"version": "v2.8.9"}`)) - return - case "/api/v2/workspaceagents/me/logs": - w.WriteHeader(http.StatusOK) - tokHdr := r.Header.Get(codersdk.SessionTokenHeader) - assert.Equal(t, token, tokHdr) - var req agentsdk.PatchLogs - err := json.NewDecoder(r.Body).Decode(&req) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - for _, log := range req.Logs { - t.Logf("got log: %+v", log) - if strings.Contains(log.Output, "Closing logs") { - close(logsDone) - return - } - } - return - default: - t.Errorf("unexpected request to %s", r.URL.Path) - w.WriteHeader(http.StatusNotFound) - return - } - } - logSrv := httptest.NewServer(http.HandlerFunc(logHandler)) - defer logSrv.Close() - - // Ensures that a Git repository with a devcontainer.json is cloned and built. - srv := gittest.CreateGitServer(t, gittest.Options{ - Files: map[string]string{ - "devcontainer.json": `{ - "build": { - "dockerfile": "Dockerfile" - }, - }`, - "Dockerfile": fmt.Sprintf(`FROM %s`, testImageUbuntu), - }, - }) - _, err := runEnvbuilder(t, runOpts{env: []string{ - envbuilderEnv("GIT_URL", srv.URL), - "CODER_AGENT_URL=" + logSrv.URL, - "CODER_AGENT_TOKEN=" + token, - }}) - require.NoError(t, err) - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - select { - case <-ctx.Done(): - t.Fatal("timed out waiting for logs") - case <-logsDone: - } -} - func TestInitScriptInitCommand(t *testing.T) { t.Parallel() diff --git a/log/coder.go b/log/coder.go index b551140d..d8b4fe0d 100644 --- a/log/coder.go +++ b/log/coder.go @@ -6,7 +6,6 @@ import ( "fmt" "net/url" "os" - "sync" "time" "cdr.dev/slog" @@ -28,14 +27,13 @@ var ( minAgentAPIV2 = "v2.9" ) -// Coder establishes a connection to the Coder instance located at coderURL and -// authenticates using token. It then establishes a dRPC connection to the Agent -// API and begins sending logs. If the version of Coder does not support the -// Agent API, it will fall back to using the PatchLogs endpoint. The closer is -// used to close the logger and to wait at most logSendGracePeriod for logs to -// be sent. Cancelling the context will close the logs immediately without -// waiting for logs to be sent. -func Coder(ctx context.Context, coderURL *url.URL, token string) (logger Func, closer func(), err error) { +// Coder establishes a connection to the Coder instance located at +// coderURL and authenticates using token. It then establishes a +// dRPC connection to the Agent API and begins sending logs. +// If the version of Coder does not support the Agent API, it will +// fall back to using the PatchLogs endpoint. +// The returned function is used to block until all logs are sent. +func Coder(ctx context.Context, coderURL *url.URL, token string) (Func, func(), error) { // To troubleshoot issues, we need some way of logging. metaLogger := slog.Make(sloghuman.Sink(os.Stderr)) defer metaLogger.Sync() @@ -46,11 +44,9 @@ func Coder(ctx context.Context, coderURL *url.URL, token string) (logger Func, c } if semver.Compare(semver.MajorMinor(bi.Version), minAgentAPIV2) < 0 { metaLogger.Warn(ctx, "Detected Coder version incompatible with AgentAPI v2, falling back to deprecated API", slog.F("coder_version", bi.Version)) - logger, closer = sendLogsV1(ctx, client, metaLogger.Named("send_logs_v1")) - return logger, closer, nil + sendLogs, flushLogs := sendLogsV1(ctx, client, metaLogger.Named("send_logs_v1")) + return sendLogs, flushLogs, nil } - // Note that ctx passed to initRPC will be inherited by the - // underlying connection, nothing we can do about that here. dac, err := initRPC(ctx, client, metaLogger.Named("init_rpc")) if err != nil { // Logged externally @@ -58,14 +54,8 @@ func Coder(ctx context.Context, coderURL *url.URL, token string) (logger Func, c } ls := agentsdk.NewLogSender(metaLogger.Named("coder_log_sender")) metaLogger.Warn(ctx, "Sending logs via AgentAPI v2", slog.F("coder_version", bi.Version)) - logger, closer = sendLogsV2(ctx, dac, ls, metaLogger.Named("send_logs_v2")) - var closeOnce sync.Once - return logger, func() { - closer() - closeOnce.Do(func() { - _ = dac.DRPCConn().Close() - }) - }, nil + sendLogs, doneFunc := sendLogsV2(ctx, dac, ls, metaLogger.Named("send_logs_v2")) + return sendLogs, doneFunc, nil } type coderLogSender interface { @@ -84,7 +74,7 @@ func initClient(coderURL *url.URL, token string) *agentsdk.Client { func initRPC(ctx context.Context, client *agentsdk.Client, l slog.Logger) (proto.DRPCAgentClient20, error) { var c proto.DRPCAgentClient20 var err error - retryCtx, retryCancel := context.WithTimeout(ctx, rpcConnectTimeout) + retryCtx, retryCancel := context.WithTimeout(context.Background(), rpcConnectTimeout) defer retryCancel() attempts := 0 for r := retry.New(100*time.Millisecond, time.Second); r.Wait(retryCtx); { @@ -105,67 +95,65 @@ func initRPC(ctx context.Context, client *agentsdk.Client, l slog.Logger) (proto // sendLogsV1 uses the PatchLogs endpoint to send logs. // This is deprecated, but required for backward compatibility with older versions of Coder. -func sendLogsV1(ctx context.Context, client *agentsdk.Client, l slog.Logger) (logger Func, closer func()) { +func sendLogsV1(ctx context.Context, client *agentsdk.Client, l slog.Logger) (Func, func()) { // nolint: staticcheck // required for backwards compatibility - sendLog, flushAndClose := agentsdk.LogsSender(agentsdk.ExternalLogSourceID, client.PatchLogs, slog.Logger{}) - var mu sync.Mutex + sendLogs, flushLogs := agentsdk.LogsSender(agentsdk.ExternalLogSourceID, client.PatchLogs, slog.Logger{}) return func(lvl Level, msg string, args ...any) { log := agentsdk.Log{ CreatedAt: time.Now(), Output: fmt.Sprintf(msg, args...), Level: codersdk.LogLevel(lvl), } - mu.Lock() - defer mu.Unlock() - if err := sendLog(ctx, log); err != nil { + if err := sendLogs(ctx, log); err != nil { l.Warn(ctx, "failed to send logs to Coder", slog.Error(err)) } }, func() { - ctx, cancel := context.WithTimeout(ctx, logSendGracePeriod) - defer cancel() - if err := flushAndClose(ctx); err != nil { + if err := flushLogs(ctx); err != nil { l.Warn(ctx, "failed to flush logs", slog.Error(err)) } } } // sendLogsV2 uses the v2 agent API to send logs. Only compatibile with coder versions >= 2.9. -func sendLogsV2(ctx context.Context, dest agentsdk.LogDest, ls coderLogSender, l slog.Logger) (logger Func, closer func()) { - sendCtx, sendCancel := context.WithCancel(ctx) +func sendLogsV2(ctx context.Context, dest agentsdk.LogDest, ls coderLogSender, l slog.Logger) (Func, func()) { done := make(chan struct{}) uid := uuid.New() go func() { defer close(done) - if err := ls.SendLoop(sendCtx, dest); err != nil { + if err := ls.SendLoop(ctx, dest); err != nil { if !errors.Is(err, context.Canceled) { l.Warn(ctx, "failed to send logs to Coder", slog.Error(err)) } } + + // Wait for up to 10 seconds for logs to finish sending. + sendCtx, sendCancel := context.WithTimeout(context.Background(), logSendGracePeriod) + defer sendCancel() + // Try once more to send any pending logs + if err := ls.SendLoop(sendCtx, dest); err != nil { + if !errors.Is(err, context.DeadlineExceeded) { + l.Warn(ctx, "failed to send remaining logs to Coder", slog.Error(err)) + } + } + ls.Flush(uid) + if err := ls.WaitUntilEmpty(sendCtx); err != nil { + if !errors.Is(err, context.DeadlineExceeded) { + l.Warn(ctx, "log sender did not empty", slog.Error(err)) + } + } }() - var closeOnce sync.Once - return func(l Level, msg string, args ...any) { - ls.Enqueue(uid, agentsdk.Log{ - CreatedAt: time.Now(), - Output: fmt.Sprintf(msg, args...), - Level: codersdk.LogLevel(l), - }) - }, func() { - closeOnce.Do(func() { - // Trigger a flush and wait for logs to be sent. - ls.Flush(uid) - ctx, cancel := context.WithTimeout(ctx, logSendGracePeriod) - defer cancel() - err := ls.WaitUntilEmpty(ctx) - if err != nil { - l.Warn(ctx, "log sender did not empty", slog.Error(err)) - } + logFunc := func(l Level, msg string, args ...any) { + ls.Enqueue(uid, agentsdk.Log{ + CreatedAt: time.Now(), + Output: fmt.Sprintf(msg, args...), + Level: codersdk.LogLevel(l), + }) + } - // Stop the send loop. - sendCancel() - }) + doneFunc := func() { + <-done + } - // Wait for the send loop to finish. - <-done - } + return logFunc, doneFunc } diff --git a/log/coder_internal_test.go b/log/coder_internal_test.go index 8b8bb632..4895150e 100644 --- a/log/coder_internal_test.go +++ b/log/coder_internal_test.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "math/rand" "net/http" "net/http/httptest" "net/url" @@ -39,8 +38,10 @@ func TestCoder(t *testing.T) { defer closeOnce.Do(func() { close(gotLogs) }) tokHdr := r.Header.Get(codersdk.SessionTokenHeader) assert.Equal(t, token, tokHdr) - req, ok := decodeV1Logs(t, w, r) - if !ok { + var req agentsdk.PatchLogs + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) return } if assert.Len(t, req.Logs, 1) { @@ -53,44 +54,15 @@ func TestCoder(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - - logger, _ := newCoderLogger(ctx, t, srv.URL, token) - logger(LevelInfo, "hello %s", "world") + u, err := url.Parse(srv.URL) + require.NoError(t, err) + log, closeLog, err := Coder(ctx, u, token) + require.NoError(t, err) + defer closeLog() + log(LevelInfo, "hello %s", "world") <-gotLogs }) - t.Run("V1/Close", func(t *testing.T) { - t.Parallel() - - var got []agentsdk.Log - handler := func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/api/v2/buildinfo" { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"version": "v2.8.9"}`)) - return - } - req, ok := decodeV1Logs(t, w, r) - if !ok { - return - } - got = append(got, req.Logs...) - } - srv := httptest.NewServer(http.HandlerFunc(handler)) - defer srv.Close() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - logger, closer := newCoderLogger(ctx, t, srv.URL, uuid.NewString()) - logger(LevelInfo, "1") - logger(LevelInfo, "2") - closer() - logger(LevelInfo, "3") - require.Len(t, got, 2) - assert.Equal(t, "1", got[0].Output) - assert.Equal(t, "2", got[1].Output) - }) - t.Run("V1/ErrUnauthorized", func(t *testing.T) { t.Parallel() @@ -168,31 +140,42 @@ func TestCoder(t *testing.T) { require.Len(t, ld.logs, 10) }) - // In this test, we just fake out the DRPC server. - t.Run("V2/Close", func(t *testing.T) { + // In this test, we just stand up an endpoint that does not + // do dRPC. We'll try to connect, fail to websocket upgrade + // and eventually give up. + t.Run("V2/Err", func(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - ld := &fakeLogDest{t: t} - ls := agentsdk.NewLogSender(slogtest.Make(t, nil)) - logger, closer := sendLogsV2(ctx, ld, ls, slogtest.Make(t, nil)) - defer closer() - - logger(LevelInfo, "1") - logger(LevelInfo, "2") - closer() - logger(LevelInfo, "3") + token := uuid.NewString() + handlerDone := make(chan struct{}) + var closeOnce sync.Once + handler := func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v2/buildinfo" { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"version": "v2.9.0"}`)) + return + } + defer closeOnce.Do(func() { close(handlerDone) }) + w.WriteHeader(http.StatusOK) + } + srv := httptest.NewServer(http.HandlerFunc(handler)) + defer srv.Close() - require.Len(t, ld.logs, 2) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + u, err := url.Parse(srv.URL) + require.NoError(t, err) + _, _, err = Coder(ctx, u, token) + require.ErrorContains(t, err, "failed to WebSocket dial") + require.ErrorIs(t, err, context.DeadlineExceeded) + <-handlerDone }) // In this test, we validate that a 401 error on the initial connect // results in a retry. When envbuilder initially attempts to connect // using the Coder agent token, the workspace build may not yet have // completed. - t.Run("V2/Retry", func(t *testing.T) { + t.Run("V2Retry", func(t *testing.T) { t.Parallel() ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -238,99 +221,6 @@ func TestCoder(t *testing.T) { }) } -//nolint:paralleltest // We need to replace a global timeout. -func TestCoderRPCTimeout(t *testing.T) { - // This timeout is picked with the current subtests in mind, it - // should not be changed without good reason. - testReplaceTimeout(t, &rpcConnectTimeout, 500*time.Millisecond) - - // In this test, we just stand up an endpoint that does not - // do dRPC. We'll try to connect, fail to websocket upgrade - // and eventually give up after rpcConnectTimeout. - t.Run("V2/Err", func(t *testing.T) { - t.Parallel() - - token := uuid.NewString() - handlerDone := make(chan struct{}) - handlerWait := make(chan struct{}) - var closeOnce sync.Once - handler := func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/api/v2/buildinfo" { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"version": "v2.9.0"}`)) - return - } - defer closeOnce.Do(func() { close(handlerDone) }) - <-handlerWait - w.WriteHeader(http.StatusOK) - } - srv := httptest.NewServer(http.HandlerFunc(handler)) - defer srv.Close() - - ctx, cancel := context.WithTimeout(context.Background(), rpcConnectTimeout/2) - defer cancel() - u, err := url.Parse(srv.URL) - require.NoError(t, err) - _, _, err = Coder(ctx, u, token) - require.ErrorContains(t, err, "failed to WebSocket dial") - require.ErrorIs(t, err, context.DeadlineExceeded) - close(handlerWait) - <-handlerDone - }) - - t.Run("V2/Timeout", func(t *testing.T) { - t.Parallel() - - token := uuid.NewString() - handlerDone := make(chan struct{}) - handlerWait := make(chan struct{}) - var closeOnce sync.Once - handler := func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/api/v2/buildinfo" { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"version": "v2.9.0"}`)) - return - } - defer closeOnce.Do(func() { close(handlerDone) }) - <-handlerWait - w.WriteHeader(http.StatusOK) - } - srv := httptest.NewServer(http.HandlerFunc(handler)) - defer srv.Close() - - ctx, cancel := context.WithTimeout(context.Background(), rpcConnectTimeout*2) - defer cancel() - u, err := url.Parse(srv.URL) - require.NoError(t, err) - _, _, err = Coder(ctx, u, token) - require.ErrorContains(t, err, "failed to WebSocket dial") - require.ErrorIs(t, err, context.DeadlineExceeded) - close(handlerWait) - <-handlerDone - }) -} - -func decodeV1Logs(t *testing.T, w http.ResponseWriter, r *http.Request) (agentsdk.PatchLogs, bool) { - t.Helper() - var req agentsdk.PatchLogs - err := json.NewDecoder(r.Body).Decode(&req) - if !assert.NoError(t, err) { - http.Error(w, err.Error(), http.StatusBadRequest) - return req, false - } - return req, true -} - -func newCoderLogger(ctx context.Context, t *testing.T, us string, token string) (Func, func()) { - t.Helper() - u, err := url.Parse(us) - require.NoError(t, err) - logger, closer, err := Coder(ctx, u, token) - require.NoError(t, err) - t.Cleanup(closer) - return logger, closer -} - type fakeLogDest struct { t testing.TB logs []*proto.Log @@ -341,27 +231,3 @@ func (d *fakeLogDest) BatchCreateLogs(ctx context.Context, request *proto.BatchC d.logs = append(d.logs, request.Logs...) return &proto.BatchCreateLogsResponse{}, nil } - -func testReplaceTimeout(t *testing.T, v *time.Duration, d time.Duration) { - t.Helper() - if isParallel(t) { - t.Fatal("cannot replace timeout in parallel test") - } - old := *v - *v = d - t.Cleanup(func() { *v = old }) -} - -func isParallel(t *testing.T) (ret bool) { - t.Helper() - // This is a hack to determine if the test is running in parallel - // via property of t.Setenv. - defer func() { - if r := recover(); r != nil { - ret = true - } - }() - // Random variable name to avoid collisions. - t.Setenv(fmt.Sprintf("__TEST_CHECK_IS_PARALLEL_%d", rand.Int()), "1") - return false -} From ef458280148fdc78e9f867b5f92587982c6be33a Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Mon, 30 Sep 2024 20:06:46 +0300 Subject: [PATCH 66/81] reapply: "fix: refactor coder logger to allow flush without deadlock (#375)" (#377) (cherry picked from commit b3d1ec837b8f81cc16a35d3c43d6c2268ca1410d) --- cmd/envbuilder/main.go | 15 ++- envbuilder.go | 7 +- go.mod | 2 +- integration/integration_test.go | 67 ++++++++++ log/coder.go | 115 +++++++++++------- log/coder_internal_test.go | 208 ++++++++++++++++++++++++++------ 6 files changed, 328 insertions(+), 86 deletions(-) diff --git a/cmd/envbuilder/main.go b/cmd/envbuilder/main.go index 410e0897..e8dc2201 100644 --- a/cmd/envbuilder/main.go +++ b/cmd/envbuilder/main.go @@ -37,6 +37,15 @@ func envbuilderCmd() serpent.Command { Options: o.CLI(), Handler: func(inv *serpent.Invocation) error { o.SetDefaults() + var preExecs []func() + preExec := func() { + for _, fn := range preExecs { + fn() + } + preExecs = nil + } + defer preExec() // Ensure cleanup in case of error. + o.Logger = log.New(os.Stderr, o.Verbose) if o.CoderAgentURL != "" { if o.CoderAgentToken == "" { @@ -49,7 +58,9 @@ func envbuilderCmd() serpent.Command { coderLog, closeLogs, err := log.Coder(inv.Context(), u, o.CoderAgentToken) if err == nil { o.Logger = log.Wrap(o.Logger, coderLog) - defer closeLogs() + preExecs = append(preExecs, func() { + closeLogs() + }) // This adds the envbuilder subsystem. // If telemetry is enabled in a Coder deployment, // this will be reported and help us understand @@ -78,7 +89,7 @@ func envbuilderCmd() serpent.Command { return nil } - err := envbuilder.Run(inv.Context(), o) + err := envbuilder.Run(inv.Context(), o, preExec) if err != nil { o.Logger(log.LevelError, "error: %s", err) } diff --git a/envbuilder.go b/envbuilder.go index 683f6a54..94998165 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -84,7 +84,9 @@ type execArgsInfo struct { // Logger is the logf to use for all operations. // Filesystem is the filesystem to use for all operations. // Defaults to the host filesystem. -func Run(ctx context.Context, opts options.Options) error { +// preExec are any functions that should be called before exec'ing the init +// command. This is useful for ensuring that defers get run. +func Run(ctx context.Context, opts options.Options, preExec ...func()) error { var args execArgsInfo // Run in a separate function to ensure all defers run before we // setuid or exec. @@ -103,6 +105,9 @@ func Run(ctx context.Context, opts options.Options) error { } opts.Logger(log.LevelInfo, "=== Running the init command %s %+v as the %q user...", opts.InitCommand, args.InitArgs, args.UserInfo.user.Username) + for _, fn := range preExec { + fn() + } err = syscall.Exec(args.InitCommand, append([]string{args.InitCommand}, args.InitArgs...), args.Environ) if err != nil { diff --git a/go.mod b/go.mod index b3fa7843..9fa1d696 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/gliderlabs/ssh v0.3.7 github.com/go-git/go-billy/v5 v5.5.0 github.com/go-git/go-git/v5 v5.12.0 + github.com/google/go-cmp v0.6.0 github.com/google/go-containerregistry v0.20.1 github.com/google/uuid v1.6.0 github.com/hashicorp/go-multierror v1.1.1 @@ -149,7 +150,6 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.2 // indirect - github.com/google/go-cmp v0.6.0 // indirect github.com/google/nftables v0.2.0 // indirect github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect github.com/gorilla/handlers v1.5.1 // indirect diff --git a/integration/integration_test.go b/integration/integration_test.go index 66dfe846..cfe1de49 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -23,6 +23,8 @@ import ( "testing" "time" + "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/codersdk/agentsdk" "github.com/coder/envbuilder" "github.com/coder/envbuilder/devcontainer/features" "github.com/coder/envbuilder/internal/magicdir" @@ -58,6 +60,71 @@ const ( testImageUbuntu = "localhost:5000/envbuilder-test-ubuntu:latest" ) +func TestLogs(t *testing.T) { + t.Parallel() + + token := uuid.NewString() + logsDone := make(chan struct{}) + + logHandler := func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v2/buildinfo": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"version": "v2.8.9"}`)) + return + case "/api/v2/workspaceagents/me/logs": + w.WriteHeader(http.StatusOK) + tokHdr := r.Header.Get(codersdk.SessionTokenHeader) + assert.Equal(t, token, tokHdr) + var req agentsdk.PatchLogs + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + for _, log := range req.Logs { + t.Logf("got log: %+v", log) + if strings.Contains(log.Output, "Running the init command") { + close(logsDone) + return + } + } + return + default: + t.Errorf("unexpected request to %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + return + } + } + logSrv := httptest.NewServer(http.HandlerFunc(logHandler)) + defer logSrv.Close() + + // Ensures that a Git repository with a devcontainer.json is cloned and built. + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ + "devcontainer.json": `{ + "build": { + "dockerfile": "Dockerfile" + }, + }`, + "Dockerfile": fmt.Sprintf(`FROM %s`, testImageUbuntu), + }, + }) + _, err := runEnvbuilder(t, runOpts{env: []string{ + envbuilderEnv("GIT_URL", srv.URL), + "CODER_AGENT_URL=" + logSrv.URL, + "CODER_AGENT_TOKEN=" + token, + }}) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + select { + case <-ctx.Done(): + t.Fatal("timed out waiting for logs") + case <-logsDone: + } +} + func TestInitScriptInitCommand(t *testing.T) { t.Parallel() diff --git a/log/coder.go b/log/coder.go index d8b4fe0d..d31092d5 100644 --- a/log/coder.go +++ b/log/coder.go @@ -6,6 +6,7 @@ import ( "fmt" "net/url" "os" + "sync" "time" "cdr.dev/slog" @@ -27,13 +28,14 @@ var ( minAgentAPIV2 = "v2.9" ) -// Coder establishes a connection to the Coder instance located at -// coderURL and authenticates using token. It then establishes a -// dRPC connection to the Agent API and begins sending logs. -// If the version of Coder does not support the Agent API, it will -// fall back to using the PatchLogs endpoint. -// The returned function is used to block until all logs are sent. -func Coder(ctx context.Context, coderURL *url.URL, token string) (Func, func(), error) { +// Coder establishes a connection to the Coder instance located at coderURL and +// authenticates using token. It then establishes a dRPC connection to the Agent +// API and begins sending logs. If the version of Coder does not support the +// Agent API, it will fall back to using the PatchLogs endpoint. The closer is +// used to close the logger and to wait at most logSendGracePeriod for logs to +// be sent. Cancelling the context will close the logs immediately without +// waiting for logs to be sent. +func Coder(ctx context.Context, coderURL *url.URL, token string) (logger Func, closer func(), err error) { // To troubleshoot issues, we need some way of logging. metaLogger := slog.Make(sloghuman.Sink(os.Stderr)) defer metaLogger.Sync() @@ -44,9 +46,19 @@ func Coder(ctx context.Context, coderURL *url.URL, token string) (Func, func(), } if semver.Compare(semver.MajorMinor(bi.Version), minAgentAPIV2) < 0 { metaLogger.Warn(ctx, "Detected Coder version incompatible with AgentAPI v2, falling back to deprecated API", slog.F("coder_version", bi.Version)) - sendLogs, flushLogs := sendLogsV1(ctx, client, metaLogger.Named("send_logs_v1")) - return sendLogs, flushLogs, nil + logger, closer = sendLogsV1(ctx, client, metaLogger.Named("send_logs_v1")) + return logger, closer, nil } + + // Create a new context so we can ensure the connection is torn down. + ctx, cancel := context.WithCancel(ctx) + defer func() { + if err != nil { + cancel() + } + }() + // Note that ctx passed to initRPC will be inherited by the + // underlying connection, nothing we can do about that here. dac, err := initRPC(ctx, client, metaLogger.Named("init_rpc")) if err != nil { // Logged externally @@ -54,8 +66,19 @@ func Coder(ctx context.Context, coderURL *url.URL, token string) (Func, func(), } ls := agentsdk.NewLogSender(metaLogger.Named("coder_log_sender")) metaLogger.Warn(ctx, "Sending logs via AgentAPI v2", slog.F("coder_version", bi.Version)) - sendLogs, doneFunc := sendLogsV2(ctx, dac, ls, metaLogger.Named("send_logs_v2")) - return sendLogs, doneFunc, nil + logger, loggerCloser := sendLogsV2(ctx, dac, ls, metaLogger.Named("send_logs_v2")) + var closeOnce sync.Once + closer = func() { + loggerCloser() + + closeOnce.Do(func() { + // Typically cancel would be after Close, but we want to be + // sure there's nothing that might block on Close. + cancel() + _ = dac.DRPCConn().Close() + }) + } + return logger, closer, nil } type coderLogSender interface { @@ -74,7 +97,7 @@ func initClient(coderURL *url.URL, token string) *agentsdk.Client { func initRPC(ctx context.Context, client *agentsdk.Client, l slog.Logger) (proto.DRPCAgentClient20, error) { var c proto.DRPCAgentClient20 var err error - retryCtx, retryCancel := context.WithTimeout(context.Background(), rpcConnectTimeout) + retryCtx, retryCancel := context.WithTimeout(ctx, rpcConnectTimeout) defer retryCancel() attempts := 0 for r := retry.New(100*time.Millisecond, time.Second); r.Wait(retryCtx); { @@ -95,65 +118,67 @@ func initRPC(ctx context.Context, client *agentsdk.Client, l slog.Logger) (proto // sendLogsV1 uses the PatchLogs endpoint to send logs. // This is deprecated, but required for backward compatibility with older versions of Coder. -func sendLogsV1(ctx context.Context, client *agentsdk.Client, l slog.Logger) (Func, func()) { +func sendLogsV1(ctx context.Context, client *agentsdk.Client, l slog.Logger) (logger Func, closer func()) { // nolint: staticcheck // required for backwards compatibility - sendLogs, flushLogs := agentsdk.LogsSender(agentsdk.ExternalLogSourceID, client.PatchLogs, slog.Logger{}) + sendLog, flushAndClose := agentsdk.LogsSender(agentsdk.ExternalLogSourceID, client.PatchLogs, slog.Logger{}) + var mu sync.Mutex return func(lvl Level, msg string, args ...any) { log := agentsdk.Log{ CreatedAt: time.Now(), Output: fmt.Sprintf(msg, args...), Level: codersdk.LogLevel(lvl), } - if err := sendLogs(ctx, log); err != nil { + mu.Lock() + defer mu.Unlock() + if err := sendLog(ctx, log); err != nil { l.Warn(ctx, "failed to send logs to Coder", slog.Error(err)) } }, func() { - if err := flushLogs(ctx); err != nil { + ctx, cancel := context.WithTimeout(ctx, logSendGracePeriod) + defer cancel() + if err := flushAndClose(ctx); err != nil { l.Warn(ctx, "failed to flush logs", slog.Error(err)) } } } // sendLogsV2 uses the v2 agent API to send logs. Only compatibile with coder versions >= 2.9. -func sendLogsV2(ctx context.Context, dest agentsdk.LogDest, ls coderLogSender, l slog.Logger) (Func, func()) { +func sendLogsV2(ctx context.Context, dest agentsdk.LogDest, ls coderLogSender, l slog.Logger) (logger Func, closer func()) { + sendCtx, sendCancel := context.WithCancel(ctx) done := make(chan struct{}) uid := uuid.New() go func() { defer close(done) - if err := ls.SendLoop(ctx, dest); err != nil { + if err := ls.SendLoop(sendCtx, dest); err != nil { if !errors.Is(err, context.Canceled) { l.Warn(ctx, "failed to send logs to Coder", slog.Error(err)) } } - - // Wait for up to 10 seconds for logs to finish sending. - sendCtx, sendCancel := context.WithTimeout(context.Background(), logSendGracePeriod) - defer sendCancel() - // Try once more to send any pending logs - if err := ls.SendLoop(sendCtx, dest); err != nil { - if !errors.Is(err, context.DeadlineExceeded) { - l.Warn(ctx, "failed to send remaining logs to Coder", slog.Error(err)) - } - } - ls.Flush(uid) - if err := ls.WaitUntilEmpty(sendCtx); err != nil { - if !errors.Is(err, context.DeadlineExceeded) { - l.Warn(ctx, "log sender did not empty", slog.Error(err)) - } - } }() - logFunc := func(l Level, msg string, args ...any) { - ls.Enqueue(uid, agentsdk.Log{ - CreatedAt: time.Now(), - Output: fmt.Sprintf(msg, args...), - Level: codersdk.LogLevel(l), - }) - } + var closeOnce sync.Once + return func(l Level, msg string, args ...any) { + ls.Enqueue(uid, agentsdk.Log{ + CreatedAt: time.Now(), + Output: fmt.Sprintf(msg, args...), + Level: codersdk.LogLevel(l), + }) + }, func() { + closeOnce.Do(func() { + // Trigger a flush and wait for logs to be sent. + ls.Flush(uid) + ctx, cancel := context.WithTimeout(ctx, logSendGracePeriod) + defer cancel() + err := ls.WaitUntilEmpty(ctx) + if err != nil { + l.Warn(ctx, "log sender did not empty", slog.Error(err)) + } - doneFunc := func() { - <-done - } + // Stop the send loop. + sendCancel() + }) - return logFunc, doneFunc + // Wait for the send loop to finish. + <-done + } } diff --git a/log/coder_internal_test.go b/log/coder_internal_test.go index 4895150e..8b8bb632 100644 --- a/log/coder_internal_test.go +++ b/log/coder_internal_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "math/rand" "net/http" "net/http/httptest" "net/url" @@ -38,10 +39,8 @@ func TestCoder(t *testing.T) { defer closeOnce.Do(func() { close(gotLogs) }) tokHdr := r.Header.Get(codersdk.SessionTokenHeader) assert.Equal(t, token, tokHdr) - var req agentsdk.PatchLogs - err := json.NewDecoder(r.Body).Decode(&req) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) + req, ok := decodeV1Logs(t, w, r) + if !ok { return } if assert.Len(t, req.Logs, 1) { @@ -54,15 +53,44 @@ func TestCoder(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - u, err := url.Parse(srv.URL) - require.NoError(t, err) - log, closeLog, err := Coder(ctx, u, token) - require.NoError(t, err) - defer closeLog() - log(LevelInfo, "hello %s", "world") + + logger, _ := newCoderLogger(ctx, t, srv.URL, token) + logger(LevelInfo, "hello %s", "world") <-gotLogs }) + t.Run("V1/Close", func(t *testing.T) { + t.Parallel() + + var got []agentsdk.Log + handler := func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v2/buildinfo" { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"version": "v2.8.9"}`)) + return + } + req, ok := decodeV1Logs(t, w, r) + if !ok { + return + } + got = append(got, req.Logs...) + } + srv := httptest.NewServer(http.HandlerFunc(handler)) + defer srv.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + logger, closer := newCoderLogger(ctx, t, srv.URL, uuid.NewString()) + logger(LevelInfo, "1") + logger(LevelInfo, "2") + closer() + logger(LevelInfo, "3") + require.Len(t, got, 2) + assert.Equal(t, "1", got[0].Output) + assert.Equal(t, "2", got[1].Output) + }) + t.Run("V1/ErrUnauthorized", func(t *testing.T) { t.Parallel() @@ -140,42 +168,31 @@ func TestCoder(t *testing.T) { require.Len(t, ld.logs, 10) }) - // In this test, we just stand up an endpoint that does not - // do dRPC. We'll try to connect, fail to websocket upgrade - // and eventually give up. - t.Run("V2/Err", func(t *testing.T) { + // In this test, we just fake out the DRPC server. + t.Run("V2/Close", func(t *testing.T) { t.Parallel() - token := uuid.NewString() - handlerDone := make(chan struct{}) - var closeOnce sync.Once - handler := func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/api/v2/buildinfo" { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"version": "v2.9.0"}`)) - return - } - defer closeOnce.Do(func() { close(handlerDone) }) - w.WriteHeader(http.StatusOK) - } - srv := httptest.NewServer(http.HandlerFunc(handler)) - defer srv.Close() - - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() - u, err := url.Parse(srv.URL) - require.NoError(t, err) - _, _, err = Coder(ctx, u, token) - require.ErrorContains(t, err, "failed to WebSocket dial") - require.ErrorIs(t, err, context.DeadlineExceeded) - <-handlerDone + + ld := &fakeLogDest{t: t} + ls := agentsdk.NewLogSender(slogtest.Make(t, nil)) + logger, closer := sendLogsV2(ctx, ld, ls, slogtest.Make(t, nil)) + defer closer() + + logger(LevelInfo, "1") + logger(LevelInfo, "2") + closer() + logger(LevelInfo, "3") + + require.Len(t, ld.logs, 2) }) // In this test, we validate that a 401 error on the initial connect // results in a retry. When envbuilder initially attempts to connect // using the Coder agent token, the workspace build may not yet have // completed. - t.Run("V2Retry", func(t *testing.T) { + t.Run("V2/Retry", func(t *testing.T) { t.Parallel() ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -221,6 +238,99 @@ func TestCoder(t *testing.T) { }) } +//nolint:paralleltest // We need to replace a global timeout. +func TestCoderRPCTimeout(t *testing.T) { + // This timeout is picked with the current subtests in mind, it + // should not be changed without good reason. + testReplaceTimeout(t, &rpcConnectTimeout, 500*time.Millisecond) + + // In this test, we just stand up an endpoint that does not + // do dRPC. We'll try to connect, fail to websocket upgrade + // and eventually give up after rpcConnectTimeout. + t.Run("V2/Err", func(t *testing.T) { + t.Parallel() + + token := uuid.NewString() + handlerDone := make(chan struct{}) + handlerWait := make(chan struct{}) + var closeOnce sync.Once + handler := func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v2/buildinfo" { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"version": "v2.9.0"}`)) + return + } + defer closeOnce.Do(func() { close(handlerDone) }) + <-handlerWait + w.WriteHeader(http.StatusOK) + } + srv := httptest.NewServer(http.HandlerFunc(handler)) + defer srv.Close() + + ctx, cancel := context.WithTimeout(context.Background(), rpcConnectTimeout/2) + defer cancel() + u, err := url.Parse(srv.URL) + require.NoError(t, err) + _, _, err = Coder(ctx, u, token) + require.ErrorContains(t, err, "failed to WebSocket dial") + require.ErrorIs(t, err, context.DeadlineExceeded) + close(handlerWait) + <-handlerDone + }) + + t.Run("V2/Timeout", func(t *testing.T) { + t.Parallel() + + token := uuid.NewString() + handlerDone := make(chan struct{}) + handlerWait := make(chan struct{}) + var closeOnce sync.Once + handler := func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v2/buildinfo" { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"version": "v2.9.0"}`)) + return + } + defer closeOnce.Do(func() { close(handlerDone) }) + <-handlerWait + w.WriteHeader(http.StatusOK) + } + srv := httptest.NewServer(http.HandlerFunc(handler)) + defer srv.Close() + + ctx, cancel := context.WithTimeout(context.Background(), rpcConnectTimeout*2) + defer cancel() + u, err := url.Parse(srv.URL) + require.NoError(t, err) + _, _, err = Coder(ctx, u, token) + require.ErrorContains(t, err, "failed to WebSocket dial") + require.ErrorIs(t, err, context.DeadlineExceeded) + close(handlerWait) + <-handlerDone + }) +} + +func decodeV1Logs(t *testing.T, w http.ResponseWriter, r *http.Request) (agentsdk.PatchLogs, bool) { + t.Helper() + var req agentsdk.PatchLogs + err := json.NewDecoder(r.Body).Decode(&req) + if !assert.NoError(t, err) { + http.Error(w, err.Error(), http.StatusBadRequest) + return req, false + } + return req, true +} + +func newCoderLogger(ctx context.Context, t *testing.T, us string, token string) (Func, func()) { + t.Helper() + u, err := url.Parse(us) + require.NoError(t, err) + logger, closer, err := Coder(ctx, u, token) + require.NoError(t, err) + t.Cleanup(closer) + return logger, closer +} + type fakeLogDest struct { t testing.TB logs []*proto.Log @@ -231,3 +341,27 @@ func (d *fakeLogDest) BatchCreateLogs(ctx context.Context, request *proto.BatchC d.logs = append(d.logs, request.Logs...) return &proto.BatchCreateLogsResponse{}, nil } + +func testReplaceTimeout(t *testing.T, v *time.Duration, d time.Duration) { + t.Helper() + if isParallel(t) { + t.Fatal("cannot replace timeout in parallel test") + } + old := *v + *v = d + t.Cleanup(func() { *v = old }) +} + +func isParallel(t *testing.T) (ret bool) { + t.Helper() + // This is a hack to determine if the test is running in parallel + // via property of t.Setenv. + defer func() { + if r := recover(); r != nil { + ret = true + } + }() + // Random variable name to avoid collisions. + t.Setenv(fmt.Sprintf("__TEST_CHECK_IS_PARALLEL_%d", rand.Int()), "1") + return false +} From 0ec371f0023e3e6130942ccc25f0bc030793d139 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Tue, 1 Oct 2024 11:12:09 +0300 Subject: [PATCH 67/81] fix: quote output of init command and args to prevent multiline log (#378) (cherry picked from commit a9cb987173150f26f6fcb2b32983da7f8d6001d4) --- envbuilder.go | 2 +- integration/integration_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 94998165..47cc228d 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -104,7 +104,7 @@ func Run(ctx context.Context, opts options.Options, preExec ...func()) error { return fmt.Errorf("set uid: %w", err) } - opts.Logger(log.LevelInfo, "=== Running the init command %s %+v as the %q user...", opts.InitCommand, args.InitArgs, args.UserInfo.user.Username) + opts.Logger(log.LevelInfo, "=== Running init command as user %q: %q", args.UserInfo.user.Username, append([]string{opts.InitCommand}, args.InitArgs...)) for _, fn := range preExec { fn() } diff --git a/integration/integration_test.go b/integration/integration_test.go index cfe1de49..b7332c04 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -84,7 +84,7 @@ func TestLogs(t *testing.T) { } for _, log := range req.Logs { t.Logf("got log: %+v", log) - if strings.Contains(log.Output, "Running the init command") { + if strings.Contains(log.Output, "Running init command") { close(logsDone) return } @@ -2294,7 +2294,7 @@ func runEnvbuilder(t *testing.T, opts runOpts) (string, error) { logChan, errChan := streamContainerLogs(t, cli, ctr.ID) go func() { for log := range logChan { - if strings.HasPrefix(log, "=== Running the init command") { + if strings.HasPrefix(log, "=== Running init command") { errChan <- nil return } From 3759642dd2447027ed836e9b0afa2b42320a77d4 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Wed, 2 Oct 2024 12:51:27 +0300 Subject: [PATCH 68/81] fix(scripts): always return the latest tag in version.sh (#379) (cherry picked from commit 46c97a46b96f3ca12f9718766b7eb610e0f1e80d) --- scripts/version.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/version.sh b/scripts/version.sh index 75dafcc4..5666a2b6 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -36,11 +36,11 @@ if [[ -z ${tag_list} ]]; then else current_commit=$(git rev-parse HEAD) # Try to find the last tag that contains the current commit - last_tag=$(git tag --contains "$current_commit" --sort=version:refname | head -n 1) + last_tag=$(git tag --contains "$current_commit" --sort=-version:refname | head -n 1) # If there is no tag that contains the current commit, # get the latest tag sorted by semver. if [[ -z "${last_tag}" ]]; then - last_tag=$(git tag --sort=version:refname | tail -n 1) + last_tag=$(git tag --sort=-version:refname | head -n 1) fi fi From 0096b04834cb97b4889e87c2264bbae18e755afe Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Fri, 4 Oct 2024 14:54:15 +0300 Subject: [PATCH 69/81] fix: extract folder correctly from repo URLs with trailing slash (#381) Closes #380 Closes coder/customers#690 (cherry picked from commit f50ec7ad7a16f8f084961a834bee5b1323ef2aa5) --- integration/integration_test.go | 8 +++-- options/defaults.go | 11 ++++--- options/defaults_test.go | 57 +++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 7 deletions(-) diff --git a/integration/integration_test.go b/integration/integration_test.go index b7332c04..7fd732d6 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -844,15 +844,17 @@ func TestContainerEnv(t *testing.T) { require.NoError(t, err) output := execContainer(t, ctr, "cat /env") - require.Contains(t, strings.TrimSpace(output), - `DEVCONTAINER=true + want := `DEVCONTAINER=true DEVCONTAINER_CONFIG=/workspaces/empty/.devcontainer/devcontainer.json ENVBUILDER=true FROM_CONTAINER_ENV=bar FROM_DOCKERFILE=foo FROM_REMOTE_ENV=baz PATH=/usr/local/bin:/bin:/go/bin:/opt -REMOTE_BAR=bar`) +REMOTE_BAR=bar` + if diff := cmp.Diff(want, strings.TrimSpace(output)); diff != "" { + require.Failf(t, "env mismatch", "diff (-want +got):\n%s", diff) + } } func TestUnsetOptionsEnv(t *testing.T) { diff --git a/options/defaults.go b/options/defaults.go index 220480d8..2d6cd52f 100644 --- a/options/defaults.go +++ b/options/defaults.go @@ -2,6 +2,7 @@ package options import ( "fmt" + "path" "strings" "github.com/go-git/go-billy/v5/osfs" @@ -25,12 +26,14 @@ func DefaultWorkspaceFolder(repoURL string) string { if err != nil { return EmptyWorkspaceDir } - name := strings.Split(parsed.Path, "/") - hasOwnerAndRepo := len(name) >= 2 - if !hasOwnerAndRepo { + repo := path.Base(parsed.Path) + // Giturls parsing never actually fails since ParseLocal never + // errors and places the entire URL in the Path field. This check + // ensures it's at least a Unix path containing forwardslash. + if repo == repoURL || repo == "/" || repo == "." || repo == "" { return EmptyWorkspaceDir } - repo := strings.TrimSuffix(name[len(name)-1], ".git") + repo = strings.TrimSuffix(repo, ".git") return fmt.Sprintf("/workspaces/%s", repo) } diff --git a/options/defaults_test.go b/options/defaults_test.go index 4387c084..de57365d 100644 --- a/options/defaults_test.go +++ b/options/defaults_test.go @@ -35,11 +35,56 @@ func TestDefaultWorkspaceFolder(t *testing.T) { gitURL: "https://username:password@github.com/coder/envbuilder.git", expected: "/workspaces/envbuilder", }, + { + name: "trailing", + gitURL: "https://github.com/coder/envbuilder.git/", + expected: "/workspaces/envbuilder", + }, + { + name: "trailing-x2", + gitURL: "https://github.com/coder/envbuilder.git//", + expected: "/workspaces/envbuilder", + }, + { + name: "no .git", + gitURL: "https://github.com/coder/envbuilder", + expected: "/workspaces/envbuilder", + }, + { + name: "trailing no .git", + gitURL: "https://github.com/coder/envbuilder/", + expected: "/workspaces/envbuilder", + }, { name: "fragment", gitURL: "https://github.com/coder/envbuilder.git#feature-branch", expected: "/workspaces/envbuilder", }, + { + name: "fragment-trailing", + gitURL: "https://github.com/coder/envbuilder.git/#refs/heads/feature-branch", + expected: "/workspaces/envbuilder", + }, + { + name: "fragment-trailing no .git", + gitURL: "https://github.com/coder/envbuilder/#refs/heads/feature-branch", + expected: "/workspaces/envbuilder", + }, + { + name: "space", + gitURL: "https://github.com/coder/env%20builder.git", + expected: "/workspaces/env builder", + }, + { + name: "Unix path", + gitURL: "/repo", + expected: "/workspaces/repo", + }, + { + name: "Unix subpath", + gitURL: "/path/to/repo", + expected: "/workspaces/repo", + }, { name: "empty", gitURL: "", @@ -65,6 +110,18 @@ func TestDefaultWorkspaceFolder(t *testing.T) { name: "website URL", invalidURL: "www.google.com", }, + { + name: "Unix root", + invalidURL: "/", + }, + { + name: "Path consists entirely of slash", + invalidURL: "//", + }, + { + name: "Git URL with no path", + invalidURL: "http://127.0.0.1:41073", + }, } for _, tt := range invalidTests { t.Run(tt.name, func(t *testing.T) { From 7756b145940220f652aa35b421054e8116dc7163 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 15 Oct 2024 10:05:11 +0100 Subject: [PATCH 70/81] chore(README.md): add documentation regarding target user (#386) (cherry picked from commit d14ab3d95830c830a71c2f1dcbf346667e738c43) --- README.md | 1 + docs/users.md | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 docs/users.md diff --git a/README.md b/README.md index af5323de..0a54619e 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ To explore more examples, tips, and advanced usage, check out the following guid - [Git Authentication](./docs/git-auth.md) - [Caching](./docs/caching.md) - [Custom Certificates](./docs/custom-certificates.md) +- [Users](./docs/users.md) ## Setup Script diff --git a/docs/users.md b/docs/users.md new file mode 100644 index 00000000..6f121cdf --- /dev/null +++ b/docs/users.md @@ -0,0 +1,9 @@ +# Root Privileges + +Envbuilder always expects to be run as `root` in its container, as building an image will most likely require root privileges. Once the image is built, Envbuilder will drop root privileges and `exec` `ENVBUILDER_INIT_COMMAND` / `ENVBUILDER_INIT_SCRIPT` as a non-root user. + +## Choosing a target user + +Envbuilder will first attempt to switch to the `containerUser` defined `devcontainer.json`. +If this is not specified, it will look up the last `USER` directive from the specified `Dockerfile` or image. +If no alternative user is specified, Envbuilder will fallback to `root`. From 179f3c875472b8cac30fb3f0b103ad6520004870 Mon Sep 17 00:00:00 2001 From: Sas Swart Date: Tue, 15 Oct 2024 11:53:51 +0200 Subject: [PATCH 71/81] chore(envbuilder.go): rename the concept of a magicDir to workingDir (#388) (cherry picked from commit aba2f46306be70c4817976c6f40fb1673510eb1d) --- devcontainer/devcontainer_test.go | 14 ++-- envbuilder.go | 68 +++++++++---------- integration/integration_test.go | 4 +- internal/magicdir/magicdir_internal_test.go | 38 ----------- .../magicdir.go => workingdir/workingdir.go} | 38 +++++------ .../workingdir/workingdir_internal_test.go | 38 +++++++++++ options/defaults.go | 4 +- 7 files changed, 102 insertions(+), 102 deletions(-) delete mode 100644 internal/magicdir/magicdir_internal_test.go rename internal/{magicdir/magicdir.go => workingdir/workingdir.go} (67%) create mode 100644 internal/workingdir/workingdir_internal_test.go diff --git a/devcontainer/devcontainer_test.go b/devcontainer/devcontainer_test.go index 923680b9..4a475682 100644 --- a/devcontainer/devcontainer_test.go +++ b/devcontainer/devcontainer_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/require" ) -const magicDir = "/.envbuilder" +const workingDir = "/.envbuilder" func stubLookupEnv(string) (string, bool) { return "", false @@ -98,7 +98,7 @@ func TestCompileWithFeatures(t *testing.T) { featureTwoDir := fmt.Sprintf("/.envbuilder/features/two-%x", featureTwoMD5[:4]) t.Run("WithoutBuildContexts", func(t *testing.T) { - params, err := dc.Compile(fs, "", magicDir, "", "", false, stubLookupEnv) + params, err := dc.Compile(fs, "", workingDir, "", "", false, stubLookupEnv) require.NoError(t, err) require.Equal(t, `FROM localhost:5000/envbuilder-test-codercom-code-server:latest @@ -116,7 +116,7 @@ USER 1000`, params.DockerfileContent) }) t.Run("WithBuildContexts", func(t *testing.T) { - params, err := dc.Compile(fs, "", magicDir, "", "", true, stubLookupEnv) + params, err := dc.Compile(fs, "", workingDir, "", "", true, stubLookupEnv) require.NoError(t, err) registryHost := strings.TrimPrefix(registry, "http://") @@ -155,10 +155,10 @@ func TestCompileDevContainer(t *testing.T) { dc := &devcontainer.Spec{ Image: "localhost:5000/envbuilder-test-ubuntu:latest", } - params, err := dc.Compile(fs, "", magicDir, "", "", false, stubLookupEnv) + params, err := dc.Compile(fs, "", workingDir, "", "", false, stubLookupEnv) require.NoError(t, err) - require.Equal(t, filepath.Join(magicDir, "Dockerfile"), params.DockerfilePath) - require.Equal(t, magicDir, params.BuildContext) + require.Equal(t, filepath.Join(workingDir, "Dockerfile"), params.DockerfilePath) + require.Equal(t, workingDir, params.BuildContext) }) t.Run("WithBuild", func(t *testing.T) { t.Parallel() @@ -181,7 +181,7 @@ func TestCompileDevContainer(t *testing.T) { _, err = io.WriteString(file, "FROM localhost:5000/envbuilder-test-ubuntu:latest") require.NoError(t, err) _ = file.Close() - params, err := dc.Compile(fs, dcDir, magicDir, "", "/var/workspace", false, stubLookupEnv) + params, err := dc.Compile(fs, dcDir, workingDir, "", "/var/workspace", false, stubLookupEnv) require.NoError(t, err) require.Equal(t, "ARG1=value1", params.BuildArgs[0]) require.Equal(t, "ARG2=workspace", params.BuildArgs[1]) diff --git a/envbuilder.go b/envbuilder.go index 47cc228d..75829c2f 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -35,7 +35,7 @@ import ( "github.com/GoogleContainerTools/kaniko/pkg/util" "github.com/coder/envbuilder/devcontainer" "github.com/coder/envbuilder/internal/ebutil" - "github.com/coder/envbuilder/internal/magicdir" + "github.com/coder/envbuilder/internal/workingdir" "github.com/coder/envbuilder/log" "github.com/containerd/platforms" "github.com/distribution/distribution/v3/configuration" @@ -120,7 +120,7 @@ func Run(ctx context.Context, opts options.Options, preExec ...func()) error { func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) error { defer options.UnsetEnv() - magicDir := magicdir.At(opts.MagicDirBase) + workingDir := workingdir.At(opts.MagicDirBase) stageNumber := 0 startStage := func(format string, args ...any) func(format string, args ...any) { @@ -154,7 +154,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro opts.Logger(log.LevelInfo, "%s %s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder"), buildinfo.Version()) - cleanupDockerConfigJSON, err := initDockerConfigJSON(opts.Logger, magicDir, opts.DockerConfigBase64) + cleanupDockerConfigJSON, err := initDockerConfigJSON(opts.Logger, workingDir, opts.DockerConfigBase64) if err != nil { return err } @@ -168,8 +168,8 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro ContainerEnv: make(map[string]string), RemoteEnv: make(map[string]string), } - if fileExists(opts.Filesystem, magicDir.Image()) { - if err = parseMagicImageFile(opts.Filesystem, magicDir.Image(), &runtimeData); err != nil { + if fileExists(opts.Filesystem, workingDir.Image()) { + if err = parseMagicImageFile(opts.Filesystem, workingDir.Image(), &runtimeData); err != nil { return fmt.Errorf("parse magic image file: %w", err) } runtimeData.Image = true @@ -186,7 +186,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro opts.ExportEnvFile = "" } } - runtimeData.Built = fileExists(opts.Filesystem, magicDir.Built()) + runtimeData.Built = fileExists(opts.Filesystem, workingDir.Built()) buildTimeWorkspaceFolder := opts.WorkspaceFolder var fallbackErr error @@ -233,7 +233,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro if err != nil { return fmt.Errorf("git clone options: %w", err) } - cloneOpts.Path = magicDir.Join("repo") + cloneOpts.Path = workingDir.Join("repo") endStage := startStage("📦 Remote repo build mode enabled, cloning %s to %s for build context...", newColor(color.FgCyan).Sprintf(opts.GitURL), @@ -259,7 +259,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro if !runtimeData.Image { defaultBuildParams := func() (*devcontainer.Compiled, error) { - dockerfile := magicDir.Join("Dockerfile") + dockerfile := workingDir.Join("Dockerfile") file, err := opts.Filesystem.OpenFile(dockerfile, os.O_CREATE|os.O_WRONLY, 0o644) if err != nil { return nil, err @@ -281,7 +281,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro return &devcontainer.Compiled{ DockerfilePath: dockerfile, DockerfileContent: content, - BuildContext: magicDir.Path(), + BuildContext: workingDir.Path(), }, nil } @@ -318,7 +318,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro opts.Logger(log.LevelInfo, "No Dockerfile or image specified; falling back to the default image...") fallbackDockerfile = defaultParams.DockerfilePath } - buildParams, err = devContainer.Compile(opts.Filesystem, devcontainerDir, magicDir.Path(), fallbackDockerfile, opts.WorkspaceFolder, false, os.LookupEnv) + buildParams, err = devContainer.Compile(opts.Filesystem, devcontainerDir, workingDir.Path(), fallbackDockerfile, opts.WorkspaceFolder, false, os.LookupEnv) if err != nil { return fmt.Errorf("compile devcontainer.json: %w", err) } @@ -393,7 +393,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro // So we add them to the default ignore list. See: // https://github.com/GoogleContainerTools/kaniko/blob/63be4990ca5a60bdf06ddc4d10aa4eca0c0bc714/cmd/executor/cmd/root.go#L136 ignorePaths := append([]string{ - magicDir.Path(), + workingDir.Path(), opts.WorkspaceFolder, // See: https://github.com/coder/envbuilder/issues/37 "/etc/resolv.conf", @@ -421,18 +421,18 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro if err := util.AddAllowedPathToDefaultIgnoreList(opts.BinaryPath); err != nil { return fmt.Errorf("add envbuilder binary to ignore list: %w", err) } - if err := util.AddAllowedPathToDefaultIgnoreList(magicDir.Image()); err != nil { + if err := util.AddAllowedPathToDefaultIgnoreList(workingDir.Image()); err != nil { return fmt.Errorf("add magic image file to ignore list: %w", err) } - if err := util.AddAllowedPathToDefaultIgnoreList(magicDir.Features()); err != nil { + if err := util.AddAllowedPathToDefaultIgnoreList(workingDir.Features()); err != nil { return fmt.Errorf("add features to ignore list: %w", err) } - magicTempDir := magicdir.At(buildParams.BuildContext, magicdir.TempDir) + magicTempDir := workingdir.At(buildParams.BuildContext, workingdir.TempDir) if err := opts.Filesystem.MkdirAll(magicTempDir.Path(), 0o755); err != nil { return fmt.Errorf("create magic temp dir in build context: %w", err) } // Add the magic directives that embed the binary into the built image. - buildParams.DockerfileContent += magicdir.Directives + buildParams.DockerfileContent += workingdir.Directives envbuilderBinDest := filepath.Join(magicTempDir.Path(), "envbuilder") magicImageDest := magicTempDir.Image() @@ -467,7 +467,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro } // temp move of all ro mounts - tempRemountDest := magicDir.Join("mnt") + tempRemountDest := workingDir.Join("mnt") // ignorePrefixes is a superset of ignorePaths that we pass to kaniko's // IgnoreList. ignorePrefixes := append([]string{"/dev", "/proc", "/sys"}, ignorePaths...) @@ -845,7 +845,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro // Create the magic file to indicate that this build // has already been ran before! if !runtimeData.Built { - file, err := opts.Filesystem.Create(magicDir.Built()) + file, err := opts.Filesystem.Create(workingDir.Built()) if err != nil { return fmt.Errorf("create magic file: %w", err) } @@ -864,7 +864,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro opts.Logger(log.LevelInfo, "=== Running the setup command %q as the root user...", opts.SetupScript) envKey := "ENVBUILDER_ENV" - envFile := magicDir.Join("environ") + envFile := workingDir.Join("environ") file, err := opts.Filesystem.Create(envFile) if err != nil { return fmt.Errorf("create environ file: %w", err) @@ -962,7 +962,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) return nil, fmt.Errorf("--cache-repo must be set when using --get-cached-image") } - magicDir := magicdir.At(opts.MagicDirBase) + workingDir := workingdir.At(opts.MagicDirBase) stageNumber := 0 startStage := func(format string, args ...any) func(format string, args ...any) { @@ -978,7 +978,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) opts.Logger(log.LevelInfo, "%s %s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder"), buildinfo.Version()) - cleanupDockerConfigJSON, err := initDockerConfigJSON(opts.Logger, magicDir, opts.DockerConfigBase64) + cleanupDockerConfigJSON, err := initDockerConfigJSON(opts.Logger, workingDir, opts.DockerConfigBase64) if err != nil { return nil, err } @@ -1031,7 +1031,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) if err != nil { return nil, fmt.Errorf("git clone options: %w", err) } - cloneOpts.Path = magicDir.Join("repo") + cloneOpts.Path = workingDir.Join("repo") endStage := startStage("📦 Remote repo build mode enabled, cloning %s to %s for build context...", newColor(color.FgCyan).Sprintf(opts.GitURL), @@ -1056,7 +1056,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) } defaultBuildParams := func() (*devcontainer.Compiled, error) { - dockerfile := magicDir.Join("Dockerfile") + dockerfile := workingDir.Join("Dockerfile") file, err := opts.Filesystem.OpenFile(dockerfile, os.O_CREATE|os.O_WRONLY, 0o644) if err != nil { return nil, err @@ -1078,7 +1078,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) return &devcontainer.Compiled{ DockerfilePath: dockerfile, DockerfileContent: content, - BuildContext: magicDir.Path(), + BuildContext: workingDir.Path(), }, nil } @@ -1118,7 +1118,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) opts.Logger(log.LevelInfo, "No Dockerfile or image specified; falling back to the default image...") fallbackDockerfile = defaultParams.DockerfilePath } - buildParams, err = devContainer.Compile(opts.Filesystem, devcontainerDir, magicDir.Path(), fallbackDockerfile, opts.WorkspaceFolder, false, os.LookupEnv) + buildParams, err = devContainer.Compile(opts.Filesystem, devcontainerDir, workingDir.Path(), fallbackDockerfile, opts.WorkspaceFolder, false, os.LookupEnv) if err != nil { return nil, fmt.Errorf("compile devcontainer.json: %w", err) } @@ -1184,7 +1184,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) // So we add them to the default ignore list. See: // https://github.com/GoogleContainerTools/kaniko/blob/63be4990ca5a60bdf06ddc4d10aa4eca0c0bc714/cmd/executor/cmd/root.go#L136 ignorePaths := append([]string{ - magicDir.Path(), + workingDir.Path(), opts.WorkspaceFolder, // See: https://github.com/coder/envbuilder/issues/37 "/etc/resolv.conf", @@ -1207,10 +1207,10 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) // build via executor.RunCacheProbe we need to have the *exact* copy of the // envbuilder binary available used to build the image and we also need to // add the magic directives to the Dockerfile content. - // MAGICDIR - buildParams.DockerfileContent += magicdir.Directives + // WORKINGDIR + buildParams.DockerfileContent += workingdir.Directives - magicTempDir := filepath.Join(buildParams.BuildContext, magicdir.TempDir) + magicTempDir := filepath.Join(buildParams.BuildContext, workingdir.TempDir) if err := opts.Filesystem.MkdirAll(magicTempDir, 0o755); err != nil { return nil, fmt.Errorf("create magic temp dir in build context: %w", err) } @@ -1546,11 +1546,11 @@ func maybeDeleteFilesystem(logger log.Func, force bool) error { // We always expect the magic directory to be set to the default, signifying that // the user is running envbuilder in a container. // If this is set to anything else we should bail out to prevent accidental data loss. - // defaultMagicDir := magicdir.MagicDir("") + // defaultWorkingDir := workingdir.WorkingDir("") kanikoDir, ok := os.LookupEnv("KANIKO_DIR") - if !ok || strings.TrimSpace(kanikoDir) != magicdir.Default.Path() { + if !ok || strings.TrimSpace(kanikoDir) != workingdir.Default.Path() { if !force { - logger(log.LevelError, "KANIKO_DIR is not set to %s. Bailing!\n", magicdir.Default.Path()) + logger(log.LevelError, "KANIKO_DIR is not set to %s. Bailing!\n", workingdir.Default.Path()) logger(log.LevelError, "To bypass this check, set FORCE_SAFE=true.") return errors.New("safety check failed") } @@ -1627,13 +1627,13 @@ func parseMagicImageFile(fs billy.Filesystem, path string, v any) error { return nil } -func initDockerConfigJSON(logf log.Func, magicDir magicdir.MagicDir, dockerConfigBase64 string) (func() error, error) { +func initDockerConfigJSON(logf log.Func, workingDir workingdir.WorkingDir, dockerConfigBase64 string) (func() error, error) { var cleanupOnce sync.Once noop := func() error { return nil } if dockerConfigBase64 == "" { return noop, nil } - cfgPath := magicDir.Join("config.json") + cfgPath := workingDir.Join("config.json") decoded, err := base64.StdEncoding.DecodeString(dockerConfigBase64) if err != nil { return noop, fmt.Errorf("decode docker config: %w", err) @@ -1656,7 +1656,7 @@ func initDockerConfigJSON(logf log.Func, magicDir magicdir.MagicDir, dockerConfi } logf(log.LevelInfo, "Wrote Docker config JSON to %s", cfgPath) oldDockerConfig := os.Getenv("DOCKER_CONFIG") - _ = os.Setenv("DOCKER_CONFIG", magicDir.Path()) + _ = os.Setenv("DOCKER_CONFIG", workingDir.Path()) newDockerConfig := os.Getenv("DOCKER_CONFIG") logf(log.LevelInfo, "Set DOCKER_CONFIG to %s", newDockerConfig) cleanup := func() error { diff --git a/integration/integration_test.go b/integration/integration_test.go index 7fd732d6..deb21ef6 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -27,7 +27,7 @@ import ( "github.com/coder/coder/v2/codersdk/agentsdk" "github.com/coder/envbuilder" "github.com/coder/envbuilder/devcontainer/features" - "github.com/coder/envbuilder/internal/magicdir" + "github.com/coder/envbuilder/internal/workingdir" "github.com/coder/envbuilder/options" "github.com/coder/envbuilder/testutil/gittest" "github.com/coder/envbuilder/testutil/mwtest" @@ -502,7 +502,7 @@ func TestBuildFromDockerfile(t *testing.T) { require.Equal(t, "hello", strings.TrimSpace(output)) // Verify that the Docker configuration secret file is removed - configJSONContainerPath := magicdir.Default.Join("config.json") + configJSONContainerPath := workingdir.Default.Join("config.json") output = execContainer(t, ctr, "stat "+configJSONContainerPath) require.Contains(t, output, "No such file or directory") } diff --git a/internal/magicdir/magicdir_internal_test.go b/internal/magicdir/magicdir_internal_test.go deleted file mode 100644 index 43b66ba0..00000000 --- a/internal/magicdir/magicdir_internal_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package magicdir - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func Test_MagicDir(t *testing.T) { - t.Parallel() - - t.Run("Default", func(t *testing.T) { - t.Parallel() - require.Equal(t, defaultMagicDirBase+"/foo", Default.Join("foo")) - require.Equal(t, defaultMagicDirBase, Default.Path()) - require.Equal(t, defaultMagicDirBase+"/built", Default.Built()) - require.Equal(t, defaultMagicDirBase+"/image", Default.Image()) - }) - - t.Run("ZeroValue", func(t *testing.T) { - t.Parallel() - var md MagicDir - require.Equal(t, defaultMagicDirBase+"/foo", md.Join("foo")) - require.Equal(t, defaultMagicDirBase, md.Path()) - require.Equal(t, defaultMagicDirBase+"/built", md.Built()) - require.Equal(t, defaultMagicDirBase+"/image", md.Image()) - }) - - t.Run("At", func(t *testing.T) { - t.Parallel() - tmpDir := t.TempDir() - md := At(tmpDir) - require.Equal(t, tmpDir+"/foo", md.Join("foo")) - require.Equal(t, tmpDir, md.Path()) - require.Equal(t, tmpDir+"/built", md.Built()) - require.Equal(t, tmpDir+"/image", md.Image()) - }) -} diff --git a/internal/magicdir/magicdir.go b/internal/workingdir/workingdir.go similarity index 67% rename from internal/magicdir/magicdir.go rename to internal/workingdir/workingdir.go index 5e062514..5df05234 100644 --- a/internal/magicdir/magicdir.go +++ b/internal/workingdir/workingdir.go @@ -1,4 +1,4 @@ -package magicdir +package workingdir import ( "fmt" @@ -6,10 +6,10 @@ import ( ) const ( - // defaultMagicDirBase is the default working location for envbuilder. + // defaultWorkingDirBase is the default working location for envbuilder. // This is a special directory that must not be modified by the user // or images. This is intentionally unexported. - defaultMagicDirBase = "/.envbuilder" + defaultWorkingDirBase = "/.envbuilder" // TempDir is a directory inside the build context inside which // we place files referenced by MagicDirectives. @@ -20,7 +20,7 @@ var ( // Default is the default working directory for Envbuilder. // This defaults to /.envbuilder. It should only be used when Envbuilder // is known to be running as root inside a container. - Default MagicDir + Default WorkingDir // Directives are directives automatically appended to Dockerfiles // when pushing the image. These directives allow the built image to be // 're-used'. @@ -30,33 +30,33 @@ COPY --chmod=0644 %[1]s/image %[2]s/image USER root WORKDIR / ENTRYPOINT ["%[2]s/bin/envbuilder"] -`, TempDir, defaultMagicDirBase) +`, TempDir, defaultWorkingDirBase) ) -// MagicDir is a working directory for envbuilder. It +// WorkingDir is a working directory for envbuilder. It // will also be present in images built by envbuilder. -type MagicDir struct { +type WorkingDir struct { base string } -// At returns a MagicDir rooted at filepath.Join(paths...) -func At(paths ...string) MagicDir { +// At returns a WorkingDir rooted at filepath.Join(paths...) +func At(paths ...string) WorkingDir { if len(paths) == 0 { - return MagicDir{} + return WorkingDir{} } - return MagicDir{base: filepath.Join(paths...)} + return WorkingDir{base: filepath.Join(paths...)} } // Join returns the result of filepath.Join([m.Path, paths...]). -func (m MagicDir) Join(paths ...string) string { +func (m WorkingDir) Join(paths ...string) string { return filepath.Join(append([]string{m.Path()}, paths...)...) } -// String returns the string representation of the MagicDir. -func (m MagicDir) Path() string { - // Instead of the zero value, use defaultMagicDir. +// String returns the string representation of the WorkingDir. +func (m WorkingDir) Path() string { + // Instead of the zero value, use defaultWorkingDir. if m.base == "" { - return defaultMagicDirBase + return defaultWorkingDirBase } return m.base } @@ -65,7 +65,7 @@ func (m MagicDir) Path() string { // when envbuilder has already been run. This is used // to skip building when a container is restarting. // e.g. docker stop -> docker start -func (m MagicDir) Built() string { +func (m WorkingDir) Built() string { return m.Join("built") } @@ -73,11 +73,11 @@ func (m MagicDir) Built() string { // envbuilder has already been run. This is used to skip // the destructive initial build step when 'resuming' envbuilder // from a previously built image. -func (m MagicDir) Image() string { +func (m WorkingDir) Image() string { return m.Join("image") } // Features is a directory that contains feature files. -func (m MagicDir) Features() string { +func (m WorkingDir) Features() string { return m.Join("features") } diff --git a/internal/workingdir/workingdir_internal_test.go b/internal/workingdir/workingdir_internal_test.go new file mode 100644 index 00000000..5e1dfc01 --- /dev/null +++ b/internal/workingdir/workingdir_internal_test.go @@ -0,0 +1,38 @@ +package workingdir + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_WorkingDir(t *testing.T) { + t.Parallel() + + t.Run("Default", func(t *testing.T) { + t.Parallel() + require.Equal(t, defaultWorkingDirBase+"/foo", Default.Join("foo")) + require.Equal(t, defaultWorkingDirBase, Default.Path()) + require.Equal(t, defaultWorkingDirBase+"/built", Default.Built()) + require.Equal(t, defaultWorkingDirBase+"/image", Default.Image()) + }) + + t.Run("ZeroValue", func(t *testing.T) { + t.Parallel() + var md WorkingDir + require.Equal(t, defaultWorkingDirBase+"/foo", md.Join("foo")) + require.Equal(t, defaultWorkingDirBase, md.Path()) + require.Equal(t, defaultWorkingDirBase+"/built", md.Built()) + require.Equal(t, defaultWorkingDirBase+"/image", md.Image()) + }) + + t.Run("At", func(t *testing.T) { + t.Parallel() + tmpDir := t.TempDir() + md := At(tmpDir) + require.Equal(t, tmpDir+"/foo", md.Join("foo")) + require.Equal(t, tmpDir, md.Path()) + require.Equal(t, tmpDir+"/built", md.Built()) + require.Equal(t, tmpDir+"/image", md.Image()) + }) +} diff --git a/options/defaults.go b/options/defaults.go index 2d6cd52f..b9e47a80 100644 --- a/options/defaults.go +++ b/options/defaults.go @@ -9,7 +9,7 @@ import ( giturls "github.com/chainguard-dev/git-urls" "github.com/coder/envbuilder/internal/chmodfs" - "github.com/coder/envbuilder/internal/magicdir" + "github.com/coder/envbuilder/internal/workingdir" ) // EmptyWorkspaceDir is the path to a workspace that has @@ -66,6 +66,6 @@ func (o *Options) SetDefaults() { o.BinaryPath = "/.envbuilder/bin/envbuilder" } if o.MagicDirBase == "" { - o.MagicDirBase = magicdir.Default.Path() + o.MagicDirBase = workingdir.Default.Path() } } From b2400e85afcb2e078f07cfda011baf5a1fba5724 Mon Sep 17 00:00:00 2001 From: Danielle Maywood Date: Fri, 25 Oct 2024 15:05:15 +0100 Subject: [PATCH 72/81] feat: add env var for ssh private key (#396) (cherry picked from commit 08bdb8daecf680463e86a6d95deba9c9d932e803) --- cmd/envbuilder/main.go | 4 +++ docs/env-variables.md | 3 +- git/git.go | 28 +++++++++++++++ git/git_test.go | 21 ++++++++++++ integration/integration_test.go | 61 +++++++++++++++++++++++++++++++++ options/options.go | 19 +++++++--- options/testdata/options.golden | 7 +++- 7 files changed, 137 insertions(+), 6 deletions(-) diff --git a/cmd/envbuilder/main.go b/cmd/envbuilder/main.go index e8dc2201..91cde8a3 100644 --- a/cmd/envbuilder/main.go +++ b/cmd/envbuilder/main.go @@ -75,6 +75,10 @@ func envbuilderCmd() serpent.Command { } } + if o.GitSSHPrivateKeyPath != "" && o.GitSSHPrivateKeyBase64 != "" { + return errors.New("cannot have both GIT_SSH_PRIVATE_KEY_PATH and GIT_SSH_PRIVATE_KEY_BASE64 set") + } + if o.GetCachedImage { img, err := envbuilder.RunCacheProbe(inv.Context(), o) if err != nil { diff --git a/docs/env-variables.md b/docs/env-variables.md index 1c80f4fc..5513fd55 100644 --- a/docs/env-variables.md +++ b/docs/env-variables.md @@ -27,7 +27,8 @@ | `--git-clone-single-branch` | `ENVBUILDER_GIT_CLONE_SINGLE_BRANCH` | | Clone only a single branch of the Git repository. | | `--git-username` | `ENVBUILDER_GIT_USERNAME` | | The username to use for Git authentication. This is optional. | | `--git-password` | `ENVBUILDER_GIT_PASSWORD` | | The password to use for Git authentication. This is optional. | -| `--git-ssh-private-key-path` | `ENVBUILDER_GIT_SSH_PRIVATE_KEY_PATH` | | Path to an SSH private key to be used for Git authentication. | +| `--git-ssh-private-key-path` | `ENVBUILDER_GIT_SSH_PRIVATE_KEY_PATH` | | Path to an SSH private key to be used for Git authentication. If this is set, then GIT_SSH_PRIVATE_KEY_BASE64 cannot be set. | +| `--git-ssh-private-key-base64` | `ENVBUILDER_GIT_SSH_PRIVATE_KEY_BASE64` | | Base64 encoded SSH private key to be used for Git authentication. If this is set, then GIT_SSH_PRIVATE_KEY_PATH cannot be set. | | `--git-http-proxy-url` | `ENVBUILDER_GIT_HTTP_PROXY_URL` | | The URL for the HTTP proxy. This is optional. | | `--workspace-folder` | `ENVBUILDER_WORKSPACE_FOLDER` | | The path to the workspace folder that will be built. This is optional. | | `--ssl-cert-base64` | `ENVBUILDER_SSL_CERT_BASE64` | | The content of an SSL cert file. This is useful for self-signed certificates. | diff --git a/git/git.go b/git/git.go index 7d132c3a..f37b9682 100644 --- a/git/git.go +++ b/git/git.go @@ -2,6 +2,7 @@ package git import ( "context" + "encoding/base64" "errors" "fmt" "io" @@ -181,6 +182,22 @@ func ReadPrivateKey(path string) (gossh.Signer, error) { return k, nil } +// DecodeBase64PrivateKey attempts to decode a base64 encoded private +// key and returns an ssh.Signer +func DecodeBase64PrivateKey(key string) (gossh.Signer, error) { + bs, err := base64.StdEncoding.DecodeString(key) + if err != nil { + return nil, fmt.Errorf("decode base64: %w", err) + } + + k, err := gossh.ParsePrivateKey(bs) + if err != nil { + return nil, fmt.Errorf("parse private key: %w", err) + } + + return k, nil +} + // LogHostKeyCallback is a HostKeyCallback that just logs host keys // and does nothing else. func LogHostKeyCallback(logger func(string, ...any)) gossh.HostKeyCallback { @@ -273,6 +290,17 @@ func SetupRepoAuth(logf func(string, ...any), options *options.Options) transpor } } + // If no path was provided, fall back to the environment variable + if options.GitSSHPrivateKeyBase64 != "" { + s, err := DecodeBase64PrivateKey(options.GitSSHPrivateKeyBase64) + if err != nil { + logf("❌ Failed to decode base 64 private key: %s", err.Error()) + } else { + logf("🔑 Using %s key!", s.PublicKey().Type()) + signer = s + } + } + // If no SSH key set, fall back to agent auth. if signer == nil { logf("🔑 No SSH key found, falling back to agent!") diff --git a/git/git_test.go b/git/git_test.go index e7a58f90..0da5a163 100644 --- a/git/git_test.go +++ b/git/git_test.go @@ -3,6 +3,7 @@ package git_test import ( "context" "crypto/ed25519" + "encoding/base64" "fmt" "io" "net/http/httptest" @@ -433,6 +434,22 @@ func TestSetupRepoAuth(t *testing.T) { require.Equal(t, actualSigner, pk.Signer) }) + t.Run("SSH/Base64PrivateKey", func(t *testing.T) { + opts := &options.Options{ + GitURL: "ssh://git@host.tld:repo/path", + GitSSHPrivateKeyBase64: base64EncodeTestPrivateKey(), + } + auth := git.SetupRepoAuth(t.Logf, opts) + + pk, ok := auth.(*gitssh.PublicKeys) + require.True(t, ok) + require.NotNil(t, pk.Signer) + + actualSigner, err := gossh.ParsePrivateKey([]byte(testKey)) + require.NoError(t, err) + require.Equal(t, actualSigner, pk.Signer) + }) + t.Run("SSH/NoAuthMethods", func(t *testing.T) { opts := &options.Options{ GitURL: "ssh://git@host.tld:repo/path", @@ -502,3 +519,7 @@ func writeTestPrivateKey(t *testing.T) string { require.NoError(t, os.WriteFile(kPath, []byte(testKey), 0o600)) return kPath } + +func base64EncodeTestPrivateKey() string { + return base64.StdEncoding.EncodeToString([]byte(testKey)) +} diff --git a/integration/integration_test.go b/integration/integration_test.go index deb21ef6..9413c349 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "context" + "crypto/ed25519" "encoding/base64" "encoding/json" "encoding/pem" @@ -32,6 +33,8 @@ import ( "github.com/coder/envbuilder/testutil/gittest" "github.com/coder/envbuilder/testutil/mwtest" "github.com/coder/envbuilder/testutil/registrytest" + "github.com/go-git/go-billy/v5/osfs" + gossh "golang.org/x/crypto/ssh" clitypes "github.com/docker/cli/cli/config/types" "github.com/docker/docker/api/types" @@ -58,6 +61,16 @@ const ( testContainerLabel = "envbox-integration-test" testImageAlpine = "localhost:5000/envbuilder-test-alpine:latest" testImageUbuntu = "localhost:5000/envbuilder-test-ubuntu:latest" + + // nolint:gosec // Throw-away key for testing. DO NOT REUSE. + testSSHKey = `-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACBXOGgAge/EbcejqASqZa6s8PFXZle56DiGEt0VYnljuwAAAKgM05mUDNOZ +lAAAAAtzc2gtZWQyNTUxOQAAACBXOGgAge/EbcejqASqZa6s8PFXZle56DiGEt0VYnljuw +AAAEDCawwtjrM4AGYXD1G6uallnbsgMed4cfkFsQ+mLZtOkFc4aACB78Rtx6OoBKplrqzw +8VdmV7noOIYS3RVieWO7AAAAHmNpYW5AY2RyLW1icC1mdmZmdzBuOHEwNXAuaG9tZQECAw +QFBgc= +-----END OPENSSH PRIVATE KEY-----` ) func TestLogs(t *testing.T) { @@ -378,6 +391,54 @@ func TestSucceedsGitAuth(t *testing.T) { require.Contains(t, gitConfig, srv.URL) } +func TestGitSSHAuth(t *testing.T) { + t.Parallel() + + base64Key := base64.StdEncoding.EncodeToString([]byte(testSSHKey)) + + t.Run("Base64/Success", func(t *testing.T) { + signer, err := gossh.ParsePrivateKey([]byte(testSSHKey)) + require.NoError(t, err) + require.NotNil(t, signer) + + tmpDir := t.TempDir() + srvFS := osfs.New(tmpDir, osfs.WithChrootOS()) + + _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "Dockerfile", "FROM "+testImageAlpine, "Initial commit")) + tr := gittest.NewServerSSH(t, srvFS, signer.PublicKey()) + + _, err = runEnvbuilder(t, runOpts{env: []string{ + envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), + envbuilderEnv("GIT_URL", tr.String()+"."), + envbuilderEnv("GIT_SSH_PRIVATE_KEY_BASE64", base64Key), + }}) + // TODO: Ensure it actually clones but this does mean we have + // successfully authenticated. + require.ErrorContains(t, err, "repository not found") + }) + + t.Run("Base64/Failure", func(t *testing.T) { + _, randomKey, err := ed25519.GenerateKey(nil) + require.NoError(t, err) + signer, err := gossh.NewSignerFromKey(randomKey) + require.NoError(t, err) + require.NotNil(t, signer) + + tmpDir := t.TempDir() + srvFS := osfs.New(tmpDir, osfs.WithChrootOS()) + + _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "Dockerfile", "FROM "+testImageAlpine, "Initial commit")) + tr := gittest.NewServerSSH(t, srvFS, signer.PublicKey()) + + _, err = runEnvbuilder(t, runOpts{env: []string{ + envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), + envbuilderEnv("GIT_URL", tr.String()+"."), + envbuilderEnv("GIT_SSH_PRIVATE_KEY_BASE64", base64Key), + }}) + require.ErrorContains(t, err, "handshake failed") + }) +} + func TestSucceedsGitAuthInURL(t *testing.T) { t.Parallel() srv := gittest.CreateGitServer(t, gittest.Options{ diff --git a/options/options.go b/options/options.go index 18bd56d1..ed6bfa99 100644 --- a/options/options.go +++ b/options/options.go @@ -108,6 +108,9 @@ type Options struct { // GitSSHPrivateKeyPath is the path to an SSH private key to be used for // Git authentication. GitSSHPrivateKeyPath string + // GitSSHPrivateKeyBase64 is the content of an SSH private key to be used + // for Git authentication. + GitSSHPrivateKeyBase64 string // GitHTTPProxyURL is the URL for the HTTP proxy. This is optional. GitHTTPProxyURL string // WorkspaceFolder is the path to the workspace folder that will be built. @@ -358,10 +361,18 @@ func (o *Options) CLI() serpent.OptionSet { Description: "The password to use for Git authentication. This is optional.", }, { - Flag: "git-ssh-private-key-path", - Env: WithEnvPrefix("GIT_SSH_PRIVATE_KEY_PATH"), - Value: serpent.StringOf(&o.GitSSHPrivateKeyPath), - Description: "Path to an SSH private key to be used for Git authentication.", + Flag: "git-ssh-private-key-path", + Env: WithEnvPrefix("GIT_SSH_PRIVATE_KEY_PATH"), + Value: serpent.StringOf(&o.GitSSHPrivateKeyPath), + Description: "Path to an SSH private key to be used for Git authentication." + + " If this is set, then GIT_SSH_PRIVATE_KEY_BASE64 cannot be set.", + }, + { + Flag: "git-ssh-private-key-base64", + Env: WithEnvPrefix("GIT_SSH_PRIVATE_KEY_BASE64"), + Value: serpent.StringOf(&o.GitSSHPrivateKeyBase64), + Description: "Base64 encoded SSH private key to be used for Git authentication." + + " If this is set, then GIT_SSH_PRIVATE_KEY_PATH cannot be set.", }, { Flag: "git-http-proxy-url", diff --git a/options/testdata/options.golden b/options/testdata/options.golden index 0bfbd64a..518f7741 100644 --- a/options/testdata/options.golden +++ b/options/testdata/options.golden @@ -94,8 +94,13 @@ OPTIONS: --git-password string, $ENVBUILDER_GIT_PASSWORD The password to use for Git authentication. This is optional. + --git-ssh-private-key-base64 string, $ENVBUILDER_GIT_SSH_PRIVATE_KEY_BASE64 + Base64 encoded SSH private key to be used for Git authentication. If + this is set, then GIT_SSH_PRIVATE_KEY_PATH cannot be set. + --git-ssh-private-key-path string, $ENVBUILDER_GIT_SSH_PRIVATE_KEY_PATH - Path to an SSH private key to be used for Git authentication. + Path to an SSH private key to be used for Git authentication. If this + is set, then GIT_SSH_PRIVATE_KEY_BASE64 cannot be set. --git-url string, $ENVBUILDER_GIT_URL The URL of a Git repository containing a Devcontainer or Docker image From 0ab50520389dc7b00e46156ebdfc750a9dff4e18 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 28 Oct 2024 19:56:56 +0000 Subject: [PATCH 73/81] chore(integration): update TestLogs to validate that CODER_ASGENT_SUBSYSTEM is set correctly (#397) (cherry picked from commit 561cc53d791cb7dcd5caa1edc66217f40c49204c) --- integration/integration_test.go | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/integration/integration_test.go b/integration/integration_test.go index 9413c349..bb900a0e 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -123,10 +123,11 @@ func TestLogs(t *testing.T) { "Dockerfile": fmt.Sprintf(`FROM %s`, testImageUbuntu), }, }) - _, err := runEnvbuilder(t, runOpts{env: []string{ + ctrID, err := runEnvbuilder(t, runOpts{env: []string{ envbuilderEnv("GIT_URL", srv.URL), "CODER_AGENT_URL=" + logSrv.URL, "CODER_AGENT_TOKEN=" + token, + "ENVBUILDER_INIT_SCRIPT=env", }}) require.NoError(t, err) ctx, cancel := context.WithTimeout(context.Background(), time.Minute) @@ -136,6 +137,28 @@ func TestLogs(t *testing.T) { t.Fatal("timed out waiting for logs") case <-logsDone: } + + // Wait for the container to exit + client, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + require.NoError(t, err) + require.Eventually(t, func() bool { + status, err := client.ContainerInspect(ctx, ctrID) + if !assert.NoError(t, err) { + return false + } + return !status.State.Running + }, 10*time.Second, time.Second, "container never exited") + + // Check the expected log output + logReader, err := client.ContainerLogs(ctx, ctrID, container.LogsOptions{ + ShowStdout: true, + ShowStderr: true, + }) + require.NoError(t, err) + logBytes, err := io.ReadAll(logReader) + require.NoError(t, err) + logs := string(logBytes) + require.Contains(t, logs, "CODER_AGENT_SUBSYSTEM=envbuilder") } func TestInitScriptInitCommand(t *testing.T) { From dc6aa546e014e87fdfccc30c2a2284bc84bccd70 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Wed, 30 Oct 2024 18:57:19 +0200 Subject: [PATCH 74/81] fix: search `$DOCKER_CONFIG` if no base64 config is provided (#398) (cherry picked from commit c4b082e09e5b50b8b2db4805419685d8cf452eec) --- Makefile | 5 +- docs/container-registry-auth.md | 14 +- docs/env-variables.md | 2 +- envbuilder.go | 238 +++++++++++++++++++++++++------- integration/integration_test.go | 179 ++++++++++++++++++++---- options/options.go | 4 +- options/testdata/options.golden | 4 +- scripts/docsgen/main.go | 2 +- 8 files changed, 363 insertions(+), 85 deletions(-) diff --git a/Makefile b/Makefile index ca4c0e6d..ce079015 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,9 @@ develop: build: scripts/envbuilder-$(GOARCH) ./scripts/build.sh +.PHONY: gen +gen: docs/env-variables.md update-golden-files + .PHONY: update-golden-files update-golden-files: .gen-golden @@ -85,4 +88,4 @@ test-images-pull: docker push localhost:5000/envbuilder-test-ubuntu:latest .registry-cache/docker/registry/v2/repositories/envbuilder-test-codercom-code-server: - docker push localhost:5000/envbuilder-test-codercom-code-server:latest \ No newline at end of file + docker push localhost:5000/envbuilder-test-codercom-code-server:latest diff --git a/docs/container-registry-auth.md b/docs/container-registry-auth.md index e0d7663e..f14a66d4 100644 --- a/docs/container-registry-auth.md +++ b/docs/container-registry-auth.md @@ -14,9 +14,19 @@ After you have a configuration that resembles the following: } ``` -`base64` encode the JSON and provide it to envbuilder as the `ENVBUILDER_DOCKER_CONFIG_BASE64` environment variable. +`base64` encode the JSON and provide it to envbuilder as the +`ENVBUILDER_DOCKER_CONFIG_BASE64` environment variable. -Alternatively, if running `envbuilder` in Kubernetes, you can create an `ImagePullSecret` and +Alternatively, the configuration file can be placed in `/.envbuilder/config.json`. +The `DOCKER_CONFIG` environment variable can be used to define a custom path. The +path must either be the path to a directory containing `config.json` or the full +path to the JSON file itself. + +> [!NOTE] Providing the docker configuration through other means than the +> `ENVBUILDER_DOCKER_CONFIG_BASE64` environment variable will leave the +> configuration file in the container filesystem. This may be a security risk. + +When running `envbuilder` in Kubernetes, you can create an `ImagePullSecret` and pass it into the pod as a volume mount. This example will work for all registries. ```shell diff --git a/docs/env-variables.md b/docs/env-variables.md index 5513fd55..cb7054f3 100644 --- a/docs/env-variables.md +++ b/docs/env-variables.md @@ -15,7 +15,7 @@ | `--dockerfile-path` | `ENVBUILDER_DOCKERFILE_PATH` | | The relative path to the Dockerfile that will be used to build the workspace. This is an alternative to using a devcontainer that some might find simpler. | | `--build-context-path` | `ENVBUILDER_BUILD_CONTEXT_PATH` | | Can be specified when a DockerfilePath is specified outside the base WorkspaceFolder. This path MUST be relative to the WorkspaceFolder path into which the repo is cloned. | | `--cache-ttl-days` | `ENVBUILDER_CACHE_TTL_DAYS` | | The number of days to use cached layers before expiring them. Defaults to 7 days. | -| `--docker-config-base64` | `ENVBUILDER_DOCKER_CONFIG_BASE64` | | The base64 encoded Docker config file that will be used to pull images from private container registries. | +| `--docker-config-base64` | `ENVBUILDER_DOCKER_CONFIG_BASE64` | | The base64 encoded Docker config file that will be used to pull images from private container registries. When this is set, Docker configuration set via the DOCKER_CONFIG environment variable is ignored. | | `--fallback-image` | `ENVBUILDER_FALLBACK_IMAGE` | | Specifies an alternative image to use when neither an image is declared in the devcontainer.json file nor a Dockerfile is present. If there's a build failure (from a faulty Dockerfile) or a misconfiguration, this image will be the substitute. Set ExitOnBuildFailure to true to halt the container if the build faces an issue. | | `--exit-on-build-failure` | `ENVBUILDER_EXIT_ON_BUILD_FAILURE` | | Terminates the container upon a build failure. This is handy when preferring the FALLBACK_IMAGE in cases where no devcontainer.json or image is provided. However, it ensures that the container stops if the build process encounters an error. | | `--force-safe` | `ENVBUILDER_FORCE_SAFE` | | Ignores any filesystem safety checks. This could cause serious harm to your system! This is used in cases where bypass is needed to unblock customers. | diff --git a/envbuilder.go b/envbuilder.go index 75829c2f..028a7bb2 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -41,6 +41,7 @@ import ( "github.com/distribution/distribution/v3/configuration" "github.com/distribution/distribution/v3/registry/handlers" _ "github.com/distribution/distribution/v3/registry/storage/driver/filesystem" + dockerconfig "github.com/docker/cli/cli/config" "github.com/docker/cli/cli/config/configfile" "github.com/fatih/color" v1 "github.com/google/go-containerregistry/pkg/v1" @@ -56,7 +57,7 @@ import ( var ErrNoFallbackImage = errors.New("no fallback image has been specified") // DockerConfig represents the Docker configuration file. -type DockerConfig configfile.ConfigFile +type DockerConfig = configfile.ConfigFile type runtimeDataStore struct { // Runtime data. @@ -154,13 +155,13 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro opts.Logger(log.LevelInfo, "%s %s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder"), buildinfo.Version()) - cleanupDockerConfigJSON, err := initDockerConfigJSON(opts.Logger, workingDir, opts.DockerConfigBase64) + cleanupDockerConfigOverride, err := initDockerConfigOverride(opts.Filesystem, opts.Logger, workingDir, opts.DockerConfigBase64) if err != nil { return err } defer func() { - if err := cleanupDockerConfigJSON(); err != nil { - opts.Logger(log.LevelError, "failed to cleanup docker config JSON: %w", err) + if err := cleanupDockerConfigOverride(); err != nil { + opts.Logger(log.LevelError, "failed to cleanup docker config override: %w", err) } }() // best effort @@ -711,6 +712,11 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro // Sanitize the environment of any opts! options.UnsetEnv() + // Remove the Docker config secret file! + if err := cleanupDockerConfigOverride(); err != nil { + return err + } + // Set the environment from /etc/environment first, so it can be // overridden by the image and devcontainer settings. err = setEnvFromEtcEnvironment(opts.Logger) @@ -770,11 +776,6 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro exportEnvFile.Close() } - // Remove the Docker config secret file! - if err := cleanupDockerConfigJSON(); err != nil { - return err - } - if runtimeData.ContainerUser == "" { opts.Logger(log.LevelWarn, "#%d: no user specified, using root", stageNumber) } @@ -978,13 +979,13 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) opts.Logger(log.LevelInfo, "%s %s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder"), buildinfo.Version()) - cleanupDockerConfigJSON, err := initDockerConfigJSON(opts.Logger, workingDir, opts.DockerConfigBase64) + cleanupDockerConfigOverride, err := initDockerConfigOverride(opts.Filesystem, opts.Logger, workingDir, opts.DockerConfigBase64) if err != nil { return nil, err } defer func() { - if err := cleanupDockerConfigJSON(); err != nil { - opts.Logger(log.LevelError, "failed to cleanup docker config JSON: %w", err) + if err := cleanupDockerConfigOverride(); err != nil { + opts.Logger(log.LevelError, "failed to cleanup docker config override: %w", err) } }() // best effort @@ -1315,7 +1316,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) options.UnsetEnv() // Remove the Docker config secret file! - if err := cleanupDockerConfigJSON(); err != nil { + if err := cleanupDockerConfigOverride(); err != nil { return nil, err } @@ -1567,8 +1568,22 @@ func maybeDeleteFilesystem(logger log.Func, force bool) error { } func fileExists(fs billy.Filesystem, path string) bool { - _, err := fs.Stat(path) - return err == nil + fi, err := fs.Stat(path) + return err == nil && !fi.IsDir() +} + +func readFile(fs billy.Filesystem, name string) ([]byte, error) { + f, err := fs.Open(name) + if err != nil { + return nil, fmt.Errorf("open file: %w", err) + } + defer f.Close() + + b, err := io.ReadAll(f) + if err != nil { + return nil, fmt.Errorf("read file: %w", err) + } + return b, nil } func copyFile(fs billy.Filesystem, src, dst string, mode fs.FileMode) error { @@ -1595,6 +1610,21 @@ func copyFile(fs billy.Filesystem, src, dst string, mode fs.FileMode) error { return nil } +func writeFile(fs billy.Filesystem, name string, data []byte, perm fs.FileMode) error { + f, err := fs.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, perm) + if err != nil { + return fmt.Errorf("create file: %w", err) + } + _, err = f.Write(data) + if err != nil { + err = fmt.Errorf("write file: %w", err) + } + if err2 := f.Close(); err2 != nil && err == nil { + err = fmt.Errorf("close file: %w", err2) + } + return err +} + func writeMagicImageFile(fs billy.Filesystem, path string, v any) error { file, err := fs.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) if err != nil { @@ -1627,55 +1657,161 @@ func parseMagicImageFile(fs billy.Filesystem, path string, v any) error { return nil } -func initDockerConfigJSON(logf log.Func, workingDir workingdir.WorkingDir, dockerConfigBase64 string) (func() error, error) { - var cleanupOnce sync.Once - noop := func() error { return nil } - if dockerConfigBase64 == "" { - return noop, nil +const ( + dockerConfigFile = dockerconfig.ConfigFileName + dockerConfigEnvKey = dockerconfig.EnvOverrideConfigDir +) + +// initDockerConfigOverride sets the DOCKER_CONFIG environment variable +// to a path within the working directory. If a base64 encoded Docker +// config is provided, it is written to the path/config.json and the +// DOCKER_CONFIG environment variable is set to the path. If no base64 +// encoded Docker config is provided, the following paths are checked in +// order: +// +// 1. $DOCKER_CONFIG/config.json +// 2. $DOCKER_CONFIG +// 3. /.envbuilder/config.json +// +// If a Docker config file is found, its path is set as DOCKER_CONFIG. +func initDockerConfigOverride(bfs billy.Filesystem, logf log.Func, workingDir workingdir.WorkingDir, dockerConfigBase64 string) (func() error, error) { + // If dockerConfigBase64 is set, it will have priority over file + // detection. + var dockerConfigJSON []byte + var err error + if dockerConfigBase64 != "" { + logf(log.LevelInfo, "Using base64 encoded Docker config") + + dockerConfigJSON, err = base64.StdEncoding.DecodeString(dockerConfigBase64) + if err != nil { + return nil, fmt.Errorf("decode docker config: %w", err) + } + } + + oldDockerConfig := os.Getenv(dockerConfigEnvKey) + var oldDockerConfigFile string + if oldDockerConfig != "" { + oldDockerConfigFile = filepath.Join(oldDockerConfig, dockerConfigFile) + } + for _, path := range []string{ + oldDockerConfigFile, // $DOCKER_CONFIG/config.json + oldDockerConfig, // $DOCKER_CONFIG + workingDir.Join(dockerConfigFile), // /.envbuilder/config.json + } { + if path == "" || !fileExists(bfs, path) { + continue + } + + logf(log.LevelWarn, "Found Docker config at %s, this file will remain after the build", path) + + if dockerConfigJSON == nil { + logf(log.LevelInfo, "Using Docker config at %s", path) + + dockerConfigJSON, err = readFile(bfs, path) + if err != nil { + return nil, fmt.Errorf("read docker config: %w", err) + } + } else { + logf(log.LevelWarn, "Ignoring Docker config at %s, using base64 encoded Docker config instead", path) + } + break + } + + if dockerConfigJSON == nil { + // No user-provided config available. + return func() error { return nil }, nil + } + + dockerConfigJSON, err = hujson.Standardize(dockerConfigJSON) + if err != nil { + return nil, fmt.Errorf("humanize json for docker config: %w", err) } - cfgPath := workingDir.Join("config.json") - decoded, err := base64.StdEncoding.DecodeString(dockerConfigBase64) + + if err = logDockerAuthConfigs(logf, dockerConfigJSON); err != nil { + return nil, fmt.Errorf("log docker auth configs: %w", err) + } + + // We're going to set the DOCKER_CONFIG environment variable to a + // path within the working directory so that Kaniko can pick it up. + // A user should not mount a file directly to this path as we will + // write to the file. + newDockerConfig := workingDir.Join(".docker") + newDockerConfigFile := filepath.Join(newDockerConfig, dockerConfigFile) + err = bfs.MkdirAll(newDockerConfig, 0o700) + if err != nil { + return nil, fmt.Errorf("create docker config dir: %w", err) + } + + if fileExists(bfs, newDockerConfigFile) { + return nil, fmt.Errorf("unable to write Docker config file, file already exists: %s", newDockerConfigFile) + } + + restoreEnv, err := setAndRestoreEnv(logf, dockerConfigEnvKey, newDockerConfig) if err != nil { - return noop, fmt.Errorf("decode docker config: %w", err) + return nil, fmt.Errorf("set docker config override: %w", err) } - var configFile DockerConfig - decoded, err = hujson.Standardize(decoded) + + err = writeFile(bfs, newDockerConfigFile, dockerConfigJSON, 0o600) if err != nil { - return noop, fmt.Errorf("humanize json for docker config: %w", err) + _ = restoreEnv() // Best effort. + return nil, fmt.Errorf("write docker config: %w", err) } - err = json.Unmarshal(decoded, &configFile) + logf(log.LevelInfo, "Wrote Docker config JSON to %s", newDockerConfigFile) + + cleanupFile := onceErrFunc(func() error { + // Remove the Docker config secret file! + if err := bfs.Remove(newDockerConfigFile); err != nil { + logf(log.LevelError, "Failed to remove the Docker config secret file: %s", err) + return fmt.Errorf("remove docker config: %w", err) + } + return nil + }) + return func() error { return errors.Join(cleanupFile(), restoreEnv()) }, nil +} + +func logDockerAuthConfigs(logf log.Func, dockerConfigJSON []byte) error { + dc := new(DockerConfig) + err := dc.LoadFromReader(bytes.NewReader(dockerConfigJSON)) if err != nil { - return noop, fmt.Errorf("parse docker config: %w", err) + return fmt.Errorf("load docker config: %w", err) } - for k := range configFile.AuthConfigs { + for k := range dc.AuthConfigs { logf(log.LevelInfo, "Docker config contains auth for registry %q", k) } - err = os.WriteFile(cfgPath, decoded, 0o644) + return nil +} + +func setAndRestoreEnv(logf log.Func, key, value string) (restore func() error, err error) { + old := os.Getenv(key) + err = os.Setenv(key, value) if err != nil { - return noop, fmt.Errorf("write docker config: %w", err) - } - logf(log.LevelInfo, "Wrote Docker config JSON to %s", cfgPath) - oldDockerConfig := os.Getenv("DOCKER_CONFIG") - _ = os.Setenv("DOCKER_CONFIG", workingDir.Path()) - newDockerConfig := os.Getenv("DOCKER_CONFIG") - logf(log.LevelInfo, "Set DOCKER_CONFIG to %s", newDockerConfig) - cleanup := func() error { - var cleanupErr error - cleanupOnce.Do(func() { - // Restore the old DOCKER_CONFIG value. - os.Setenv("DOCKER_CONFIG", oldDockerConfig) - logf(log.LevelInfo, "Restored DOCKER_CONFIG to %s", oldDockerConfig) - // Remove the Docker config secret file! - if cleanupErr = os.Remove(cfgPath); err != nil { - if !errors.Is(err, fs.ErrNotExist) { - cleanupErr = fmt.Errorf("remove docker config: %w", cleanupErr) - } - logf(log.LevelError, "Failed to remove the Docker config secret file: %s", cleanupErr) + logf(log.LevelError, "Failed to set %s: %s", key, err) + return nil, fmt.Errorf("set %s: %w", key, err) + } + logf(log.LevelInfo, "Set %s to %s", key, value) + return onceErrFunc(func() error { + if err := func() error { + if old == "" { + return os.Unsetenv(key) } + return os.Setenv(key, old) + }(); err != nil { + return fmt.Errorf("restore %s: %w", key, err) + } + logf(log.LevelInfo, "Restored %s to %s", key, old) + return nil + }), nil +} + +func onceErrFunc(f func() error) func() error { + var once sync.Once + return func() error { + var err error + once.Do(func() { + err = f() }) - return cleanupErr + return err } - return cleanup, err } // Allows quick testing of layer caching using a local directory! diff --git a/integration/integration_test.go b/integration/integration_test.go index bb900a0e..46e0e88c 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -568,27 +568,142 @@ func TestBuildFromDevcontainerWithFeatures(t *testing.T) { require.Equal(t, "hello from test 3!", strings.TrimSpace(test3Output)) } -func TestBuildFromDockerfile(t *testing.T) { - // Ensures that a Git repository with a Dockerfile is cloned and built. - srv := gittest.CreateGitServer(t, gittest.Options{ - Files: map[string]string{ - "Dockerfile": "FROM " + testImageAlpine, +func TestBuildFromDockerfileAndConfig(t *testing.T) { + t.Parallel() + + type configFile struct { + name string + data string + } + type testCase struct { + name string + env []string + configFile configFile + configBase64 string + validate func(t *testing.T, tc testCase, ctrID, logs string) + } + + validateDockerConfig := func(t *testing.T, tc testCase, ctrID, logs string) { + t.Helper() + + // Ensure that the config matches the expected value, base64 is + // always prioritized over a file. + got := execContainer(t, ctrID, "cat /docker_config_json") + got = strings.TrimSpace(got) + want := tc.configBase64 + if want == "" { + want = tc.configFile.data + } + if want != "" { + require.Contains(t, logs, "Set DOCKER_CONFIG to /.envbuilder/.docker") + require.Equal(t, want, got) + } + + // Ensure that a warning message is printed if config secrets + // will remain in the container after build. + warningMessage := "this file will remain after the build" + if tc.configFile.name != "" { + require.Contains(t, logs, warningMessage) + } else { + require.NotContains(t, logs, warningMessage) + } + } + + configJSONContainerPath := workingdir.Default.Join(".docker", "config.json") + defaultConfigJSON := `{"experimental": "enabled"}` + + tests := []testCase{ + { + name: "Plain", + validate: func(t *testing.T, tc testCase, ctrID, logs string) { + output := execContainer(t, ctrID, "echo hello") + require.Equal(t, "hello", strings.TrimSpace(output)) + }, }, - }) - ctr, err := runEnvbuilder(t, runOpts{env: []string{ - envbuilderEnv("GIT_URL", srv.URL), - envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), - envbuilderEnv("DOCKER_CONFIG_BASE64", base64.StdEncoding.EncodeToString([]byte(`{"experimental": "enabled"}`))), - }}) - require.NoError(t, err) + { + name: "ConfigBase64", + configBase64: defaultConfigJSON, + validate: validateDockerConfig, + }, + { + name: "BindConfigToKnownLocation", + configFile: configFile{"/.envbuilder/config.json", defaultConfigJSON}, + validate: validateDockerConfig, + }, + { + name: "BindConfigToPath", + env: []string{"DOCKER_CONFIG=/secret"}, + configFile: configFile{"/secret/config.json", defaultConfigJSON}, + validate: validateDockerConfig, + }, + { + name: "BindConfigToCustomFile", + env: []string{"DOCKER_CONFIG=/secret/my.json"}, + configFile: configFile{"/secret/my.json", defaultConfigJSON}, + validate: validateDockerConfig, + }, + { + name: "ConfigBase64AndBindUsesBase64", + configFile: configFile{"/.envbuilder/config.json", `{"experimental": "disabled"}`}, + configBase64: defaultConfigJSON, + validate: validateDockerConfig, + }, + { + name: "ConfigBase64AndCustomConfigPath", + env: []string{"DOCKER_CONFIG=/secret"}, + configBase64: defaultConfigJSON, + validate: validateDockerConfig, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() - output := execContainer(t, ctr, "echo hello") - require.Equal(t, "hello", strings.TrimSpace(output)) + // Ensures that a Git repository with a Dockerfile is cloned and built. + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ + "Dockerfile": fmt.Sprintf(` + FROM %[1]s + RUN if [ -f %[2]q ]; then cat %[2]q > /docker_config_json; fi + `, testImageAlpine, configJSONContainerPath), + }, + }) + + logbuf := new(bytes.Buffer) + opts := runOpts{ + env: []string{ + envbuilderEnv("GIT_URL", srv.URL), + envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), + }, + logbuf: logbuf, + } + + if tt.configFile.name != "" { + dir := t.TempDir() + configFile := filepath.Join(dir, filepath.Base(tt.configFile.name)) + err := os.WriteFile(configFile, []byte(tt.configFile.data), 0o600) + require.NoError(t, err, "failed to write config") + + opts.privileged = true + opts.binds = []string{fmt.Sprintf("%s:%s:rw", configFile, tt.configFile.name)} + } + if tt.configBase64 != "" { + enc := base64.StdEncoding.EncodeToString([]byte(tt.configBase64)) + tt.env = append(tt.env, envbuilderEnv("DOCKER_CONFIG_BASE64", enc)) + } + + opts.env = append(opts.env, tt.env...) + + ctrID, err := runEnvbuilder(t, opts) + require.NoError(t, err) - // Verify that the Docker configuration secret file is removed - configJSONContainerPath := workingdir.Default.Join("config.json") - output = execContainer(t, ctr, "stat "+configJSONContainerPath) - require.Contains(t, output, "No such file or directory") + tt.validate(t, tt, ctrID, logbuf.String()) + + // Always verify that the Docker configuration secret file is removed. + output := execContainer(t, ctrID, "stat "+configJSONContainerPath) + require.Contains(t, output, "No such file or directory") + }) + } } func TestBuildPrintBuildOutput(t *testing.T) { @@ -2315,10 +2430,12 @@ func startContainerFromRef(ctx context.Context, t *testing.T, cli *client.Client } type runOpts struct { - image string - binds []string - env []string - volumes map[string]string + image string + privileged bool // Required for remounting. + binds []string + env []string + volumes map[string]string + logbuf *bytes.Buffer } // runEnvbuilder starts the envbuilder container with the given environment @@ -2356,17 +2473,22 @@ func runEnvbuilder(t *testing.T, opts runOpts) (string, error) { require.NoError(t, err, "failed to read image pull response") img = opts.image } + hostConfig := &container.HostConfig{ + NetworkMode: container.NetworkMode("host"), + Binds: opts.binds, + Mounts: mounts, + } + if opts.privileged { + hostConfig.CapAdd = append(hostConfig.CapAdd, "SYS_ADMIN") + hostConfig.Privileged = true + } ctr, err := cli.ContainerCreate(ctx, &container.Config{ Image: img, Env: opts.env, Labels: map[string]string{ testContainerLabel: "true", }, - }, &container.HostConfig{ - NetworkMode: container.NetworkMode("host"), - Binds: opts.binds, - Mounts: mounts, - }, nil, nil, "") + }, hostConfig, nil, nil, "") require.NoError(t, err) t.Cleanup(func() { _ = cli.ContainerRemove(ctx, ctr.ID, container.RemoveOptions{ @@ -2380,6 +2502,9 @@ func runEnvbuilder(t *testing.T, opts runOpts) (string, error) { logChan, errChan := streamContainerLogs(t, cli, ctr.ID) go func() { for log := range logChan { + if opts.logbuf != nil { + opts.logbuf.WriteString(log + "\n") + } if strings.HasPrefix(log, "=== Running init command") { errChan <- nil return diff --git a/options/options.go b/options/options.go index ed6bfa99..ebda2eeb 100644 --- a/options/options.go +++ b/options/options.go @@ -277,7 +277,9 @@ func (o *Options) CLI() serpent.OptionSet { Env: WithEnvPrefix("DOCKER_CONFIG_BASE64"), Value: serpent.StringOf(&o.DockerConfigBase64), Description: "The base64 encoded Docker config file that " + - "will be used to pull images from private container registries.", + "will be used to pull images from private container registries. " + + "When this is set, Docker configuration set via the DOCKER_CONFIG " + + "environment variable is ignored.", }, { Flag: "fallback-image", diff --git a/options/testdata/options.golden b/options/testdata/options.golden index 518f7741..397d5e0b 100644 --- a/options/testdata/options.golden +++ b/options/testdata/options.golden @@ -47,7 +47,9 @@ OPTIONS: --docker-config-base64 string, $ENVBUILDER_DOCKER_CONFIG_BASE64 The base64 encoded Docker config file that will be used to pull images - from private container registries. + from private container registries. When this is set, Docker + configuration set via the DOCKER_CONFIG environment variable is + ignored. --dockerfile-path string, $ENVBUILDER_DOCKERFILE_PATH The relative path to the Dockerfile that will be used to build the diff --git a/scripts/docsgen/main.go b/scripts/docsgen/main.go index b61de096..deb308f8 100644 --- a/scripts/docsgen/main.go +++ b/scripts/docsgen/main.go @@ -16,5 +16,5 @@ func main() { if err != nil { panic(err) } - fmt.Printf("%s updated successfully with the latest flags!", path) + fmt.Printf("%s updated successfully with the latest flags!\n", path) } From 532c6390801aa43cf611047f4e75cf3072e67909 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 2 Dec 2024 11:26:18 +0000 Subject: [PATCH 75/81] fix: enable logging for setup script when not a tty (#402) (#422) Co-authored-by: Mathias Fredriksson --- envbuilder.go | 30 +++++++++++++++++++----------- integration/integration_test.go | 3 +++ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 028a7bb2..a78c7666 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -1,7 +1,6 @@ package envbuilder import ( - "bufio" "bytes" "context" "encoding/base64" @@ -885,16 +884,8 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro cmd.Stderr = os.Stderr cmd.Stdin = os.Stdin } else { - var buf bytes.Buffer - go func() { - scanner := bufio.NewScanner(&buf) - for scanner.Scan() { - opts.Logger(log.LevelInfo, "%s", scanner.Text()) - } - }() - - cmd.Stdout = &buf - cmd.Stderr = &buf + cmd.Stdout = newWriteLogger(opts.Logger, log.LevelInfo) + cmd.Stderr = newWriteLogger(opts.Logger, log.LevelError) } err = cmd.Run() if err != nil { @@ -1814,6 +1805,23 @@ func onceErrFunc(f func() error) func() error { } } +type writeLogger struct { + logf log.Func + level log.Level +} + +func newWriteLogger(logf log.Func, level log.Level) io.Writer { + return writeLogger{logf: logf, level: level} +} + +func (l writeLogger) Write(p []byte) (n int, err error) { + lines := bytes.Split(p, []byte("\n")) + for _, line := range lines { + l.logf(l.level, "%s", line) + } + return len(p), nil +} + // Allows quick testing of layer caching using a local directory! func serveLocalRegistry(ctx context.Context, logf log.Func, layerCacheDir string) (string, func(), error) { noop := func() {} diff --git a/integration/integration_test.go b/integration/integration_test.go index 46e0e88c..a00525fc 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -127,6 +127,7 @@ func TestLogs(t *testing.T) { envbuilderEnv("GIT_URL", srv.URL), "CODER_AGENT_URL=" + logSrv.URL, "CODER_AGENT_TOKEN=" + token, + "ENVBUILDER_SETUP_SCRIPT=/bin/sh -c 'echo MY${NO_MATCH_ENV}_SETUP_SCRIPT_OUT; echo MY${NO_MATCH_ENV}_SETUP_SCRIPT_ERR' 1>&2", "ENVBUILDER_INIT_SCRIPT=env", }}) require.NoError(t, err) @@ -159,6 +160,8 @@ func TestLogs(t *testing.T) { require.NoError(t, err) logs := string(logBytes) require.Contains(t, logs, "CODER_AGENT_SUBSYSTEM=envbuilder") + require.Contains(t, logs, "MY_SETUP_SCRIPT_OUT") + require.Contains(t, logs, "MY_SETUP_SCRIPT_ERR") } func TestInitScriptInitCommand(t *testing.T) { From 4cba91f430f868f6307849b67e77fb2450adc368 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 2 Dec 2024 11:26:36 +0000 Subject: [PATCH 76/81] fix: do not error if failed to push image (#390) (#423) --- docs/env-variables.md | 1 + envbuilder.go | 7 +++-- integration/integration_test.go | 52 ++++++++++++++++++++++++++++++--- options/options.go | 12 ++++++++ options/testdata/options.golden | 5 ++++ 5 files changed, 71 insertions(+), 6 deletions(-) diff --git a/docs/env-variables.md b/docs/env-variables.md index cb7054f3..c198c705 100644 --- a/docs/env-variables.md +++ b/docs/env-variables.md @@ -18,6 +18,7 @@ | `--docker-config-base64` | `ENVBUILDER_DOCKER_CONFIG_BASE64` | | The base64 encoded Docker config file that will be used to pull images from private container registries. When this is set, Docker configuration set via the DOCKER_CONFIG environment variable is ignored. | | `--fallback-image` | `ENVBUILDER_FALLBACK_IMAGE` | | Specifies an alternative image to use when neither an image is declared in the devcontainer.json file nor a Dockerfile is present. If there's a build failure (from a faulty Dockerfile) or a misconfiguration, this image will be the substitute. Set ExitOnBuildFailure to true to halt the container if the build faces an issue. | | `--exit-on-build-failure` | `ENVBUILDER_EXIT_ON_BUILD_FAILURE` | | Terminates the container upon a build failure. This is handy when preferring the FALLBACK_IMAGE in cases where no devcontainer.json or image is provided. However, it ensures that the container stops if the build process encounters an error. | +| `--exit-on-push-failure` | `ENVBUILDER_EXIT_ON_PUSH_FAILURE` | | ExitOnPushFailure terminates the container upon a push failure. This is useful if failure to push the built image should abort execution and result in an error. | | `--force-safe` | `ENVBUILDER_FORCE_SAFE` | | Ignores any filesystem safety checks. This could cause serious harm to your system! This is used in cases where bypass is needed to unblock customers. | | `--insecure` | `ENVBUILDER_INSECURE` | | Bypass TLS verification when cloning and pulling from container registries. | | `--ignore-paths` | `ENVBUILDER_IGNORE_PATHS` | | The comma separated list of paths to ignore when building the workspace. | diff --git a/envbuilder.go b/envbuilder.go index a78c7666..276c7fc3 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -581,10 +581,13 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro endStage("🏗️ Built image!") if opts.PushImage { endStage = startStage("🏗️ Pushing image...") - if err := executor.DoPush(image, kOpts); err != nil { + if err := executor.DoPush(image, kOpts); err == nil { + endStage("🏗️ Pushed image!") + } else if !opts.ExitOnPushFailure { + endStage("⚠️️ Failed to push image!") + } else { return nil, xerrors.Errorf("do push: %w", err) } - endStage("🏗️ Pushed image!") } return image, err diff --git a/integration/integration_test.go b/integration/integration_test.go index a00525fc..225509eb 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -1882,9 +1882,10 @@ RUN date --utc > /root/date.txt`, testImageAlpine), _, err = remote.Image(ref, remoteAuthOpt) require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push") - // When: we run envbuilder with PUSH_IMAGE set + // When: we run envbuilder with PUSH_IMAGE and EXIT_ON_PUSH_FAILURE set _, err = runEnvbuilder(t, runOpts{env: append(opts, envbuilderEnv("PUSH_IMAGE", "1"), + envbuilderEnv("EXIT_ON_PUSH_FAILURE", "1"), )}) // Then: it should fail with an Unauthorized error require.ErrorContains(t, err, "401 Unauthorized", "expected unauthorized error using no auth when cache repo requires it") @@ -2077,7 +2078,7 @@ RUN date --utc > /root/date.txt`, testImageAlpine), require.ErrorContains(t, err, "--cache-repo must be set when using --push-image") }) - t.Run("PushErr", func(t *testing.T) { + t.Run("PushErr/ExitOnPushFail", func(t *testing.T) { t.Parallel() srv := gittest.CreateGitServer(t, gittest.Options{ @@ -2107,12 +2108,50 @@ RUN date --utc > /root/date.txt`, testImageAlpine), envbuilderEnv("GIT_URL", srv.URL), envbuilderEnv("CACHE_REPO", notRegURL), envbuilderEnv("PUSH_IMAGE", "1"), + envbuilderEnv("EXIT_ON_PUSH_FAILURE", "1"), }}) // Then: envbuilder should fail with a descriptive error require.ErrorContains(t, err, "failed to push to destination") }) + t.Run("PushErr/NoExitOnPushFail", func(t *testing.T) { + t.Parallel() + + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ + ".devcontainer/Dockerfile": fmt.Sprintf(`FROM %s +USER root +ARG WORKDIR=/ +WORKDIR $WORKDIR +ENV FOO=bar +RUN echo $FOO > /root/foo.txt +RUN date --utc > /root/date.txt`, testImageAlpine), + ".devcontainer/devcontainer.json": `{ + "name": "Test", + "build": { + "dockerfile": "Dockerfile" + }, + }`, + }, + }) + + // Given: registry is not set up (in this case, not a registry) + notRegSrv := httptest.NewServer(http.NotFoundHandler()) + notRegURL := strings.TrimPrefix(notRegSrv.URL, "http://") + "/test" + + // When: we run envbuilder with PUSH_IMAGE set + _, err := runEnvbuilder(t, runOpts{env: []string{ + envbuilderEnv("GIT_URL", srv.URL), + envbuilderEnv("CACHE_REPO", notRegURL), + envbuilderEnv("PUSH_IMAGE", "1"), + envbuilderEnv("EXIT_ON_PUSH_FAILURE", "0"), + }}) + + // Then: envbuilder should not fail + require.NoError(t, err) + }) + t.Run("CacheAndPushDevcontainerFeatures", func(t *testing.T) { t.Parallel() @@ -2354,8 +2393,13 @@ func pushImage(t *testing.T, ref name.Reference, remoteOpt remote.Option, env .. if remoteOpt != nil { remoteOpts = append(remoteOpts, remoteOpt) } - - _, err := runEnvbuilder(t, runOpts{env: append(env, envbuilderEnv("PUSH_IMAGE", "1"))}) + opts := runOpts{ + env: append(env, + envbuilderEnv("PUSH_IMAGE", "1"), + envbuilderEnv("EXIT_ON_PUSH_FAILURE", "1"), + ), + } + _, err := runEnvbuilder(t, opts) require.NoError(t, err, "envbuilder push image failed") img, err := remote.Image(ref, remoteOpts...) diff --git a/options/options.go b/options/options.go index ebda2eeb..b2497d3c 100644 --- a/options/options.go +++ b/options/options.go @@ -78,6 +78,10 @@ type Options struct { // devcontainer.json or image is provided. However, it ensures that the // container stops if the build process encounters an error. ExitOnBuildFailure bool + // ExitOnPushFailure terminates the container upon a push failure. This is + // useful if failure to push the built image should abort execution + // and result in an error. + ExitOnPushFailure bool // ForceSafe ignores any filesystem safety checks. This could cause serious // harm to your system! This is used in cases where bypass is needed to // unblock customers. @@ -301,6 +305,14 @@ func (o *Options) CLI() serpent.OptionSet { "no devcontainer.json or image is provided. However, it ensures " + "that the container stops if the build process encounters an error.", }, + { + Flag: "exit-on-push-failure", + Env: WithEnvPrefix("EXIT_ON_PUSH_FAILURE"), + Value: serpent.BoolOf(&o.ExitOnPushFailure), + Description: "ExitOnPushFailure terminates the container upon a push failure. " + + "This is useful if failure to push the built image should abort execution " + + "and result in an error.", + }, { Flag: "force-safe", Env: WithEnvPrefix("FORCE_SAFE"), diff --git a/options/testdata/options.golden b/options/testdata/options.golden index 397d5e0b..f5c5729d 100644 --- a/options/testdata/options.golden +++ b/options/testdata/options.golden @@ -62,6 +62,11 @@ OPTIONS: image is provided. However, it ensures that the container stops if the build process encounters an error. + --exit-on-push-failure bool, $ENVBUILDER_EXIT_ON_PUSH_FAILURE + ExitOnPushFailure terminates the container upon a push failure. This + is useful if failure to push the built image should abort execution + and result in an error. + --export-env-file string, $ENVBUILDER_EXPORT_ENV_FILE Optional file path to a .env file where envbuilder will dump environment variables from devcontainer.json and the built container From 5b1acda0ce03b5974f29ace38ad40c7b08e69f2d Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 2 Dec 2024 11:27:02 +0000 Subject: [PATCH 77/81] feat: allow changing default workspaces folder (#406) (#424) (cherry picked from commit 24ef801) Co-authored-by: Mathias Fredriksson --- docs/env-variables.md | 3 +- integration/integration_test.go | 22 ++++++++++++++ options/defaults.go | 23 ++++++++------- options/defaults_test.go | 52 +++++++++++++++++++++++++-------- options/options.go | 19 ++++++++++-- options/testdata/options.golden | 6 ++++ 6 files changed, 99 insertions(+), 26 deletions(-) diff --git a/docs/env-variables.md b/docs/env-variables.md index c198c705..427a3f63 100644 --- a/docs/env-variables.md +++ b/docs/env-variables.md @@ -31,7 +31,8 @@ | `--git-ssh-private-key-path` | `ENVBUILDER_GIT_SSH_PRIVATE_KEY_PATH` | | Path to an SSH private key to be used for Git authentication. If this is set, then GIT_SSH_PRIVATE_KEY_BASE64 cannot be set. | | `--git-ssh-private-key-base64` | `ENVBUILDER_GIT_SSH_PRIVATE_KEY_BASE64` | | Base64 encoded SSH private key to be used for Git authentication. If this is set, then GIT_SSH_PRIVATE_KEY_PATH cannot be set. | | `--git-http-proxy-url` | `ENVBUILDER_GIT_HTTP_PROXY_URL` | | The URL for the HTTP proxy. This is optional. | -| `--workspace-folder` | `ENVBUILDER_WORKSPACE_FOLDER` | | The path to the workspace folder that will be built. This is optional. | +| `--workspace-base-dir` | `ENVBUILDER_WORKSPACE_BASE_DIR` | `/workspaces` | The path under which workspaces will be placed when workspace folder option is not given. | +| `--workspace-folder` | `ENVBUILDER_WORKSPACE_FOLDER` | | The path to the workspace folder that will be built. This is optional. Defaults to `[workspace base dir]/[name]` where name is the name of the repository or `empty`. | | `--ssl-cert-base64` | `ENVBUILDER_SSL_CERT_BASE64` | | The content of an SSL cert file. This is useful for self-signed certificates. | | `--export-env-file` | `ENVBUILDER_EXPORT_ENV_FILE` | | Optional file path to a .env file where envbuilder will dump environment variables from devcontainer.json and the built container image. | | `--post-start-script-path` | `ENVBUILDER_POST_START_SCRIPT_PATH` | | The path to a script that will be created by envbuilder based on the postStartCommand in devcontainer.json, if any is specified (otherwise the script is not created). If this is set, the specified InitCommand should check for the presence of this script and execute it after successful startup. | diff --git a/integration/integration_test.go b/integration/integration_test.go index 225509eb..d10fac1c 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -821,6 +821,28 @@ func TestBuildFromDevcontainerInCustomPath(t *testing.T) { require.Equal(t, "hello", strings.TrimSpace(output)) } +func TestBuildFromCustomWorkspaceBaseDir(t *testing.T) { + t.Parallel() + + // Ensures that a Git repository with a devcontainer.json is cloned and built. + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ + "Dockerfile": "FROM " + testImageUbuntu, + }, + }) + ctr, err := runEnvbuilder(t, runOpts{ + env: []string{ + envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"), + envbuilderEnv("WORKSPACE_BASE_DIR", "/foo"), + envbuilderEnv("GIT_URL", srv.URL), + }, + }) + require.NoError(t, err) + + output := execContainer(t, ctr, "readlink /proc/1/cwd") + require.Contains(t, output, "/foo/") +} + func TestBuildFromDevcontainerInSubfolder(t *testing.T) { t.Parallel() diff --git a/options/defaults.go b/options/defaults.go index b9e47a80..a6fd145b 100644 --- a/options/defaults.go +++ b/options/defaults.go @@ -12,29 +12,29 @@ import ( "github.com/coder/envbuilder/internal/workingdir" ) -// EmptyWorkspaceDir is the path to a workspace that has -// nothing going on... it's empty! -var EmptyWorkspaceDir = "/workspaces/empty" - // DefaultWorkspaceFolder returns the default workspace folder // for a given repository URL. -func DefaultWorkspaceFolder(repoURL string) string { +func DefaultWorkspaceFolder(workspacesFolder, repoURL string) string { + // emptyWorkspaceDir is the path to a workspace that has + // nothing going on... it's empty! + emptyWorkspaceDir := workspacesFolder + "/empty" + if repoURL == "" { - return EmptyWorkspaceDir + return emptyWorkspaceDir } parsed, err := giturls.Parse(repoURL) if err != nil { - return EmptyWorkspaceDir + return emptyWorkspaceDir } repo := path.Base(parsed.Path) // Giturls parsing never actually fails since ParseLocal never // errors and places the entire URL in the Path field. This check // ensures it's at least a Unix path containing forwardslash. if repo == repoURL || repo == "/" || repo == "." || repo == "" { - return EmptyWorkspaceDir + return emptyWorkspaceDir } repo = strings.TrimSuffix(repo, ".git") - return fmt.Sprintf("/workspaces/%s", repo) + return fmt.Sprintf("%s/%s", workspacesFolder, repo) } func (o *Options) SetDefaults() { @@ -59,8 +59,11 @@ func (o *Options) SetDefaults() { if o.Filesystem == nil { o.Filesystem = chmodfs.New(osfs.New("/")) } + if o.WorkspaceBaseDir == "" { + o.WorkspaceBaseDir = "/workspaces" + } if o.WorkspaceFolder == "" { - o.WorkspaceFolder = DefaultWorkspaceFolder(o.GitURL) + o.WorkspaceFolder = DefaultWorkspaceFolder(o.WorkspaceBaseDir, o.GitURL) } if o.BinaryPath == "" { o.BinaryPath = "/.envbuilder/bin/envbuilder" diff --git a/options/defaults_test.go b/options/defaults_test.go index de57365d..3efd8f25 100644 --- a/options/defaults_test.go +++ b/options/defaults_test.go @@ -17,83 +17,110 @@ func TestDefaultWorkspaceFolder(t *testing.T) { successTests := []struct { name string + baseDir string gitURL string expected string }{ { name: "HTTP", + baseDir: "/workspaces", gitURL: "https://github.com/coder/envbuilder.git", expected: "/workspaces/envbuilder", }, { name: "SSH", + baseDir: "/workspaces", gitURL: "git@github.com:coder/envbuilder.git", expected: "/workspaces/envbuilder", }, { name: "username and password", + baseDir: "/workspaces", gitURL: "https://username:password@github.com/coder/envbuilder.git", expected: "/workspaces/envbuilder", }, { name: "trailing", + baseDir: "/workspaces", gitURL: "https://github.com/coder/envbuilder.git/", expected: "/workspaces/envbuilder", }, { name: "trailing-x2", + baseDir: "/workspaces", gitURL: "https://github.com/coder/envbuilder.git//", expected: "/workspaces/envbuilder", }, { name: "no .git", + baseDir: "/workspaces", gitURL: "https://github.com/coder/envbuilder", expected: "/workspaces/envbuilder", }, { name: "trailing no .git", + baseDir: "/workspaces", gitURL: "https://github.com/coder/envbuilder/", expected: "/workspaces/envbuilder", }, { name: "fragment", + baseDir: "/workspaces", gitURL: "https://github.com/coder/envbuilder.git#feature-branch", expected: "/workspaces/envbuilder", }, { name: "fragment-trailing", + baseDir: "/workspaces", gitURL: "https://github.com/coder/envbuilder.git/#refs/heads/feature-branch", expected: "/workspaces/envbuilder", }, { name: "fragment-trailing no .git", + baseDir: "/workspaces", gitURL: "https://github.com/coder/envbuilder/#refs/heads/feature-branch", expected: "/workspaces/envbuilder", }, { name: "space", + baseDir: "/workspaces", gitURL: "https://github.com/coder/env%20builder.git", expected: "/workspaces/env builder", }, { name: "Unix path", + baseDir: "/workspaces", gitURL: "/repo", expected: "/workspaces/repo", }, { name: "Unix subpath", + baseDir: "/workspaces", gitURL: "/path/to/repo", expected: "/workspaces/repo", }, { name: "empty", + baseDir: "/workspaces", gitURL: "", - expected: options.EmptyWorkspaceDir, + expected: "/workspaces/empty", + }, + { + name: "non default workspaces folder", + baseDir: "/foo", + gitURL: "https://github.com/coder/envbuilder.git", + expected: "/foo/envbuilder", + }, + { + name: "non default workspaces folder empty git URL", + baseDir: "/foo", + gitURL: "", + expected: "/foo/empty", }, } for _, tt := range successTests { t.Run(tt.name, func(t *testing.T) { - dir := options.DefaultWorkspaceFolder(tt.gitURL) + dir := options.DefaultWorkspaceFolder(tt.baseDir, tt.gitURL) require.Equal(t, tt.expected, dir) }) } @@ -125,8 +152,8 @@ func TestDefaultWorkspaceFolder(t *testing.T) { } for _, tt := range invalidTests { t.Run(tt.name, func(t *testing.T) { - dir := options.DefaultWorkspaceFolder(tt.invalidURL) - require.Equal(t, options.EmptyWorkspaceDir, dir) + dir := options.DefaultWorkspaceFolder("/workspaces", tt.invalidURL) + require.Equal(t, "/workspaces/empty", dir) }) } } @@ -135,14 +162,15 @@ func TestOptions_SetDefaults(t *testing.T) { t.Parallel() expected := options.Options{ - InitScript: "sleep infinity", - InitCommand: "/bin/sh", - IgnorePaths: []string{"/var/run", "/product_uuid", "/product_name"}, - Filesystem: chmodfs.New(osfs.New("/")), - GitURL: "", - WorkspaceFolder: options.EmptyWorkspaceDir, - MagicDirBase: "/.envbuilder", - BinaryPath: "/.envbuilder/bin/envbuilder", + InitScript: "sleep infinity", + InitCommand: "/bin/sh", + IgnorePaths: []string{"/var/run", "/product_uuid", "/product_name"}, + Filesystem: chmodfs.New(osfs.New("/")), + GitURL: "", + WorkspaceBaseDir: "/workspaces", + WorkspaceFolder: "/workspaces/empty", + MagicDirBase: "/.envbuilder", + BinaryPath: "/.envbuilder/bin/envbuilder", } var actual options.Options diff --git a/options/options.go b/options/options.go index b2497d3c..9b9edda2 100644 --- a/options/options.go +++ b/options/options.go @@ -117,8 +117,12 @@ type Options struct { GitSSHPrivateKeyBase64 string // GitHTTPProxyURL is the URL for the HTTP proxy. This is optional. GitHTTPProxyURL string + // WorkspaceBaseDir is the path under which workspaces will be placed when + // workspace folder option is not given. + WorkspaceBaseDir string // WorkspaceFolder is the path to the workspace folder that will be built. - // This is optional. + // This is optional. Defaults to `[workspace base dir]/[name]` where name is + // the name of the repository or "empty". WorkspaceFolder string // SSLCertBase64 is the content of an SSL cert file. This is useful for // self-signed certificates. @@ -394,12 +398,21 @@ func (o *Options) CLI() serpent.OptionSet { Value: serpent.StringOf(&o.GitHTTPProxyURL), Description: "The URL for the HTTP proxy. This is optional.", }, + { + Flag: "workspace-base-dir", + Env: WithEnvPrefix("WORKSPACE_BASE_DIR"), + Value: serpent.StringOf(&o.WorkspaceBaseDir), + Default: "/workspaces", + Description: "The path under which workspaces will be placed when " + + "workspace folder option is not given.", + }, { Flag: "workspace-folder", Env: WithEnvPrefix("WORKSPACE_FOLDER"), Value: serpent.StringOf(&o.WorkspaceFolder), - Description: "The path to the workspace folder that will " + - "be built. This is optional.", + Description: "The path to the workspace folder that will be built. " + + "This is optional. Defaults to `[workspace base dir]/[name]` where " + + "name is the name of the repository or `empty`.", }, { Flag: "ssl-cert-base64", diff --git a/options/testdata/options.golden b/options/testdata/options.golden index f5c5729d..203c197b 100644 --- a/options/testdata/options.golden +++ b/options/testdata/options.golden @@ -177,6 +177,12 @@ OPTIONS: --verbose bool, $ENVBUILDER_VERBOSE Enable verbose logging. + --workspace-base-dir string, $ENVBUILDER_WORKSPACE_BASE_DIR (default: /workspaces) + The path under which workspaces will be placed when workspace folder + option is not given. + --workspace-folder string, $ENVBUILDER_WORKSPACE_FOLDER The path to the workspace folder that will be built. This is optional. + Defaults to `[workspace base dir]/[name]` where name is the name of + the repository or `empty`. From 84e1c40b1c51f79c675710273e308eec99a7f696 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 2 Dec 2024 11:29:08 +0000 Subject: [PATCH 78/81] fix: update kaniko fork to fix BLOB_UNKNOWN errors (#410) (#425) Co-authored-by: Mathias Fredriksson --- Makefile | 6 ++- devcontainer/devcontainer.go | 6 +-- envbuilder.go | 9 ++++ go.mod | 27 +++++----- go.sum | 52 ++++++++++---------- integration/integration_test.go | 41 +++++++++++++-- integration/testdata/blob-unknown/Dockerfile | 7 +++ 7 files changed, 101 insertions(+), 47 deletions(-) create mode 100644 integration/testdata/blob-unknown/Dockerfile diff --git a/Makefile b/Makefile index ce079015..14ed5182 100644 --- a/Makefile +++ b/Makefile @@ -67,7 +67,7 @@ test-registry-container: .registry-cache # Pulls images referenced in integration tests and pushes them to the local cache. .PHONY: test-images-push -test-images-push: .registry-cache/docker/registry/v2/repositories/envbuilder-test-alpine .registry-cache/docker/registry/v2/repositories/envbuilder-test-ubuntu .registry-cache/docker/registry/v2/repositories/envbuilder-test-codercom-code-server +test-images-push: .registry-cache/docker/registry/v2/repositories/envbuilder-test-alpine .registry-cache/docker/registry/v2/repositories/envbuilder-test-ubuntu .registry-cache/docker/registry/v2/repositories/envbuilder-test-codercom-code-server .registry-cache/docker/registry/v2/repositories/envbuilder-test-blob-unknown .PHONY: test-images-pull test-images-pull: @@ -77,6 +77,7 @@ test-images-pull: docker tag ubuntu:latest localhost:5000/envbuilder-test-ubuntu:latest docker pull codercom/code-server:latest docker tag codercom/code-server:latest localhost:5000/envbuilder-test-codercom-code-server:latest + docker build -t localhost:5000/envbuilder-test-blob-unknown:latest -f integration/testdata/blob-unknown/Dockerfile integration/testdata/blob-unknown .registry-cache: mkdir -p .registry-cache && chmod -R ag+w .registry-cache @@ -89,3 +90,6 @@ test-images-pull: .registry-cache/docker/registry/v2/repositories/envbuilder-test-codercom-code-server: docker push localhost:5000/envbuilder-test-codercom-code-server:latest + +.registry-cache/docker/registry/v2/repositories/envbuilder-test-blob-unknown: + docker push localhost:5000/envbuilder-test-blob-unknown:latest diff --git a/devcontainer/devcontainer.go b/devcontainer/devcontainer.go index 6135c0ef..0bf7cc35 100644 --- a/devcontainer/devcontainer.go +++ b/devcontainer/devcontainer.go @@ -400,11 +400,11 @@ func ImageFromDockerfile(dockerfileContent string) (name.Reference, error) { arg = strings.TrimSpace(arg) if strings.Contains(arg, "=") { parts := strings.SplitN(arg, "=", 2) - key, err := lexer.ProcessWord(parts[0], args) + key, _, err := lexer.ProcessWord(parts[0], shell.EnvsFromSlice(args)) if err != nil { return nil, fmt.Errorf("processing %q: %w", line, err) } - val, err := lexer.ProcessWord(parts[1], args) + val, _, err := lexer.ProcessWord(parts[1], shell.EnvsFromSlice(args)) if err != nil { return nil, fmt.Errorf("processing %q: %w", line, err) } @@ -421,7 +421,7 @@ func ImageFromDockerfile(dockerfileContent string) (name.Reference, error) { if imageRef == "" { return nil, fmt.Errorf("no FROM directive found") } - imageRef, err := lexer.ProcessWord(imageRef, args) + imageRef, _, err := lexer.ProcessWord(imageRef, shell.EnvsFromSlice(args)) if err != nil { return nil, fmt.Errorf("processing %q: %w", imageRef, err) } diff --git a/envbuilder.go b/envbuilder.go index 276c7fc3..6f3231d1 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -581,6 +581,15 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro endStage("🏗️ Built image!") if opts.PushImage { endStage = startStage("🏗️ Pushing image...") + // To debug registry issues, enable logging: + // + // import ( + // stdlog "log" + // reglogs "github.com/google/go-containerregistry/pkg/logs" + // ) + // reglogs.Debug = stdlog.New(os.Stderr, "", 0) + // reglogs.Warn = stdlog.New(os.Stderr, "", 0) + // reglogs.Progress = stdlog.New(os.Stderr, "", 0) if err := executor.DoPush(image, kOpts); err == nil { endStage("🏗️ Pushed image!") } else if !opts.ExitOnPushFailure { diff --git a/go.mod b/go.mod index 9fa1d696..c51f4227 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22.4 // There are a few options we need added to Kaniko! // See: https://github.com/GoogleContainerTools/kaniko/compare/main...coder:kaniko:main -replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240925122543-caa18967f374 +replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20241120132148-131d6094d781 // Required to import codersdk due to gvisor dependency. replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20240702054557-aa558fbe5374 @@ -19,8 +19,8 @@ require ( github.com/coder/serpent v0.7.0 github.com/containerd/platforms v0.2.1 github.com/distribution/distribution/v3 v3.0.0-alpha.1 - github.com/docker/cli v27.2.0+incompatible - github.com/docker/docker v26.1.5+incompatible + github.com/docker/cli v27.2.1+incompatible + github.com/docker/docker v27.3.1+incompatible github.com/fatih/color v1.17.0 github.com/gliderlabs/ssh v0.3.7 github.com/go-git/go-billy/v5 v5.5.0 @@ -31,7 +31,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/mattn/go-isatty v0.0.20 - github.com/moby/buildkit v0.13.1 + github.com/moby/buildkit v0.16.0 github.com/otiai10/copy v1.14.0 github.com/prometheus/procfs v0.15.1 github.com/sirupsen/logrus v1.9.3 @@ -100,14 +100,12 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/charmbracelet/lipgloss v0.8.0 // indirect github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 // indirect - github.com/cilium/ebpf v0.12.3 // indirect github.com/cloudflare/circl v1.3.7 // indirect github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 // indirect github.com/coder/quartz v0.1.0 // indirect github.com/coder/terraform-provider-coder v0.23.0 // indirect github.com/containerd/cgroups v1.1.0 // indirect - github.com/containerd/cgroups/v3 v3.0.2 // indirect - github.com/containerd/containerd v1.7.19 // indirect + github.com/containerd/containerd v1.7.21 // indirect github.com/containerd/containerd/api v1.7.19 // indirect github.com/containerd/continuity v0.4.3 // indirect github.com/containerd/errdefs v0.1.0 // indirect @@ -115,10 +113,9 @@ require ( github.com/containerd/log v0.1.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect github.com/containerd/ttrpc v1.2.5 // indirect - github.com/containerd/typeurl/v2 v2.1.1 // indirect + github.com/containerd/typeurl/v2 v2.2.0 // indirect github.com/coreos/go-iptables v0.6.0 // indirect github.com/coreos/go-oidc/v3 v3.10.0 // indirect - github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect @@ -151,7 +148,7 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.2 // indirect github.com/google/nftables v0.2.0 // indirect - github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect + github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect github.com/gorilla/handlers v1.5.1 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 // indirect @@ -164,7 +161,7 @@ require ( github.com/hashicorp/go-version v1.7.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/golang-lru/arc/v2 v2.0.5 // indirect - github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl/v2 v2.21.0 // indirect github.com/hashicorp/logutils v1.0.0 // indirect github.com/hashicorp/terraform-plugin-go v0.12.0 // indirect @@ -202,11 +199,12 @@ require ( github.com/moby/patternmatcher v0.6.0 // indirect github.com/moby/swarmkit/v2 v2.0.0-20230315203717-e28e8ba9bc83 // indirect github.com/moby/sys/mount v0.3.3 // indirect - github.com/moby/sys/mountinfo v0.7.1 // indirect + github.com/moby/sys/mountinfo v0.7.2 // indirect github.com/moby/sys/sequential v0.5.0 // indirect - github.com/moby/sys/signal v0.7.0 // indirect + github.com/moby/sys/signal v0.7.1 // indirect github.com/moby/sys/symlink v0.2.0 // indirect - github.com/moby/sys/user v0.1.0 // indirect + github.com/moby/sys/user v0.3.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.0 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/muesli/reflow v0.3.0 // indirect @@ -245,6 +243,7 @@ require ( github.com/tailscale/wireguard-go v0.0.0-20231121184858-cc193a0b3272 // indirect github.com/tcnksm/go-httpstat v0.2.0 // indirect github.com/tinylib/msgp v1.1.8 // indirect + github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4 // indirect github.com/twpayne/go-vfs/v5 v5.0.4 // indirect github.com/u-root/uio v0.0.0-20240209044354-b3d14b93376a // indirect github.com/valyala/fasthttp v1.55.0 // indirect diff --git a/go.sum b/go.sum index 07dc01db..29fba2ef 100644 --- a/go.sum +++ b/go.sum @@ -171,8 +171,8 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352 h1:L/EjCuZxs5tOcqqCaASj/nu65TRYEFcTt8qRQfHZXX0= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352/go.mod h1:P1KoQSgnKEAG6Mnd3YlGzAophty+yKA9VV48LpfNRvo= -github.com/coder/kaniko v0.0.0-20240925122543-caa18967f374 h1:/cyXf0vTSwFh7evQqeWHXXl14aRfC4CsNIYxOenJytQ= -github.com/coder/kaniko v0.0.0-20240925122543-caa18967f374/go.mod h1:XoTDIhNF0Ll4tLmRYdOn31udU9w5zFrY2PME/crSRCA= +github.com/coder/kaniko v0.0.0-20241120132148-131d6094d781 h1:/4SMdrjLQL1BseLSnMd9nYQSI+E63CXcyFGC7ZHHj8I= +github.com/coder/kaniko v0.0.0-20241120132148-131d6094d781/go.mod h1:3rM/KOQ4LgF8mE+O1P6pLDa/E57mzxIxNdUOMKi1qpg= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 h1:3A0ES21Ke+FxEM8CXx9n47SZOKOpgSE1bbJzlE4qPVs= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0/go.mod h1:5UuS2Ts+nTToAMeOjNlnHFkPahrtDkmpydBen/3wgZc= github.com/coder/quartz v0.1.0 h1:cLL+0g5l7xTf6ordRnUMMiZtRE8Sq5LxpghS63vEXrQ= @@ -187,10 +187,8 @@ github.com/coder/terraform-provider-coder v0.23.0 h1:DuNLWxhnGlXyG0g+OCAZRI6xd8+ github.com/coder/terraform-provider-coder v0.23.0/go.mod h1:wMun9UZ9HT2CzF6qPPBup1odzBpVUc0/xSFoXgdI3tk= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= -github.com/containerd/cgroups/v3 v3.0.2 h1:f5WFqIVSgo5IZmtTT3qVBo6TzI1ON6sycSBKkymb9L0= -github.com/containerd/cgroups/v3 v3.0.2/go.mod h1:JUgITrzdFqp42uI2ryGA+ge0ap/nxzYgkGmIcetmErE= -github.com/containerd/containerd v1.7.19 h1:/xQ4XRJ0tamDkdzrrBAUy/LE5nCcxFKdBm4EcPrSMEE= -github.com/containerd/containerd v1.7.19/go.mod h1:h4FtNYUUMB4Phr6v+xG89RYKj9XccvbNSCKjdufCrkc= +github.com/containerd/containerd v1.7.21 h1:USGXRK1eOC/SX0L195YgxTHb0a00anxajOzgfN0qrCA= +github.com/containerd/containerd v1.7.21/go.mod h1:e3Jz1rYRUZ2Lt51YrH9Rz0zPyJBOlSvB3ghr2jbVD8g= github.com/containerd/containerd/api v1.7.19 h1:VWbJL+8Ap4Ju2mx9c9qS1uFSB1OVYr5JJrW2yT5vFoA= github.com/containerd/containerd/api v1.7.19/go.mod h1:fwGavl3LNwAV5ilJ0sbrABL44AQxmNjDRcwheXDb6Ig= github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8= @@ -207,15 +205,13 @@ github.com/containerd/stargz-snapshotter/estargz v0.15.1 h1:eXJjw9RbkLFgioVaTG+G github.com/containerd/stargz-snapshotter/estargz v0.15.1/go.mod h1:gr2RNwukQ/S9Nv33Lt6UC7xEx58C+LHRdoqbEKjz1Kk= github.com/containerd/ttrpc v1.2.5 h1:IFckT1EFQoFBMG4c3sMdT8EP3/aKfumK1msY+Ze4oLU= github.com/containerd/ttrpc v1.2.5/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= -github.com/containerd/typeurl/v2 v2.1.1 h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9FqcttATPO/4= -github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0= +github.com/containerd/typeurl/v2 v2.2.0 h1:6NBDbQzr7I5LHgp34xAXYF5DOTQDn05X58lsPEmzLso= +github.com/containerd/typeurl/v2 v2.2.0/go.mod h1:8XOOxnyatxSWuG8OfsZXVnAF4iZfedjS/8UHSPJnX4g= github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk= github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= github.com/coreos/go-oidc/v3 v3.10.0 h1:tDnXHnLyiTVyT/2zLDGj09pFPkhND8Gl8lnTRhoEaJU= github.com/coreos/go-oidc/v3 v3.10.0/go.mod h1:5j11xcw0D3+SGxn6Z/WFADsgcWVMyNAlSQupk0KK3ac= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0= github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= @@ -236,12 +232,12 @@ github.com/distribution/distribution/v3 v3.0.0-alpha.1 h1:jn7I1gvjOvmLztH1+1cLiU github.com/distribution/distribution/v3 v3.0.0-alpha.1/go.mod h1:LCp4JZp1ZalYg0W/TN05jarCQu+h4w7xc7ZfQF4Y/cY= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/cli v27.2.0+incompatible h1:yHD1QEB1/0vr5eBNpu8tncu8gWxg8EydFPOSKHzXSMM= -github.com/docker/cli v27.2.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v27.2.1+incompatible h1:U5BPtiD0viUzjGAjV1p0MGB8eVA3L3cbIrnyWmSJI70= +github.com/docker/cli v27.2.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v26.1.5+incompatible h1:NEAxTwEjxV6VbBMBoGG3zPqbiJosIApZjxlbrG9q3/g= -github.com/docker/docker v26.1.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.3.1+incompatible h1:KttF0XoteNTicmUtBO0L2tP+J7FGRFTjaEF4k6WdhfI= +github.com/docker/docker v27.3.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo= github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= @@ -394,8 +390,8 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/nftables v0.2.0 h1:PbJwaBmbVLzpeldoeUKGkE2RjstrjPKMl6oLrfEJ6/8= github.com/google/nftables v0.2.0/go.mod h1:Beg6V6zZ3oEn0JuiUQ4wqwuyqqzasOltcoXPtgLbFp4= -github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b h1:h9U78+dx9a4BKdQkBBos92HalKpaGKHrp+3Uo6yTodo= -github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -439,8 +435,8 @@ github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iP github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/arc/v2 v2.0.5 h1:l2zaLDubNhW4XO3LnliVj0GXO3+/CGNJAg1dcN2Fpfw= github.com/hashicorp/golang-lru/arc/v2 v2.0.5/go.mod h1:ny6zBSQZi2JxIeYcv7kt2sH2PXJtirBN7RDhRpxPkxU= -github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4= -github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hc-install v0.7.0 h1:Uu9edVqjKQxxuD28mR5TikkKDd/p55S8vzPC1659aBk= github.com/hashicorp/hc-install v0.7.0/go.mod h1:ELmmzZlGnEcqoUMKUuykHaPCIR1sYLYX+KSggWSKZuA= github.com/hashicorp/hcl/v2 v2.21.0 h1:lve4q/o/2rqwYOgUg3y3V2YPyD1/zkCLGjIV74Jit14= @@ -556,8 +552,8 @@ github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c h1:cqn374 github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/moby/buildkit v0.13.1 h1:L8afOFhPq2RPJJSr/VyzbufwID7jquZVB7oFHbPRcPE= -github.com/moby/buildkit v0.13.1/go.mod h1:aNmNQKLBFYAOFuzQjR3VA27/FijlvtBD1pjNwTSN37k= +github.com/moby/buildkit v0.16.0 h1:wOVBj1o5YNVad/txPQNXUXdelm7Hs/i0PUFjzbK0VKE= +github.com/moby/buildkit v0.16.0/go.mod h1:Xqx/5GlrqE1yIRORk0NSCVDFpQAU1WjlT6KHYZdisIQ= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= @@ -569,16 +565,18 @@ github.com/moby/swarmkit/v2 v2.0.0-20230315203717-e28e8ba9bc83/go.mod h1:GvjR7mC github.com/moby/sys/mount v0.3.3 h1:fX1SVkXFJ47XWDoeFW4Sq7PdQJnV2QIDZAqjNqgEjUs= github.com/moby/sys/mount v0.3.3/go.mod h1:PBaEorSNTLG5t/+4EgukEQVlAvVEc6ZjTySwKdqp5K0= github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= -github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g= -github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= +github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= +github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= -github.com/moby/sys/signal v0.7.0 h1:25RW3d5TnQEoKvRbEKUGay6DCQ46IxAVTT9CUMgmsSI= -github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= +github.com/moby/sys/signal v0.7.1 h1:PrQxdvxcGijdo6UXXo/lU/TvHUWyPhj7UOpSo8tuvk0= +github.com/moby/sys/signal v0.7.1/go.mod h1:Se1VGehYokAkrSQwL4tDzHvETwUZlnY7S5XtQ50mQp8= github.com/moby/sys/symlink v0.2.0 h1:tk1rOM+Ljp0nFmfOIBtlV3rTDlWOwFRhjEeAhZB0nZc= github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs= -github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= -github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= +github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo= +github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -729,6 +727,8 @@ github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ0 github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8= github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= +github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4 h1:7I5c2Ig/5FgqkYOh/N87NzoyI9U15qUPXhDD8uCupv8= +github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4/go.mod h1:278M4p8WsNh3n4a1eqiFcV2FGk7wE5fwUpUom9mK9lE= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/twpayne/go-vfs/v5 v5.0.4 h1:/ne3h+rW7f5YOyOFguz+3ztfUwzOLR0Vts3y0mMAitg= diff --git a/integration/integration_test.go b/integration/integration_test.go index d10fac1c..cb39988d 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -58,9 +58,10 @@ import ( ) const ( - testContainerLabel = "envbox-integration-test" - testImageAlpine = "localhost:5000/envbuilder-test-alpine:latest" - testImageUbuntu = "localhost:5000/envbuilder-test-ubuntu:latest" + testContainerLabel = "envbox-integration-test" + testImageAlpine = "localhost:5000/envbuilder-test-alpine:latest" + testImageUbuntu = "localhost:5000/envbuilder-test-ubuntu:latest" + testImageBlobUnknown = "localhost:5000/envbuilder-test-blob-unknown:latest" // nolint:gosec // Throw-away key for testing. DO NOT REUSE. testSSHKey = `-----BEGIN OPENSSH PRIVATE KEY----- @@ -2287,6 +2288,38 @@ USER devalot } require.Fail(t, "expected pid 1 to be running as devalot") }) + + t.Run("PushDuplicateLayersNoBlobUnknown", func(t *testing.T) { + t.Parallel() + + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: map[string]string{ + ".devcontainer/Dockerfile": fmt.Sprintf(`FROM %s +USER root +RUN echo "hi i r empty" +RUN echo "who u" +`, testImageBlobUnknown), + ".devcontainer/devcontainer.json": `{ + "name": "Test", + "build": { + "dockerfile": "Dockerfile" + }, + }`, + }, + }) + + // NOTE(mafredri): The in-memory registry doesn't catch this error so we + // have to use registry:2. + ref, err := name.ParseReference(fmt.Sprintf("localhost:5000/test-blob-unknown-%s", uuid.NewString())) + require.NoError(t, err) + opts := []string{ + envbuilderEnv("GIT_URL", srv.URL), + envbuilderEnv("CACHE_REPO", ref.String()), + envbuilderEnv("VERBOSE", "1"), + } + + _ = pushImage(t, ref, nil, opts...) + }) } func TestChownHomedir(t *testing.T) { @@ -2465,6 +2498,8 @@ func getCachedImage(ctx context.Context, t *testing.T, cli *client.Client, env . } func startContainerFromRef(ctx context.Context, t *testing.T, cli *client.Client, ref name.Reference) container.CreateResponse { + t.Helper() + // Ensure that we can pull the image. rc, err := cli.ImagePull(ctx, ref.String(), image.PullOptions{}) require.NoError(t, err) diff --git a/integration/testdata/blob-unknown/Dockerfile b/integration/testdata/blob-unknown/Dockerfile new file mode 100644 index 00000000..fffcc574 --- /dev/null +++ b/integration/testdata/blob-unknown/Dockerfile @@ -0,0 +1,7 @@ +FROM alpine:latest + +# This will produce an empty layer via Docker. It will allow us to test for a +# conflicting empty layer produced by Kaniko. This is to check against the +# BLOB_UNKNOWN error when trying to upload the built image to a registry and +# Kaniko having overwritten this blob with its own. +WORKDIR /home From 0b4600f7f4c308cdc6379ea7e9f7efd07dc37f13 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 2 Dec 2024 11:30:49 +0000 Subject: [PATCH 79/81] chore: bump github.com/golang-jwt/jwt/v4 from 4.5.0 to 4.5.1 (#405) (#426) (cherry picked from commit b1dc272) Co-authored-by: Mathias Fredriksson Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- envbuilder.go | 1 + go.mod | 2 +- go.sum | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 6f3231d1..f19f5e4b 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -581,6 +581,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro endStage("🏗️ Built image!") if opts.PushImage { endStage = startStage("🏗️ Pushing image...") + // To debug registry issues, enable logging: // // import ( diff --git a/go.mod b/go.mod index c51f4227..09c0bab4 100644 --- a/go.mod +++ b/go.mod @@ -143,7 +143,7 @@ require ( github.com/go-ole/go-ole v1.2.6 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.5.0 // indirect + github.com/golang-jwt/jwt/v4 v4.5.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.2 // indirect diff --git a/go.sum b/go.sum index 29fba2ef..429f4d8a 100644 --- a/go.sum +++ b/go.sum @@ -340,8 +340,9 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= +github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= From 15b5420c36e92f328dff676c4a2bf0746ea7bb32 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 2 Dec 2024 11:31:10 +0000 Subject: [PATCH 80/81] chore: improve logging around lifecycle scripts (#420) (#427) (cherry picked from commit fafad44) Co-authored-by: Mathias Fredriksson Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- envbuilder.go | 8 +++ integration/integration_test.go | 111 +++++++++++++++++++++++--------- 2 files changed, 89 insertions(+), 30 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index f19f5e4b..ea7c4436 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -169,6 +169,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro RemoteEnv: make(map[string]string), } if fileExists(opts.Filesystem, workingDir.Image()) { + opts.Logger(log.LevelInfo, "Found magic image file at %s", workingDir.Image()) if err = parseMagicImageFile(opts.Filesystem, workingDir.Image(), &runtimeData); err != nil { return fmt.Errorf("parse magic image file: %w", err) } @@ -287,6 +288,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro var buildParams *devcontainer.Compiled if opts.DockerfilePath == "" { + opts.Logger(log.LevelInfo, "No Dockerfile specified, looking for a devcontainer.json...") // Only look for a devcontainer if a Dockerfile wasn't specified. // devcontainer is a standard, so it's reasonable to be the default. var devcontainerDir string @@ -296,6 +298,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro opts.Logger(log.LevelError, "Failed to locate devcontainer.json: %s", err.Error()) opts.Logger(log.LevelError, "Falling back to the default image...") } else { + opts.Logger(log.LevelInfo, "Building in Devcontainer mode using %s", strings.TrimPrefix(runtimeData.DevcontainerPath, buildTimeWorkspaceFolder)) // We know a devcontainer exists. // Let's parse it and use it! file, err := opts.Filesystem.Open(runtimeData.DevcontainerPath) @@ -334,6 +337,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro } else { // If a Dockerfile was specified, we use that. dockerfilePath := filepath.Join(buildTimeWorkspaceFolder, opts.DockerfilePath) + opts.Logger(log.LevelInfo, "Building in Dockerfile-only mode using %s", opts.DockerfilePath) // If the dockerfilePath is specified and deeper than the base of WorkspaceFolder AND the BuildContextPath is // not defined, show a warning @@ -1402,6 +1406,7 @@ func execOneLifecycleScript( userInfo userInfo, ) error { if s.IsEmpty() { + logf(log.LevelInfo, "=== No %s script specified", scriptName) return nil } logf(log.LevelInfo, "=== Running %s as the %q user...", scriptName, userInfo.user.Username) @@ -1420,6 +1425,7 @@ func execLifecycleScripts( userInfo userInfo, ) error { if options.PostStartScriptPath != "" { + options.Logger(log.LevelDebug, "Removing postStartScriptPath %s", options.PostStartScriptPath) _ = os.Remove(options.PostStartScriptPath) } @@ -1428,6 +1434,8 @@ func execLifecycleScripts( // skip remaining lifecycle commands return nil } + } else { + options.Logger(log.LevelDebug, "Skipping onCreateCommand for subsequent starts...") } if err := execOneLifecycleScript(ctx, options.Logger, scripts.UpdateContentCommand, "updateContentCommand", userInfo); err != nil { // skip remaining lifecycle commands diff --git a/integration/integration_test.go b/integration/integration_test.go index cb39988d..e7fbc959 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -1125,37 +1125,88 @@ func TestUnsetOptionsEnv(t *testing.T) { func TestLifecycleScripts(t *testing.T) { t.Parallel() - // Ensures that a Git repository with a devcontainer.json is cloned and built. - srv := gittest.CreateGitServer(t, gittest.Options{ - Files: map[string]string{ - ".devcontainer/devcontainer.json": `{ - "name": "Test", - "build": { - "dockerfile": "Dockerfile" - }, - "onCreateCommand": "echo create > /tmp/out", - "updateContentCommand": ["sh", "-c", "echo update >> /tmp/out"], - "postCreateCommand": "(echo -n postCreate. ; id -un) >> /tmp/out", - "postStartCommand": { - "parallel1": "echo parallel1 > /tmp/parallel1", - "parallel2": ["sh", "-c", "echo parallel2 > /tmp/parallel2"] - } - }`, - ".devcontainer/Dockerfile": "FROM " + testImageAlpine + "\nUSER nobody", + for _, tt := range []struct { + name string + files map[string]string + outputCmd string + expectOutput string + }{ + { + name: "build", + files: map[string]string{ + ".devcontainer/devcontainer.json": `{ + "name": "Test", + "build": { + "dockerfile": "Dockerfile" + }, + "onCreateCommand": "echo create > /tmp/out", + "updateContentCommand": ["sh", "-c", "echo update >> /tmp/out"], + "postCreateCommand": "(echo -n postCreate. ; id -un) >> /tmp/out", + "postStartCommand": { + "parallel1": "echo parallel1 > /tmp/parallel1", + "parallel2": ["sh", "-c", "echo parallel2 > /tmp/parallel2"] + } + }`, + ".devcontainer/Dockerfile": "FROM " + testImageAlpine + "\nUSER nobody", + }, + outputCmd: "cat /tmp/out /tmp/parallel1 /tmp/parallel2", + expectOutput: "create\nupdate\npostCreate.nobody\nparallel1\nparallel2", }, - }) - ctr, err := runEnvbuilder(t, runOpts{env: []string{ - envbuilderEnv("GIT_URL", srv.URL), - }}) - require.NoError(t, err) - - output := execContainer(t, ctr, "cat /tmp/out /tmp/parallel1 /tmp/parallel2") - require.Equal(t, - `create -update -postCreate.nobody -parallel1 -parallel2`, strings.TrimSpace(output)) + { + name: "image", + files: map[string]string{ + ".devcontainer/devcontainer.json": fmt.Sprintf(`{ + "name": "Test", + "image": %q, + "containerUser": "nobody", + "onCreateCommand": "echo create > /tmp/out", + "updateContentCommand": ["sh", "-c", "echo update >> /tmp/out"], + "postCreateCommand": "(echo -n postCreate. ; id -un) >> /tmp/out", + "postStartCommand": { + "parallel1": "echo parallel1 > /tmp/parallel1", + "parallel2": ["sh", "-c", "echo parallel2 > /tmp/parallel2"] + } + }`, testImageAlpine), + }, + outputCmd: "cat /tmp/out /tmp/parallel1 /tmp/parallel2", + expectOutput: "create\nupdate\npostCreate.nobody\nparallel1\nparallel2", + }, + { + name: "label", + files: map[string]string{ + ".devcontainer/Dockerfile": fmt.Sprintf(`FROM %s + LABEL devcontainer.metadata='[{ \ + "onCreateCommand": "echo create > /tmp/out", \ + "updateContentCommand": ["sh", "-c", "echo update >> /tmp/out"], \ + "postCreateCommand": "(echo -n postCreate. ; id -un) >> /tmp/out", \ + "postStartCommand": { \ + "parallel1": "echo parallel1 > /tmp/parallel1", \ + "parallel2": ["sh", "-c", "echo parallel2 > /tmp/parallel2"] \ + } \ + }]' + USER nobody`, testImageAlpine), + }, + outputCmd: "cat /tmp/out /tmp/parallel1 /tmp/parallel2", + expectOutput: "create\nupdate\npostCreate.nobody\nparallel1\nparallel2", + }, + } { + tt := tt + t.Run(tt.name, func(t *testing.T) { + srv := gittest.CreateGitServer(t, gittest.Options{ + Files: tt.files, + }) + env := []string{ + envbuilderEnv("GIT_URL", srv.URL), + } + if _, ok := tt.files[".devcontainer/devcontainer.json"]; !ok { + env = append(env, envbuilderEnv("DOCKERFILE_PATH", ".devcontainer/Dockerfile")) + } + ctr, err := runEnvbuilder(t, runOpts{env: env}) + require.NoError(t, err, "failed to run envbuilder") + output := execContainer(t, ctr, tt.outputCmd) + require.Equal(t, tt.expectOutput, strings.TrimSpace(output)) + }) + } } func TestPostStartScript(t *testing.T) { From a96ba9294b75828161a7718cfa408849950763e9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 11:43:52 +0000 Subject: [PATCH 81/81] chore: bump golang.org/x/crypto from 0.26.0 to 0.29.0 (#417) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Cian Johnston (cherry picked from commit a3a6a837d369d2735445ab7c7c7d5f3193f4b44f) --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 09c0bab4..1283d599 100644 --- a/go.mod +++ b/go.mod @@ -39,9 +39,9 @@ require ( github.com/stretchr/testify v1.9.0 github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a go.uber.org/mock v0.4.0 - golang.org/x/crypto v0.26.0 + golang.org/x/crypto v0.29.0 golang.org/x/mod v0.21.0 - golang.org/x/sync v0.8.0 + golang.org/x/sync v0.9.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 ) @@ -274,9 +274,9 @@ require ( golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect golang.org/x/net v0.26.0 // indirect golang.org/x/oauth2 v0.20.0 // indirect - golang.org/x/sys v0.23.0 // indirect - golang.org/x/term v0.23.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/sys v0.27.0 // indirect + golang.org/x/term v0.26.0 // indirect + golang.org/x/text v0.20.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.22.0 // indirect golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect diff --git a/go.sum b/go.sum index 429f4d8a..96fa5e7c 100644 --- a/go.sum +++ b/go.sum @@ -836,8 +836,8 @@ golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= +golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= @@ -885,8 +885,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= +golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -925,15 +925,15 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.1-0.20230131160137-e7d7f63158de/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= +golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -943,8 +943,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= +golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=