From 96c20c4ec0d40456cb836407cdbf1723a91690b7 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 8 Apr 2025 15:22:10 +0000 Subject: [PATCH 01/30] chore: add sample README --- registry/coder/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 registry/coder/README.md diff --git a/registry/coder/README.md b/registry/coder/README.md new file mode 100644 index 00000000..05fbefb3 --- /dev/null +++ b/registry/coder/README.md @@ -0,0 +1,13 @@ +--- +display_name: Coder +bio: Coder provisions cloud development environments via Terraform, supporting Linux, macOS, Windows, X86, ARM, Kubernetes and more. +github: coder +linkedin: https://www.linkedin.com/company/coderhq +website: https://www.coder.com +support_email: support@coder.com +status: official +--- + +# Coder + +Coder provisions cloud development environments via Terraform, supporting Linux, macOS, Windows, X86, ARM, Kubernetes and more. From a5c4495e3e3c3c2ebebd0846c2d823fecd6fa4e2 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 8 Apr 2025 15:22:51 +0000 Subject: [PATCH 02/30] wip: commit progress on CI validation script --- .github/scripts/readme-validation/go.mod | 5 + .github/scripts/readme-validation/go.sum | 4 + .github/scripts/readme-validation/main.go | 531 ++++++++++++++++++++++ .github/workflows/readme-validation.yaml | 0 4 files changed, 540 insertions(+) create mode 100644 .github/scripts/readme-validation/go.mod create mode 100644 .github/scripts/readme-validation/go.sum create mode 100644 .github/scripts/readme-validation/main.go create mode 100644 .github/workflows/readme-validation.yaml diff --git a/.github/scripts/readme-validation/go.mod b/.github/scripts/readme-validation/go.mod new file mode 100644 index 00000000..32d1940a --- /dev/null +++ b/.github/scripts/readme-validation/go.mod @@ -0,0 +1,5 @@ +module coder.com/static_terraform_registry + +go 1.23.2 + +require sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/.github/scripts/readme-validation/go.sum b/.github/scripts/readme-validation/go.sum new file mode 100644 index 00000000..8c724249 --- /dev/null +++ b/.github/scripts/readme-validation/go.sum @@ -0,0 +1,4 @@ +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/.github/scripts/readme-validation/main.go b/.github/scripts/readme-validation/main.go new file mode 100644 index 00000000..775a32a6 --- /dev/null +++ b/.github/scripts/readme-validation/main.go @@ -0,0 +1,531 @@ +package main + +import ( + "bufio" + "errors" + "fmt" + "log" + "net/url" + "os" + "path" + "slices" + "strings" + "sync" + + "sigs.k8s.io/yaml" +) + +const rootRegistryPath = "./registry" + +type directoryReadme struct { + FilePath string + RawText string +} + +type rawContributorProfileFrontmatter struct { + DisplayName string `yaml:"display_name"` + Bio string `yaml:"bio"` + GithubUsername string `yaml:"github"` + AvatarUrl *string `yaml:"avatar"` + LinkedinURL *string `yaml:"linkedin"` + WebsiteURL *string `yaml:"website"` + SupportEmail *string `yaml:"support_email"` + CompanyGithub *string `yaml:"company_github"` + ContributorStatus *string `yaml:"status"` +} + +type trackableContributorFrontmatter struct { + rawContributorProfileFrontmatter + FilePath string +} + +type contributorProfileStatus int + +const ( + // Community should always be the first value defined via iota; it should be + // treated as the zero value of the type in the event that a more specific + // status wasn't defined + profileStatusCommunity contributorProfileStatus = iota + profileStatusPartner + profileStatusOfficial +) + +type contributorProfile struct { + EmployeeGithubUsernames []string + GithubUsername string + DisplayName string + Bio string + AvatarUrl string + WebsiteURL *string + LinkedinURL *string + SupportEmail *string + Status contributorProfileStatus +} + +var _ error = workflowPhaseError{} + +type workflowPhaseError struct { + Phase string + Errors []error +} + +func (wpe workflowPhaseError) Error() string { + msg := fmt.Sprintf("Error during phase %q of README validation:", wpe.Phase) + for _, e := range wpe.Errors { + msg += fmt.Sprintf("\n- %v", e) + } + msg += "\n" + + return msg +} + +func extractFrontmatter(readmeText string) (string, error) { + if readmeText == "" { + return "", errors.New("README is empty") + } + + const fence = "---" + fm := "" + fenceCount := 0 + lineScanner := bufio.NewScanner( + strings.NewReader(strings.TrimSpace(readmeText)), + ) + for lineScanner.Scan() { + nextLine := lineScanner.Text() + if fenceCount == 0 && nextLine != fence { + return "", errors.New("README does not start with frontmatter fence") + } + + if nextLine != fence { + fm += nextLine + continue + } + + fenceCount++ + if fenceCount >= 2 { + break + } + } + + if fenceCount == 1 { + return "", errors.New("README does not have two sets of frontmatter fences") + } + return fm, nil +} + +func validateContributorYaml(yml trackableContributorFrontmatter) []error { + // This function needs to aggregate a bunch of different errors, rather than + // stopping at the first one found, so using code blocks to section off + // logic for different fields + errors := []error{} + + // GitHub Username + { + if yml.GithubUsername == "" { + errors = append( + errors, + fmt.Errorf( + "missing GitHub username for %q", + yml.FilePath, + ), + ) + } + + lower := strings.ToLower(yml.GithubUsername) + if uriSafe := url.PathEscape(lower); uriSafe != lower { + errors = append( + errors, + fmt.Errorf( + "gitHub username %q (%q) is not a valid URL path segment", + yml.GithubUsername, + yml.FilePath, + ), + ) + } + } + + // Company GitHub + if yml.CompanyGithub != nil { + if *yml.CompanyGithub == "" { + errors = append( + errors, + fmt.Errorf( + "company_github field is defined but has empty value for %q", + yml.FilePath, + ), + ) + } + + lower := strings.ToLower(*yml.CompanyGithub) + if uriSafe := url.PathEscape(lower); uriSafe != lower { + errors = append( + errors, + fmt.Errorf( + "gitHub company username %q (%q) is not a valid URL path segment", + *yml.CompanyGithub, + yml.FilePath, + ), + ) + } + + if *yml.CompanyGithub == yml.GithubUsername { + errors = append( + errors, + fmt.Errorf( + "cannot list own GitHub name (%q) as employer (%q)", + yml.GithubUsername, + yml.FilePath, + ), + ) + } + } + + // Display name + { + if yml.DisplayName == "" { + errors = append( + errors, + fmt.Errorf( + "%q (%q) is missing display name", + yml.GithubUsername, + yml.FilePath, + ), + ) + } + } + + // LinkedIn URL + if yml.LinkedinURL != nil { + if _, err := url.ParseRequestURI(*yml.LinkedinURL); err != nil { + errors = append( + errors, + fmt.Errorf( + "linkedIn URL %q (%q) is not valid: %v", + *yml.LinkedinURL, + yml.FilePath, + err, + ), + ) + } + } + + // Email + if yml.SupportEmail != nil { + // Can't 100% validate that this is correct without actually sending + // an email, and especially with some contributors being individual + // developers, we don't want to do that on every single run of the CI + // pipeline. Best we can do is verify the general structure + username, server, ok := strings.Cut(*yml.SupportEmail, "@") + if !ok { + errors = append( + errors, + fmt.Errorf( + "email address %q (%q) is missing @ symbol", + *yml.LinkedinURL, + yml.FilePath, + ), + ) + goto website + } + + if username == "" { + errors = append( + errors, + fmt.Errorf( + "email address %q (%q) is missing username", + *yml.LinkedinURL, + yml.FilePath, + ), + ) + } + + domain, tld, ok := strings.Cut(server, ".") + if !ok { + errors = append( + errors, + fmt.Errorf( + "email address %q (%q) is missing period for server segment", + *yml.LinkedinURL, + yml.FilePath, + ), + ) + goto website + } + + if domain == "" { + errors = append( + errors, + fmt.Errorf( + "email address %q (%q) is missing domain", + *yml.LinkedinURL, + yml.FilePath, + ), + ) + } + + if tld == "" { + errors = append( + errors, + fmt.Errorf( + "email address %q (%q) is missing top-level domain", + *yml.LinkedinURL, + yml.FilePath, + ), + ) + } + } + + // Website +website: + if yml.WebsiteURL != nil { + if _, err := url.ParseRequestURI(*yml.WebsiteURL); err != nil { + errors = append( + errors, + fmt.Errorf( + "LinkedIn URL %q (%q) is not valid: %v", + *yml.WebsiteURL, + yml.FilePath, + err, + ), + ) + } + } + + // Contributor status + if yml.ContributorStatus != nil { + validStatuses := []string{"official", "partner", "community"} + if !slices.Contains(validStatuses, *yml.ContributorStatus) { + errors = append( + errors, + fmt.Errorf( + "contributor status %q (%q) is not valid", + *yml.ContributorStatus, + yml.FilePath, + ), + ) + } + } + + return errors +} + +func remapContributorProfile( + frontmatter trackableContributorFrontmatter, + employeeGitHubNames []string, +) contributorProfile { + // Function assumes that fields are previously validated and are safe to + // copy over verbatim when appropriate + remapped := contributorProfile{ + DisplayName: frontmatter.DisplayName, + GithubUsername: frontmatter.GithubUsername, + Bio: frontmatter.Bio, + LinkedinURL: frontmatter.LinkedinURL, + SupportEmail: frontmatter.SupportEmail, + } + + if frontmatter.AvatarUrl != nil { + remapped.AvatarUrl = *frontmatter.AvatarUrl + } + if frontmatter.ContributorStatus != nil { + switch *frontmatter.ContributorStatus { + case "partner": + remapped.Status = profileStatusPartner + case "official": + remapped.Status = profileStatusOfficial + default: + remapped.Status = profileStatusCommunity + } + } + if employeeGitHubNames != nil { + remapped.EmployeeGithubUsernames = employeeGitHubNames[:] + slices.Sort(remapped.EmployeeGithubUsernames) + } + + return remapped +} + +func parseContributorFiles(input []directoryReadme) ( + map[string]contributorProfile, + error, +) { + frontmatterByGithub := map[string]trackableContributorFrontmatter{} + yamlParsingErrors := workflowPhaseError{ + Phase: "YAML parsing", + } + for _, dirReadme := range input { + fmText, err := extractFrontmatter(dirReadme.RawText) + if err != nil { + yamlParsingErrors.Errors = append( + yamlParsingErrors.Errors, + fmt.Errorf("failed to parse %q: %v", dirReadme.FilePath, err), + ) + continue + } + + yml := rawContributorProfileFrontmatter{} + if err := yaml.Unmarshal([]byte(fmText), &yml); err != nil { + yamlParsingErrors.Errors = append( + yamlParsingErrors.Errors, + fmt.Errorf("failed to parse %q: %v", dirReadme.FilePath, err), + ) + continue + } + trackable := trackableContributorFrontmatter{ + FilePath: dirReadme.FilePath, + rawContributorProfileFrontmatter: yml, + } + + if prev, conflict := frontmatterByGithub[trackable.GithubUsername]; conflict { + yamlParsingErrors.Errors = append( + yamlParsingErrors.Errors, + fmt.Errorf( + "GitHub name conflict for %q for files %q and %q", + trackable.GithubUsername, + trackable.FilePath, + prev.FilePath, + ), + ) + continue + } + + frontmatterByGithub[trackable.GithubUsername] = trackable + } + + employeeGithubGroups := map[string][]string{} + yamlValidationErrors := workflowPhaseError{ + Phase: "Raw YAML Validation", + } + for _, yml := range frontmatterByGithub { + errors := validateContributorYaml(yml) + if len(errors) > 0 { + yamlValidationErrors.Errors = append( + yamlValidationErrors.Errors, + errors..., + ) + continue + } + + if yml.CompanyGithub != nil { + employeeGithubGroups[*yml.CompanyGithub] = append( + employeeGithubGroups[*yml.CompanyGithub], + yml.GithubUsername, + ) + } + } + if len(yamlValidationErrors.Errors) != 0 { + return nil, yamlValidationErrors + } + + contributorError := workflowPhaseError{ + Phase: "Contributor struct remapping", + } + structured := map[string]contributorProfile{} + for _, yml := range frontmatterByGithub { + group := employeeGithubGroups[yml.GithubUsername] + remapped := remapContributorProfile(yml, group) + structured[yml.GithubUsername] = remapped + } + for companyName, group := range employeeGithubGroups { + if _, found := structured[companyName]; found { + continue + } + contributorError.Errors = append( + contributorError.Errors, + fmt.Errorf( + "company %q does not exist in %q directory but is referenced by these profiles: [%s]", + rootRegistryPath, + companyName, + strings.Join(group, ", "), + ), + ) + } + if len(contributorError.Errors) != 0 { + return nil, contributorError + } + + return structured, nil +} + +func backfillAvatarUrls(contributors map[string]contributorProfile) error { + wg := sync.WaitGroup{} + requestBuffer := make(chan struct{}, 10) + errors := []error{} + + for _, c := range contributors { + wg.Add(1) + go func() { + requestBuffer <- struct{}{} + // Do request stuff + + <-requestBuffer + wg.Done() + }() + } + + wg.Wait() + if len(errors) == 0 { + return nil + } + + slices.SortFunc(errors, func(e1 error, e2 error) int { + return strings.Compare(e1.Error(), e2.Error()) + }) + return workflowPhaseError{ + Phase: "Avatar Backfill", + Errors: errors, + } +} + +func main() { + dirEntries, err := os.ReadDir(rootRegistryPath) + if err != nil { + log.Panic(err) + } + allReadmeFiles := []directoryReadme{} + fsErrors := workflowPhaseError{ + Phase: "FileSystem reading", + Errors: []error{}, + } + + for _, e := range dirEntries { + dirPath := path.Join(rootRegistryPath, e.Name()) + if !e.IsDir() { + fsErrors.Errors = append( + fsErrors.Errors, + fmt.Errorf( + "Detected non-directory file %q at base of main Registry directory", + dirPath, + ), + ) + continue + } + + readmePath := path.Join(dirPath, "README.md") + rmBytes, err := os.ReadFile(readmePath) + if err != nil { + fsErrors.Errors = append(fsErrors.Errors, err) + continue + } + allReadmeFiles = append(allReadmeFiles, directoryReadme{ + FilePath: readmePath, + RawText: string(rmBytes), + }) + } + if len(fsErrors.Errors) != 0 { + log.Panic(fsErrors) + } + + contributors, err := parseContributorFiles(allReadmeFiles) + if err != nil { + log.Panic(err) + } + err = backfillAvatarUrls(contributors) + if err != nil { + log.Panic(err) + } + + log.Printf( + "Processed all READMEs in the %q directory\n", + rootRegistryPath, + ) +} diff --git a/.github/workflows/readme-validation.yaml b/.github/workflows/readme-validation.yaml new file mode 100644 index 00000000..e69de29b From 20204b01538a5e4ca5c97628fc900416a626be67 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 8 Apr 2025 16:04:33 +0000 Subject: [PATCH 03/30] chore: add extra sample data file --- registry/nataindata/README.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 registry/nataindata/README.md diff --git a/registry/nataindata/README.md b/registry/nataindata/README.md new file mode 100644 index 00000000..5f291817 --- /dev/null +++ b/registry/nataindata/README.md @@ -0,0 +1,7 @@ +--- +display_name: Nataindata +bio: Data engineer +github: nataindata +website: https://www.nataindata.com +status: partner +--- From 902b32fd2950fc5ab991b4627db7c6b2776f8392 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 8 Apr 2025 16:07:07 +0000 Subject: [PATCH 04/30] wip: commit more progress on script --- .github/scripts/readme-validation/go.mod | 6 ++- .github/scripts/readme-validation/go.sum | 8 ++-- .github/scripts/readme-validation/main.go | 48 ++++++++++++++++++----- 3 files changed, 47 insertions(+), 15 deletions(-) diff --git a/.github/scripts/readme-validation/go.mod b/.github/scripts/readme-validation/go.mod index 32d1940a..a6fe657f 100644 --- a/.github/scripts/readme-validation/go.mod +++ b/.github/scripts/readme-validation/go.mod @@ -1,5 +1,7 @@ -module coder.com/static_terraform_registry +module coder.com/readme-validation go 1.23.2 -require sigs.k8s.io/yaml v1.4.0 // indirect +require github.com/ghodss/yaml v1.0.0 + +require gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/.github/scripts/readme-validation/go.sum b/.github/scripts/readme-validation/go.sum index 8c724249..0f65cfd9 100644 --- a/.github/scripts/readme-validation/go.sum +++ b/.github/scripts/readme-validation/go.sum @@ -1,4 +1,6 @@ -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= -sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/.github/scripts/readme-validation/main.go b/.github/scripts/readme-validation/main.go index 775a32a6..8c0ae833 100644 --- a/.github/scripts/readme-validation/main.go +++ b/.github/scripts/readme-validation/main.go @@ -12,7 +12,7 @@ import ( "strings" "sync" - "sigs.k8s.io/yaml" + "github.com/ghodss/yaml" ) const rootRegistryPath = "./registry" @@ -306,6 +306,11 @@ website: } } + // Avatar URL + if yml.AvatarUrl != nil { + + } + return errors } @@ -338,7 +343,12 @@ func remapContributorProfile( } if employeeGitHubNames != nil { remapped.EmployeeGithubUsernames = employeeGitHubNames[:] - slices.Sort(remapped.EmployeeGithubUsernames) + slices.SortFunc( + remapped.EmployeeGithubUsernames, + func(name1 string, name2 string) int { + return strings.Compare(name1, name2) + }, + ) } return remapped @@ -447,18 +457,36 @@ func parseContributorFiles(input []directoryReadme) ( } func backfillAvatarUrls(contributors map[string]contributorProfile) error { + if contributors == nil { + return errors.New("provided map is nil") + } + wg := sync.WaitGroup{} - requestBuffer := make(chan struct{}, 10) errors := []error{} + errorsMutex := sync.Mutex{} + + // Todo: Add actual fetching logic once everything else has been verified + requestAvatarUrl := func(string) (string, error) { + return "", nil + } + + for ghUsername, conCopy := range contributors { + if conCopy.AvatarUrl != "" { + continue + } - for _, c := range contributors { wg.Add(1) go func() { - requestBuffer <- struct{}{} - // Do request stuff - - <-requestBuffer - wg.Done() + defer wg.Done() + url, err := requestAvatarUrl(ghUsername) + if err != nil { + errorsMutex.Lock() + errors = append(errors, err) + errorsMutex.Unlock() + return + } + conCopy.AvatarUrl = url + contributors[ghUsername] = conCopy }() } @@ -481,12 +509,12 @@ func main() { if err != nil { log.Panic(err) } + allReadmeFiles := []directoryReadme{} fsErrors := workflowPhaseError{ Phase: "FileSystem reading", Errors: []error{}, } - for _, e := range dirEntries { dirPath := path.Join(rootRegistryPath, e.Name()) if !e.IsDir() { From 1906520b4a216f36c225a52e7113b523448e540e Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 8 Apr 2025 16:31:25 +0000 Subject: [PATCH 05/30] chore: add logs for better feedback --- .github/scripts/readme-validation/main.go | 59 +++++++++++++++++------ 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/.github/scripts/readme-validation/main.go b/.github/scripts/readme-validation/main.go index 8c0ae833..49b7b505 100644 --- a/.github/scripts/readme-validation/main.go +++ b/.github/scripts/readme-validation/main.go @@ -15,7 +15,7 @@ import ( "github.com/ghodss/yaml" ) -const rootRegistryPath = "./registry" +const rootRegistryPath = "../../../registry" type directoryReadme struct { FilePath string @@ -70,7 +70,7 @@ type workflowPhaseError struct { } func (wpe workflowPhaseError) Error() string { - msg := fmt.Sprintf("Error during phase %q of README validation:", wpe.Phase) + msg := fmt.Sprintf("Error during %q phase of README validation:", wpe.Phase) for _, e := range wpe.Errors { msg += fmt.Sprintf("\n- %v", e) } @@ -456,59 +456,75 @@ func parseContributorFiles(input []directoryReadme) ( return structured, nil } -func backfillAvatarUrls(contributors map[string]contributorProfile) error { +// backfillAvatarUrls takes a map of contributor information, each keyed by +// GitHub username, and tries to mutate each entry to fill in its missing avatar +// URL. The first integer indicates the number of avatars that needed to be +// backfilled, while the second indicates the number that could be backfilled +// without any errors. +// +// The function will collect all request errors, rather than return the first +// one found. +func backfillAvatarUrls(contributors map[string]contributorProfile) (int, int, error) { if contributors == nil { - return errors.New("provided map is nil") + return 0, 0, errors.New("provided map is nil") } wg := sync.WaitGroup{} + mtx := sync.Mutex{} errors := []error{} - errorsMutex := sync.Mutex{} + successfulBackfills := 0 // Todo: Add actual fetching logic once everything else has been verified requestAvatarUrl := func(string) (string, error) { return "", nil } - for ghUsername, conCopy := range contributors { - if conCopy.AvatarUrl != "" { + avatarsThatNeedBackfill := 0 + for ghUsername, con := range contributors { + if con.AvatarUrl != "" { continue } + avatarsThatNeedBackfill++ wg.Add(1) go func() { defer wg.Done() url, err := requestAvatarUrl(ghUsername) + mtx.Lock() + defer mtx.Unlock() + if err != nil { - errorsMutex.Lock() errors = append(errors, err) - errorsMutex.Unlock() return } - conCopy.AvatarUrl = url - contributors[ghUsername] = conCopy + + successfulBackfills++ + con.AvatarUrl = url + contributors[ghUsername] = con }() } wg.Wait() if len(errors) == 0 { - return nil + return avatarsThatNeedBackfill, successfulBackfills, nil } slices.SortFunc(errors, func(e1 error, e2 error) int { return strings.Compare(e1.Error(), e2.Error()) }) - return workflowPhaseError{ + return avatarsThatNeedBackfill, successfulBackfills, workflowPhaseError{ Phase: "Avatar Backfill", Errors: errors, } } func main() { + log.Println("Starting README validation") dirEntries, err := os.ReadDir(rootRegistryPath) if err != nil { log.Panic(err) } + log.Printf("Identified %d top-level directory entries\n", len(dirEntries)) allReadmeFiles := []directoryReadme{} fsErrors := workflowPhaseError{ @@ -543,14 +559,29 @@ func main() { log.Panic(fsErrors) } + log.Printf("Processing %d README files\n", len(allReadmeFiles)) contributors, err := parseContributorFiles(allReadmeFiles) if err != nil { log.Panic(err) } - err = backfillAvatarUrls(contributors) + log.Printf( + "Processed %d README files as valid contributor profiles", + len(contributors), + ) + + backfillsNeeded, successCount, err := backfillAvatarUrls(contributors) if err != nil { log.Panic(err) } + if backfillsNeeded == 0 { + log.Println("No GitHub avatar backfills needed") + } else { + log.Printf( + "Backfilled %d/%d missing GitHub avatars", + backfillsNeeded, + successCount, + ) + } log.Printf( "Processed all READMEs in the %q directory\n", From da735dafd66bdcbd2a4171a4eb717ce7ca460427 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 8 Apr 2025 17:12:28 +0000 Subject: [PATCH 06/30] fix: remove parsing bugs --- .github/scripts/readme-validation/go.mod | 5 +- .github/scripts/readme-validation/go.sum | 2 + .github/scripts/readme-validation/main.go | 68 ++++++++++++----------- 3 files changed, 42 insertions(+), 33 deletions(-) diff --git a/.github/scripts/readme-validation/go.mod b/.github/scripts/readme-validation/go.mod index a6fe657f..3eae387a 100644 --- a/.github/scripts/readme-validation/go.mod +++ b/.github/scripts/readme-validation/go.mod @@ -4,4 +4,7 @@ go 1.23.2 require github.com/ghodss/yaml v1.0.0 -require gopkg.in/yaml.v2 v2.4.0 // indirect +require ( + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/.github/scripts/readme-validation/go.sum b/.github/scripts/readme-validation/go.sum index 0f65cfd9..aa535201 100644 --- a/.github/scripts/readme-validation/go.sum +++ b/.github/scripts/readme-validation/go.sum @@ -4,3 +4,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/.github/scripts/readme-validation/main.go b/.github/scripts/readme-validation/main.go index 49b7b505..7ee22c31 100644 --- a/.github/scripts/readme-validation/main.go +++ b/.github/scripts/readme-validation/main.go @@ -12,7 +12,7 @@ import ( "strings" "sync" - "github.com/ghodss/yaml" + "gopkg.in/yaml.v3" ) const rootRegistryPath = "../../../registry" @@ -23,18 +23,18 @@ type directoryReadme struct { } type rawContributorProfileFrontmatter struct { - DisplayName string `yaml:"display_name"` - Bio string `yaml:"bio"` - GithubUsername string `yaml:"github"` - AvatarUrl *string `yaml:"avatar"` - LinkedinURL *string `yaml:"linkedin"` - WebsiteURL *string `yaml:"website"` - SupportEmail *string `yaml:"support_email"` - CompanyGithub *string `yaml:"company_github"` - ContributorStatus *string `yaml:"status"` + DisplayName string `yaml:"display_name"` + Bio string `yaml:"bio"` + GithubUsername string `yaml:"github"` + AvatarUrl *string `yaml:"avatar"` + LinkedinURL *string `yaml:"linkedin"` + WebsiteURL *string `yaml:"website"` + SupportEmail *string `yaml:"support_email"` + EmployerGithubUsername *string `yaml:"employer_github"` + ContributorStatus *string `yaml:"status"` } -type trackableContributorFrontmatter struct { +type contributorFrontmatterWithFilepath struct { rawContributorProfileFrontmatter FilePath string } @@ -97,7 +97,7 @@ func extractFrontmatter(readmeText string) (string, error) { } if nextLine != fence { - fm += nextLine + fm += nextLine + "\n" continue } @@ -113,7 +113,7 @@ func extractFrontmatter(readmeText string) (string, error) { return fm, nil } -func validateContributorYaml(yml trackableContributorFrontmatter) []error { +func validateContributorYaml(yml contributorFrontmatterWithFilepath) []error { // This function needs to aggregate a bunch of different errors, rather than // stopping at the first one found, so using code blocks to section off // logic for different fields @@ -145,8 +145,8 @@ func validateContributorYaml(yml trackableContributorFrontmatter) []error { } // Company GitHub - if yml.CompanyGithub != nil { - if *yml.CompanyGithub == "" { + if yml.EmployerGithubUsername != nil { + if *yml.EmployerGithubUsername == "" { errors = append( errors, fmt.Errorf( @@ -156,19 +156,19 @@ func validateContributorYaml(yml trackableContributorFrontmatter) []error { ) } - lower := strings.ToLower(*yml.CompanyGithub) + lower := strings.ToLower(*yml.EmployerGithubUsername) if uriSafe := url.PathEscape(lower); uriSafe != lower { errors = append( errors, fmt.Errorf( "gitHub company username %q (%q) is not a valid URL path segment", - *yml.CompanyGithub, + *yml.EmployerGithubUsername, yml.FilePath, ), ) } - if *yml.CompanyGithub == yml.GithubUsername { + if *yml.EmployerGithubUsername == yml.GithubUsername { errors = append( errors, fmt.Errorf( @@ -315,7 +315,7 @@ website: } func remapContributorProfile( - frontmatter trackableContributorFrontmatter, + frontmatter contributorFrontmatterWithFilepath, employeeGitHubNames []string, ) contributorProfile { // Function assumes that fields are previously validated and are safe to @@ -354,20 +354,20 @@ func remapContributorProfile( return remapped } -func parseContributorFiles(input []directoryReadme) ( +func parseContributorFiles(readmeEntries []directoryReadme) ( map[string]contributorProfile, error, ) { - frontmatterByGithub := map[string]trackableContributorFrontmatter{} + frontmatterByGithub := map[string]contributorFrontmatterWithFilepath{} yamlParsingErrors := workflowPhaseError{ Phase: "YAML parsing", } - for _, dirReadme := range input { - fmText, err := extractFrontmatter(dirReadme.RawText) + for _, rm := range readmeEntries { + fmText, err := extractFrontmatter(rm.RawText) if err != nil { yamlParsingErrors.Errors = append( yamlParsingErrors.Errors, - fmt.Errorf("failed to parse %q: %v", dirReadme.FilePath, err), + fmt.Errorf("failed to parse %q: %v", rm.FilePath, err), ) continue } @@ -376,12 +376,13 @@ func parseContributorFiles(input []directoryReadme) ( if err := yaml.Unmarshal([]byte(fmText), &yml); err != nil { yamlParsingErrors.Errors = append( yamlParsingErrors.Errors, - fmt.Errorf("failed to parse %q: %v", dirReadme.FilePath, err), + fmt.Errorf("failed to parse %q: %v", rm.FilePath, err), ) continue } - trackable := trackableContributorFrontmatter{ - FilePath: dirReadme.FilePath, + + trackable := contributorFrontmatterWithFilepath{ + FilePath: rm.FilePath, rawContributorProfileFrontmatter: yml, } @@ -391,8 +392,8 @@ func parseContributorFiles(input []directoryReadme) ( fmt.Errorf( "GitHub name conflict for %q for files %q and %q", trackable.GithubUsername, - trackable.FilePath, prev.FilePath, + trackable.FilePath, ), ) continue @@ -400,6 +401,9 @@ func parseContributorFiles(input []directoryReadme) ( frontmatterByGithub[trackable.GithubUsername] = trackable } + if len(yamlParsingErrors.Errors) != 0 { + return nil, yamlParsingErrors + } employeeGithubGroups := map[string][]string{} yamlValidationErrors := workflowPhaseError{ @@ -415,9 +419,9 @@ func parseContributorFiles(input []directoryReadme) ( continue } - if yml.CompanyGithub != nil { - employeeGithubGroups[*yml.CompanyGithub] = append( - employeeGithubGroups[*yml.CompanyGithub], + if yml.EmployerGithubUsername != nil { + employeeGithubGroups[*yml.EmployerGithubUsername] = append( + employeeGithubGroups[*yml.EmployerGithubUsername], yml.GithubUsername, ) } @@ -443,8 +447,8 @@ func parseContributorFiles(input []directoryReadme) ( contributorError.Errors, fmt.Errorf( "company %q does not exist in %q directory but is referenced by these profiles: [%s]", - rootRegistryPath, companyName, + rootRegistryPath, strings.Join(group, ", "), ), ) From b19132140a45c4707dfed9c622690cf1713cd1d8 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 8 Apr 2025 17:29:34 +0000 Subject: [PATCH 07/30] fix: make logging better --- .github/scripts/readme-validation/main.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/scripts/readme-validation/main.go b/.github/scripts/readme-validation/main.go index 7ee22c31..c942be6b 100644 --- a/.github/scripts/readme-validation/main.go +++ b/.github/scripts/readme-validation/main.go @@ -50,6 +50,17 @@ const ( profileStatusOfficial ) +func (status contributorProfileStatus) String() string { + switch status { + case profileStatusOfficial: + return "official" + case profileStatusPartner: + return "partner" + default: + return "community" + } +} + type contributorProfile struct { EmployeeGithubUsernames []string GithubUsername string @@ -186,7 +197,7 @@ func validateContributorYaml(yml contributorFrontmatterWithFilepath) []error { errors = append( errors, fmt.Errorf( - "%q (%q) is missing display name", + "GitHub user %q (%q) is missing display name", yml.GithubUsername, yml.FilePath, ), @@ -326,6 +337,7 @@ func remapContributorProfile( Bio: frontmatter.Bio, LinkedinURL: frontmatter.LinkedinURL, SupportEmail: frontmatter.SupportEmail, + WebsiteURL: frontmatter.WebsiteURL, } if frontmatter.AvatarUrl != nil { @@ -503,7 +515,7 @@ func backfillAvatarUrls(contributors map[string]contributorProfile) (int, int, e } successfulBackfills++ - con.AvatarUrl = url + con.AvatarUrl = url + "Not implemented yet" contributors[ghUsername] = con }() } From 629f4b3a651fc8a03b3cdb99ac0f0e17bc54e98a Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 8 Apr 2025 18:53:44 +0000 Subject: [PATCH 08/30] chore: rename directory for script --- .../go.mod | 0 .../go.sum | 0 .../main.go | 88 ++----------------- 3 files changed, 5 insertions(+), 83 deletions(-) rename .github/scripts/{readme-validation => validate-contributor-readmes}/go.mod (100%) rename .github/scripts/{readme-validation => validate-contributor-readmes}/go.sum (100%) rename .github/scripts/{readme-validation => validate-contributor-readmes}/main.go (84%) diff --git a/.github/scripts/readme-validation/go.mod b/.github/scripts/validate-contributor-readmes/go.mod similarity index 100% rename from .github/scripts/readme-validation/go.mod rename to .github/scripts/validate-contributor-readmes/go.mod diff --git a/.github/scripts/readme-validation/go.sum b/.github/scripts/validate-contributor-readmes/go.sum similarity index 100% rename from .github/scripts/readme-validation/go.sum rename to .github/scripts/validate-contributor-readmes/go.sum diff --git a/.github/scripts/readme-validation/main.go b/.github/scripts/validate-contributor-readmes/main.go similarity index 84% rename from .github/scripts/readme-validation/main.go rename to .github/scripts/validate-contributor-readmes/main.go index c942be6b..59d73259 100644 --- a/.github/scripts/readme-validation/main.go +++ b/.github/scripts/validate-contributor-readmes/main.go @@ -10,7 +10,6 @@ import ( "path" "slices" "strings" - "sync" "gopkg.in/yaml.v3" ) @@ -66,7 +65,7 @@ type contributorProfile struct { GithubUsername string DisplayName string Bio string - AvatarUrl string + AvatarUrl *string WebsiteURL *string LinkedinURL *string SupportEmail *string @@ -329,8 +328,9 @@ func remapContributorProfile( frontmatter contributorFrontmatterWithFilepath, employeeGitHubNames []string, ) contributorProfile { - // Function assumes that fields are previously validated and are safe to - // copy over verbatim when appropriate + // Function assumes that (1) fields are previously validated and are safe to + // copy over verbatim when appropriate, and (2) any missing avatar URLs will + // be backfilled during the main Registry site build step remapped := contributorProfile{ DisplayName: frontmatter.DisplayName, GithubUsername: frontmatter.GithubUsername, @@ -338,11 +338,9 @@ func remapContributorProfile( LinkedinURL: frontmatter.LinkedinURL, SupportEmail: frontmatter.SupportEmail, WebsiteURL: frontmatter.WebsiteURL, + AvatarUrl: frontmatter.AvatarUrl, } - if frontmatter.AvatarUrl != nil { - remapped.AvatarUrl = *frontmatter.AvatarUrl - } if frontmatter.ContributorStatus != nil { switch *frontmatter.ContributorStatus { case "partner": @@ -472,68 +470,6 @@ func parseContributorFiles(readmeEntries []directoryReadme) ( return structured, nil } -// backfillAvatarUrls takes a map of contributor information, each keyed by -// GitHub username, and tries to mutate each entry to fill in its missing avatar -// URL. The first integer indicates the number of avatars that needed to be -// backfilled, while the second indicates the number that could be backfilled -// without any errors. -// -// The function will collect all request errors, rather than return the first -// one found. -func backfillAvatarUrls(contributors map[string]contributorProfile) (int, int, error) { - if contributors == nil { - return 0, 0, errors.New("provided map is nil") - } - - wg := sync.WaitGroup{} - mtx := sync.Mutex{} - errors := []error{} - successfulBackfills := 0 - - // Todo: Add actual fetching logic once everything else has been verified - requestAvatarUrl := func(string) (string, error) { - return "", nil - } - - avatarsThatNeedBackfill := 0 - for ghUsername, con := range contributors { - if con.AvatarUrl != "" { - continue - } - - avatarsThatNeedBackfill++ - wg.Add(1) - go func() { - defer wg.Done() - url, err := requestAvatarUrl(ghUsername) - mtx.Lock() - defer mtx.Unlock() - - if err != nil { - errors = append(errors, err) - return - } - - successfulBackfills++ - con.AvatarUrl = url + "Not implemented yet" - contributors[ghUsername] = con - }() - } - - wg.Wait() - if len(errors) == 0 { - return avatarsThatNeedBackfill, successfulBackfills, nil - } - - slices.SortFunc(errors, func(e1 error, e2 error) int { - return strings.Compare(e1.Error(), e2.Error()) - }) - return avatarsThatNeedBackfill, successfulBackfills, workflowPhaseError{ - Phase: "Avatar Backfill", - Errors: errors, - } -} - func main() { log.Println("Starting README validation") dirEntries, err := os.ReadDir(rootRegistryPath) @@ -585,20 +521,6 @@ func main() { len(contributors), ) - backfillsNeeded, successCount, err := backfillAvatarUrls(contributors) - if err != nil { - log.Panic(err) - } - if backfillsNeeded == 0 { - log.Println("No GitHub avatar backfills needed") - } else { - log.Printf( - "Backfilled %d/%d missing GitHub avatars", - backfillsNeeded, - successCount, - ) - } - log.Printf( "Processed all READMEs in the %q directory\n", rootRegistryPath, From 2b9da92259bb7a2ad214b6c36e3417092bc3bc0d Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 8 Apr 2025 19:28:33 +0000 Subject: [PATCH 09/30] refactor: remove goto statements --- .../go.mod | 0 .../go.sum | 0 .../main.go | 177 +++++++++++++----- 3 files changed, 125 insertions(+), 52 deletions(-) rename .github/scripts/{validate-contributor-readmes => validate-profile-readmes}/go.mod (100%) rename .github/scripts/{validate-contributor-readmes => validate-profile-readmes}/go.sum (100%) rename .github/scripts/{validate-contributor-readmes => validate-profile-readmes}/main.go (81%) diff --git a/.github/scripts/validate-contributor-readmes/go.mod b/.github/scripts/validate-profile-readmes/go.mod similarity index 100% rename from .github/scripts/validate-contributor-readmes/go.mod rename to .github/scripts/validate-profile-readmes/go.mod diff --git a/.github/scripts/validate-contributor-readmes/go.sum b/.github/scripts/validate-profile-readmes/go.sum similarity index 100% rename from .github/scripts/validate-contributor-readmes/go.sum rename to .github/scripts/validate-profile-readmes/go.sum diff --git a/.github/scripts/validate-contributor-readmes/main.go b/.github/scripts/validate-profile-readmes/main.go similarity index 81% rename from .github/scripts/validate-contributor-readmes/main.go rename to .github/scripts/validate-profile-readmes/main.go index 59d73259..6143fb80 100644 --- a/.github/scripts/validate-contributor-readmes/main.go +++ b/.github/scripts/validate-profile-readmes/main.go @@ -124,27 +124,33 @@ func extractFrontmatter(readmeText string) (string, error) { } func validateContributorYaml(yml contributorFrontmatterWithFilepath) []error { - // This function needs to aggregate a bunch of different errors, rather than - // stopping at the first one found, so using code blocks to section off + // This function needs to aggregate a bunch of different problems, rather + // than stopping at the first one found, so using code blocks to section off // logic for different fields - errors := []error{} + problems := []error{} + + // Using a bunch of closures to group validations for each field and add + // support for ending validations for a group early. The alternatives were + // making a bunch of functions in the top-level that would only be used + // once, or using goto statements, which would've made refactoring fragile // GitHub Username - { + func() { if yml.GithubUsername == "" { - errors = append( - errors, + problems = append( + problems, fmt.Errorf( "missing GitHub username for %q", yml.FilePath, ), ) + return } lower := strings.ToLower(yml.GithubUsername) if uriSafe := url.PathEscape(lower); uriSafe != lower { - errors = append( - errors, + problems = append( + problems, fmt.Errorf( "gitHub username %q (%q) is not a valid URL path segment", yml.GithubUsername, @@ -152,24 +158,29 @@ func validateContributorYaml(yml contributorFrontmatterWithFilepath) []error { ), ) } - } + }() // Company GitHub - if yml.EmployerGithubUsername != nil { + func() { + if yml.EmployerGithubUsername == nil { + return + } + if *yml.EmployerGithubUsername == "" { - errors = append( - errors, + problems = append( + problems, fmt.Errorf( "company_github field is defined but has empty value for %q", yml.FilePath, ), ) + return } lower := strings.ToLower(*yml.EmployerGithubUsername) if uriSafe := url.PathEscape(lower); uriSafe != lower { - errors = append( - errors, + problems = append( + problems, fmt.Errorf( "gitHub company username %q (%q) is not a valid URL path segment", *yml.EmployerGithubUsername, @@ -179,8 +190,8 @@ func validateContributorYaml(yml contributorFrontmatterWithFilepath) []error { } if *yml.EmployerGithubUsername == yml.GithubUsername { - errors = append( - errors, + problems = append( + problems, fmt.Errorf( "cannot list own GitHub name (%q) as employer (%q)", yml.GithubUsername, @@ -188,13 +199,13 @@ func validateContributorYaml(yml contributorFrontmatterWithFilepath) []error { ), ) } - } + }() // Display name - { + func() { if yml.DisplayName == "" { - errors = append( - errors, + problems = append( + problems, fmt.Errorf( "GitHub user %q (%q) is missing display name", yml.GithubUsername, @@ -202,13 +213,17 @@ func validateContributorYaml(yml contributorFrontmatterWithFilepath) []error { ), ) } - } + }() // LinkedIn URL - if yml.LinkedinURL != nil { + func() { + if yml.LinkedinURL == nil { + return + } + if _, err := url.ParseRequestURI(*yml.LinkedinURL); err != nil { - errors = append( - errors, + problems = append( + problems, fmt.Errorf( "linkedIn URL %q (%q) is not valid: %v", *yml.LinkedinURL, @@ -217,30 +232,34 @@ func validateContributorYaml(yml contributorFrontmatterWithFilepath) []error { ), ) } - } + }() // Email - if yml.SupportEmail != nil { + func() { + if yml.SupportEmail == nil { + return + } + // Can't 100% validate that this is correct without actually sending // an email, and especially with some contributors being individual // developers, we don't want to do that on every single run of the CI // pipeline. Best we can do is verify the general structure username, server, ok := strings.Cut(*yml.SupportEmail, "@") if !ok { - errors = append( - errors, + problems = append( + problems, fmt.Errorf( "email address %q (%q) is missing @ symbol", *yml.LinkedinURL, yml.FilePath, ), ) - goto website + return } if username == "" { - errors = append( - errors, + problems = append( + problems, fmt.Errorf( "email address %q (%q) is missing username", *yml.LinkedinURL, @@ -251,20 +270,20 @@ func validateContributorYaml(yml contributorFrontmatterWithFilepath) []error { domain, tld, ok := strings.Cut(server, ".") if !ok { - errors = append( - errors, + problems = append( + problems, fmt.Errorf( "email address %q (%q) is missing period for server segment", *yml.LinkedinURL, yml.FilePath, ), ) - goto website + return } if domain == "" { - errors = append( - errors, + problems = append( + problems, fmt.Errorf( "email address %q (%q) is missing domain", *yml.LinkedinURL, @@ -274,8 +293,8 @@ func validateContributorYaml(yml contributorFrontmatterWithFilepath) []error { } if tld == "" { - errors = append( - errors, + problems = append( + problems, fmt.Errorf( "email address %q (%q) is missing top-level domain", *yml.LinkedinURL, @@ -283,14 +302,27 @@ func validateContributorYaml(yml contributorFrontmatterWithFilepath) []error { ), ) } - } + + if strings.Contains(*yml.AvatarUrl, "?") { + problems = append( + problems, + fmt.Errorf( + "email for %q is not allowed to contain search parameters", + yml.FilePath, + ), + ) + } + }() // Website -website: - if yml.WebsiteURL != nil { + func() { + if yml.WebsiteURL == nil { + return + } + if _, err := url.ParseRequestURI(*yml.WebsiteURL); err != nil { - errors = append( - errors, + problems = append( + problems, fmt.Errorf( "LinkedIn URL %q (%q) is not valid: %v", *yml.WebsiteURL, @@ -299,14 +331,18 @@ website: ), ) } - } + }() // Contributor status - if yml.ContributorStatus != nil { + func() { + if yml.ContributorStatus == nil { + return + } + validStatuses := []string{"official", "partner", "community"} if !slices.Contains(validStatuses, *yml.ContributorStatus) { - errors = append( - errors, + problems = append( + problems, fmt.Errorf( "contributor status %q (%q) is not valid", *yml.ContributorStatus, @@ -314,14 +350,51 @@ website: ), ) } - } + }() - // Avatar URL - if yml.AvatarUrl != nil { + // Avatar URL - can't validate the image actually leads to a valid resource + // in a pure function, but can at least catch obvious problems + func() { + if yml.AvatarUrl == nil { + return + } - } + if *yml.AvatarUrl == "" { + problems = append( + problems, + fmt.Errorf( + "avatar URL for %q must be omitted or non-empty string", + yml.FilePath, + ), + ) + return + } + + // Have to use .Parse instead of .ParseRequestURI because this is the + // one field that's allowed to be a relative URL + if _, err := url.Parse(*yml.AvatarUrl); err != nil { + problems = append( + problems, + fmt.Errorf( + "error %q (%q) is not a valid relative or absolute URL", + *yml.AvatarUrl, + yml.FilePath, + ), + ) + } + + if strings.Contains(*yml.AvatarUrl, "?") { + problems = append( + problems, + fmt.Errorf( + "avatar URL for %q is not allowed to contain search parameters", + yml.FilePath, + ), + ) + } + }() - return errors + return problems } func remapContributorProfile( From 23f1cee6408281484539b33e432965c88ad00c1e Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 8 Apr 2025 19:32:49 +0000 Subject: [PATCH 10/30] fix: remove accidental segfault --- .github/scripts/validate-profile-readmes/main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/scripts/validate-profile-readmes/main.go b/.github/scripts/validate-profile-readmes/main.go index 6143fb80..5bd409f8 100644 --- a/.github/scripts/validate-profile-readmes/main.go +++ b/.github/scripts/validate-profile-readmes/main.go @@ -213,6 +213,7 @@ func validateContributorYaml(yml contributorFrontmatterWithFilepath) []error { ), ) } + }() // LinkedIn URL @@ -303,7 +304,7 @@ func validateContributorYaml(yml contributorFrontmatterWithFilepath) []error { ) } - if strings.Contains(*yml.AvatarUrl, "?") { + if strings.Contains(*yml.SupportEmail, "?") { problems = append( problems, fmt.Errorf( From e20e679e08b0165a511f8f30971b35e74f8a5569 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 8 Apr 2025 20:02:52 +0000 Subject: [PATCH 11/30] wip: scaffold relative URL validation --- .../scripts/validate-profile-readmes/main.go | 72 +++++++++++++++---- .github/workflows/readme-validation.yaml | 0 2 files changed, 59 insertions(+), 13 deletions(-) delete mode 100644 .github/workflows/readme-validation.yaml diff --git a/.github/scripts/validate-profile-readmes/main.go b/.github/scripts/validate-profile-readmes/main.go index 5bd409f8..abc3b010 100644 --- a/.github/scripts/validate-profile-readmes/main.go +++ b/.github/scripts/validate-profile-readmes/main.go @@ -63,6 +63,7 @@ func (status contributorProfileStatus) String() string { type contributorProfile struct { EmployeeGithubUsernames []string GithubUsername string + FilePath string DisplayName string Bio string AvatarUrl *string @@ -406,6 +407,7 @@ func remapContributorProfile( // copy over verbatim when appropriate, and (2) any missing avatar URLs will // be backfilled during the main Registry site build step remapped := contributorProfile{ + FilePath: frontmatter.FilePath, DisplayName: frontmatter.DisplayName, GithubUsername: frontmatter.GithubUsername, Bio: frontmatter.Bio, @@ -544,24 +546,19 @@ func parseContributorFiles(readmeEntries []directoryReadme) ( return structured, nil } -func main() { - log.Println("Starting README validation") +func aggregateReadmeFiles() ([]directoryReadme, error) { dirEntries, err := os.ReadDir(rootRegistryPath) if err != nil { - log.Panic(err) + return nil, err } - log.Printf("Identified %d top-level directory entries\n", len(dirEntries)) allReadmeFiles := []directoryReadme{} - fsErrors := workflowPhaseError{ - Phase: "FileSystem reading", - Errors: []error{}, - } + problems := []error{} for _, e := range dirEntries { dirPath := path.Join(rootRegistryPath, e.Name()) if !e.IsDir() { - fsErrors.Errors = append( - fsErrors.Errors, + problems = append( + problems, fmt.Errorf( "Detected non-directory file %q at base of main Registry directory", dirPath, @@ -573,7 +570,7 @@ func main() { readmePath := path.Join(dirPath, "README.md") rmBytes, err := os.ReadFile(readmePath) if err != nil { - fsErrors.Errors = append(fsErrors.Errors, err) + problems = append(problems, err) continue } allReadmeFiles = append(allReadmeFiles, directoryReadme{ @@ -581,8 +578,51 @@ func main() { RawText: string(rmBytes), }) } - if len(fsErrors.Errors) != 0 { - log.Panic(fsErrors) + + if len(problems) != 0 { + return nil, workflowPhaseError{ + Phase: "FileSystem reading", + Errors: problems, + } + } + + return allReadmeFiles, nil +} + +func validateRelativeUrls( + contributors map[string]contributorProfile, +) error { + // This function only validates relative avatar URLs for now, but it can be + // beefed up to validate more in the future + problems := []error{} + + for _, con := range contributors { + if con.AvatarUrl == nil { + continue + } + isRelativeUrl := strings.HasPrefix(*con.AvatarUrl, ".") || + strings.HasPrefix(*con.AvatarUrl, "/") + if !isRelativeUrl { + continue + } + + fmt.Println(con.GithubUsername, con.AvatarUrl) + } + + if len(problems) == 0 { + return nil + } + return workflowPhaseError{ + Phase: "Relative URL validation", + Errors: problems, + } +} + +func main() { + log.Println("Starting README validation") + allReadmeFiles, err := aggregateReadmeFiles() + if err != nil { + panic(err) } log.Printf("Processing %d README files\n", len(allReadmeFiles)) @@ -595,6 +635,12 @@ func main() { len(contributors), ) + err = validateRelativeUrls(contributors) + if err != nil { + log.Panic(err) + } + log.Println("all relative URLs for READMEs are valid") + log.Printf( "Processed all READMEs in the %q directory\n", rootRegistryPath, diff --git a/.github/workflows/readme-validation.yaml b/.github/workflows/readme-validation.yaml deleted file mode 100644 index e69de29b..00000000 From df2e47e51e68ba4c453536c2821468d2d208f423 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 8 Apr 2025 20:11:43 +0000 Subject: [PATCH 12/30] refactor: remove unnecessary intermediary data types --- .../scripts/validate-profile-readmes/main.go | 126 +++--------------- 1 file changed, 18 insertions(+), 108 deletions(-) diff --git a/.github/scripts/validate-profile-readmes/main.go b/.github/scripts/validate-profile-readmes/main.go index abc3b010..a7c8902f 100644 --- a/.github/scripts/validate-profile-readmes/main.go +++ b/.github/scripts/validate-profile-readmes/main.go @@ -38,41 +38,6 @@ type contributorFrontmatterWithFilepath struct { FilePath string } -type contributorProfileStatus int - -const ( - // Community should always be the first value defined via iota; it should be - // treated as the zero value of the type in the event that a more specific - // status wasn't defined - profileStatusCommunity contributorProfileStatus = iota - profileStatusPartner - profileStatusOfficial -) - -func (status contributorProfileStatus) String() string { - switch status { - case profileStatusOfficial: - return "official" - case profileStatusPartner: - return "partner" - default: - return "community" - } -} - -type contributorProfile struct { - EmployeeGithubUsernames []string - GithubUsername string - FilePath string - DisplayName string - Bio string - AvatarUrl *string - WebsiteURL *string - LinkedinURL *string - SupportEmail *string - Status contributorProfileStatus -} - var _ error = workflowPhaseError{} type workflowPhaseError struct { @@ -399,57 +364,16 @@ func validateContributorYaml(yml contributorFrontmatterWithFilepath) []error { return problems } -func remapContributorProfile( - frontmatter contributorFrontmatterWithFilepath, - employeeGitHubNames []string, -) contributorProfile { - // Function assumes that (1) fields are previously validated and are safe to - // copy over verbatim when appropriate, and (2) any missing avatar URLs will - // be backfilled during the main Registry site build step - remapped := contributorProfile{ - FilePath: frontmatter.FilePath, - DisplayName: frontmatter.DisplayName, - GithubUsername: frontmatter.GithubUsername, - Bio: frontmatter.Bio, - LinkedinURL: frontmatter.LinkedinURL, - SupportEmail: frontmatter.SupportEmail, - WebsiteURL: frontmatter.WebsiteURL, - AvatarUrl: frontmatter.AvatarUrl, - } - - if frontmatter.ContributorStatus != nil { - switch *frontmatter.ContributorStatus { - case "partner": - remapped.Status = profileStatusPartner - case "official": - remapped.Status = profileStatusOfficial - default: - remapped.Status = profileStatusCommunity - } - } - if employeeGitHubNames != nil { - remapped.EmployeeGithubUsernames = employeeGitHubNames[:] - slices.SortFunc( - remapped.EmployeeGithubUsernames, - func(name1 string, name2 string) int { - return strings.Compare(name1, name2) - }, - ) - } - - return remapped -} - func parseContributorFiles(readmeEntries []directoryReadme) ( - map[string]contributorProfile, + map[string]contributorFrontmatterWithFilepath, error, ) { - frontmatterByGithub := map[string]contributorFrontmatterWithFilepath{} + frontmatterByUsername := map[string]contributorFrontmatterWithFilepath{} yamlParsingErrors := workflowPhaseError{ Phase: "YAML parsing", } for _, rm := range readmeEntries { - fmText, err := extractFrontmatter(rm.RawText) + fm, err := extractFrontmatter(rm.RawText) if err != nil { yamlParsingErrors.Errors = append( yamlParsingErrors.Errors, @@ -459,33 +383,32 @@ func parseContributorFiles(readmeEntries []directoryReadme) ( } yml := rawContributorProfileFrontmatter{} - if err := yaml.Unmarshal([]byte(fmText), &yml); err != nil { + if err := yaml.Unmarshal([]byte(fm), &yml); err != nil { yamlParsingErrors.Errors = append( yamlParsingErrors.Errors, fmt.Errorf("failed to parse %q: %v", rm.FilePath, err), ) continue } - - trackable := contributorFrontmatterWithFilepath{ + processed := contributorFrontmatterWithFilepath{ FilePath: rm.FilePath, rawContributorProfileFrontmatter: yml, } - if prev, conflict := frontmatterByGithub[trackable.GithubUsername]; conflict { + if prev, conflict := frontmatterByUsername[processed.GithubUsername]; conflict { yamlParsingErrors.Errors = append( yamlParsingErrors.Errors, fmt.Errorf( "GitHub name conflict for %q for files %q and %q", - trackable.GithubUsername, + processed.GithubUsername, prev.FilePath, - trackable.FilePath, + processed.FilePath, ), ) continue } - frontmatterByGithub[trackable.GithubUsername] = trackable + frontmatterByUsername[processed.GithubUsername] = processed } if len(yamlParsingErrors.Errors) != 0 { return nil, yamlParsingErrors @@ -495,7 +418,7 @@ func parseContributorFiles(readmeEntries []directoryReadme) ( yamlValidationErrors := workflowPhaseError{ Phase: "Raw YAML Validation", } - for _, yml := range frontmatterByGithub { + for _, yml := range frontmatterByUsername { errors := validateContributorYaml(yml) if len(errors) > 0 { yamlValidationErrors.Errors = append( @@ -512,25 +435,12 @@ func parseContributorFiles(readmeEntries []directoryReadme) ( ) } } - if len(yamlValidationErrors.Errors) != 0 { - return nil, yamlValidationErrors - } - - contributorError := workflowPhaseError{ - Phase: "Contributor struct remapping", - } - structured := map[string]contributorProfile{} - for _, yml := range frontmatterByGithub { - group := employeeGithubGroups[yml.GithubUsername] - remapped := remapContributorProfile(yml, group) - structured[yml.GithubUsername] = remapped - } for companyName, group := range employeeGithubGroups { - if _, found := structured[companyName]; found { + if _, found := frontmatterByUsername[companyName]; found { continue } - contributorError.Errors = append( - contributorError.Errors, + yamlValidationErrors.Errors = append( + yamlValidationErrors.Errors, fmt.Errorf( "company %q does not exist in %q directory but is referenced by these profiles: [%s]", companyName, @@ -539,11 +449,11 @@ func parseContributorFiles(readmeEntries []directoryReadme) ( ), ) } - if len(contributorError.Errors) != 0 { - return nil, contributorError + if len(yamlValidationErrors.Errors) != 0 { + return nil, yamlValidationErrors } - return structured, nil + return frontmatterByUsername, nil } func aggregateReadmeFiles() ([]directoryReadme, error) { @@ -590,7 +500,7 @@ func aggregateReadmeFiles() ([]directoryReadme, error) { } func validateRelativeUrls( - contributors map[string]contributorProfile, + contributors map[string]contributorFrontmatterWithFilepath, ) error { // This function only validates relative avatar URLs for now, but it can be // beefed up to validate more in the future @@ -639,7 +549,7 @@ func main() { if err != nil { log.Panic(err) } - log.Println("all relative URLs for READMEs are valid") + log.Println("All relative URLs for READMEs are valid") log.Printf( "Processed all READMEs in the %q directory\n", From 36ebd8d68b14dae7cb121effb0f089c48efe790d Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 8 Apr 2025 20:26:24 +0000 Subject: [PATCH 13/30] fix: update script to be runnable from root directory --- .github/scripts/validate-profile-readmes/go.mod | 10 ---------- .github/scripts/validate-profile-readmes/go.sum | 8 -------- .../scripts/contributor-readme-validation.go | 2 +- go.mod | 5 +++++ go.sum | 2 ++ 5 files changed, 8 insertions(+), 19 deletions(-) delete mode 100644 .github/scripts/validate-profile-readmes/go.mod delete mode 100644 .github/scripts/validate-profile-readmes/go.sum rename .github/scripts/validate-profile-readmes/main.go => cmd/scripts/contributor-readme-validation.go (99%) create mode 100644 go.mod create mode 100644 go.sum diff --git a/.github/scripts/validate-profile-readmes/go.mod b/.github/scripts/validate-profile-readmes/go.mod deleted file mode 100644 index 3eae387a..00000000 --- a/.github/scripts/validate-profile-readmes/go.mod +++ /dev/null @@ -1,10 +0,0 @@ -module coder.com/readme-validation - -go 1.23.2 - -require github.com/ghodss/yaml v1.0.0 - -require ( - gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect -) diff --git a/.github/scripts/validate-profile-readmes/go.sum b/.github/scripts/validate-profile-readmes/go.sum deleted file mode 100644 index aa535201..00000000 --- a/.github/scripts/validate-profile-readmes/go.sum +++ /dev/null @@ -1,8 +0,0 @@ -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/.github/scripts/validate-profile-readmes/main.go b/cmd/scripts/contributor-readme-validation.go similarity index 99% rename from .github/scripts/validate-profile-readmes/main.go rename to cmd/scripts/contributor-readme-validation.go index a7c8902f..f37a1fe2 100644 --- a/.github/scripts/validate-profile-readmes/main.go +++ b/cmd/scripts/contributor-readme-validation.go @@ -14,7 +14,7 @@ import ( "gopkg.in/yaml.v3" ) -const rootRegistryPath = "../../../registry" +const rootRegistryPath = "./registry" type directoryReadme struct { FilePath string diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..e4074228 --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module coder.com/coder-registry + +go 1.23.2 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..d5428116 --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From c3f998dbd9ed2a9a45aa2da3dc683a1880f6cc64 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 8 Apr 2025 20:28:01 +0000 Subject: [PATCH 14/30] refactor: rename script --- ...ion.go => validate-contributor-readmes.go} | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) rename cmd/scripts/{contributor-readme-validation.go => validate-contributor-readmes.go} (95%) diff --git a/cmd/scripts/contributor-readme-validation.go b/cmd/scripts/validate-contributor-readmes.go similarity index 95% rename from cmd/scripts/contributor-readme-validation.go rename to cmd/scripts/validate-contributor-readmes.go index f37a1fe2..87bf3e01 100644 --- a/cmd/scripts/contributor-readme-validation.go +++ b/cmd/scripts/validate-contributor-readmes.go @@ -21,7 +21,7 @@ type directoryReadme struct { RawText string } -type rawContributorProfileFrontmatter struct { +type contributorProfileFrontmatter struct { DisplayName string `yaml:"display_name"` Bio string `yaml:"bio"` GithubUsername string `yaml:"github"` @@ -33,8 +33,8 @@ type rawContributorProfileFrontmatter struct { ContributorStatus *string `yaml:"status"` } -type contributorFrontmatterWithFilepath struct { - rawContributorProfileFrontmatter +type contributorFrontmatterWithFilePath struct { + contributorProfileFrontmatter FilePath string } @@ -89,7 +89,7 @@ func extractFrontmatter(readmeText string) (string, error) { return fm, nil } -func validateContributorYaml(yml contributorFrontmatterWithFilepath) []error { +func validateContributorYaml(yml contributorFrontmatterWithFilePath) []error { // This function needs to aggregate a bunch of different problems, rather // than stopping at the first one found, so using code blocks to section off // logic for different fields @@ -365,10 +365,10 @@ func validateContributorYaml(yml contributorFrontmatterWithFilepath) []error { } func parseContributorFiles(readmeEntries []directoryReadme) ( - map[string]contributorFrontmatterWithFilepath, + map[string]contributorFrontmatterWithFilePath, error, ) { - frontmatterByUsername := map[string]contributorFrontmatterWithFilepath{} + frontmatterByUsername := map[string]contributorFrontmatterWithFilePath{} yamlParsingErrors := workflowPhaseError{ Phase: "YAML parsing", } @@ -382,7 +382,7 @@ func parseContributorFiles(readmeEntries []directoryReadme) ( continue } - yml := rawContributorProfileFrontmatter{} + yml := contributorProfileFrontmatter{} if err := yaml.Unmarshal([]byte(fm), &yml); err != nil { yamlParsingErrors.Errors = append( yamlParsingErrors.Errors, @@ -390,9 +390,9 @@ func parseContributorFiles(readmeEntries []directoryReadme) ( ) continue } - processed := contributorFrontmatterWithFilepath{ - FilePath: rm.FilePath, - rawContributorProfileFrontmatter: yml, + processed := contributorFrontmatterWithFilePath{ + FilePath: rm.FilePath, + contributorProfileFrontmatter: yml, } if prev, conflict := frontmatterByUsername[processed.GithubUsername]; conflict { @@ -500,7 +500,7 @@ func aggregateReadmeFiles() ([]directoryReadme, error) { } func validateRelativeUrls( - contributors map[string]contributorFrontmatterWithFilepath, + contributors map[string]contributorFrontmatterWithFilePath, ) error { // This function only validates relative avatar URLs for now, but it can be // beefed up to validate more in the future From 9e48eb806f20c07cfa65e37d482de9fde0e71f83 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 8 Apr 2025 21:01:36 +0000 Subject: [PATCH 15/30] refactor: reorganize scripts again --- go.sum | 2 ++ .../validate-contributor-readmes/main.go | 9 ++++----- 2 files changed, 6 insertions(+), 5 deletions(-) rename cmd/scripts/validate-contributor-readmes.go => scripts/validate-contributor-readmes/main.go (97%) diff --git a/go.sum b/go.sum index d5428116..a62c313c 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cmd/scripts/validate-contributor-readmes.go b/scripts/validate-contributor-readmes/main.go similarity index 97% rename from cmd/scripts/validate-contributor-readmes.go rename to scripts/validate-contributor-readmes/main.go index 87bf3e01..940b123c 100644 --- a/cmd/scripts/validate-contributor-readmes.go +++ b/scripts/validate-contributor-readmes/main.go @@ -25,7 +25,7 @@ type contributorProfileFrontmatter struct { DisplayName string `yaml:"display_name"` Bio string `yaml:"bio"` GithubUsername string `yaml:"github"` - AvatarUrl *string `yaml:"avatar"` + AvatarUrl *string `yaml:"avatar"` // Script assumes that if value is nil, the Registry site build step will backfill the value with the user's GitHub avatar URL LinkedinURL *string `yaml:"linkedin"` WebsiteURL *string `yaml:"website"` SupportEmail *string `yaml:"support_email"` @@ -510,13 +510,12 @@ func validateRelativeUrls( if con.AvatarUrl == nil { continue } - isRelativeUrl := strings.HasPrefix(*con.AvatarUrl, ".") || - strings.HasPrefix(*con.AvatarUrl, "/") - if !isRelativeUrl { + if isRelativeUrl := strings.HasPrefix(*con.AvatarUrl, ".") || + strings.HasPrefix(*con.AvatarUrl, "/"); !isRelativeUrl { continue } - fmt.Println(con.GithubUsername, con.AvatarUrl) + fmt.Println(con.GithubUsername, con.FilePath, con.AvatarUrl) } if len(problems) == 0 { From 3b9ec5ec41985ef8716d5c8582d4318af44ebd23 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 8 Apr 2025 21:23:36 +0000 Subject: [PATCH 16/30] chore: finish initial version of validation script --- scripts/validate-contributor-readmes/main.go | 33 +++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/scripts/validate-contributor-readmes/main.go b/scripts/validate-contributor-readmes/main.go index 940b123c..06f93f5c 100644 --- a/scripts/validate-contributor-readmes/main.go +++ b/scripts/validate-contributor-readmes/main.go @@ -359,6 +359,25 @@ func validateContributorYaml(yml contributorFrontmatterWithFilePath) []error { ), ) } + + supportedFileFormats := []string{".png", ".jpeg", ".jpg", ".gif", ".svg"} + matched := false + for _, ff := range supportedFileFormats { + matched = strings.HasSuffix(*yml.AvatarUrl, ff) + if matched { + break + } + } + if !matched { + problems = append( + problems, + fmt.Errorf( + "avatar URL for %q does not end in a supported file format: [%s]", + yml.FilePath, + strings.Join(supportedFileFormats, ", "), + ), + ) + } }() return problems @@ -515,7 +534,19 @@ func validateRelativeUrls( continue } - fmt.Println(con.GithubUsername, con.FilePath, con.AvatarUrl) + absolutePath := strings.TrimSuffix(con.FilePath, "README.md") + + *con.AvatarUrl + _, err := os.ReadFile(absolutePath) + if err != nil { + problems = append( + problems, + fmt.Errorf( + "relative avatar path %q for %q does not point to image in file system", + *con.AvatarUrl, + con.FilePath, + ), + ) + } } if len(problems) == 0 { From 88f7be27ec2ec6ff67848564a04c53654bdd4da7 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Wed, 9 Apr 2025 15:01:08 +0000 Subject: [PATCH 17/30] chore: set up initial version of CI --- .github/workflows/validate-readme-files.yaml | 21 +++++++++++++ scripts/validate-contributor-readmes/main.go | 32 ++++++++++---------- 2 files changed, 37 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/validate-readme-files.yaml diff --git a/.github/workflows/validate-readme-files.yaml b/.github/workflows/validate-readme-files.yaml new file mode 100644 index 00000000..ed6c22d1 --- /dev/null +++ b/.github/workflows/validate-readme-files.yaml @@ -0,0 +1,21 @@ +name: Validate README files +on: + pull_request: + branches: + - main +jobs: + validate-contributors: + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + go-version: ["1.23.x"] + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Set up Go ${{ matrix.go-version }} + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + - name: Validate + run: go run ./scripts/validate-contributor-readmes/main.go diff --git a/scripts/validate-contributor-readmes/main.go b/scripts/validate-contributor-readmes/main.go index 06f93f5c..0a00f995 100644 --- a/scripts/validate-contributor-readmes/main.go +++ b/scripts/validate-contributor-readmes/main.go @@ -16,7 +16,7 @@ import ( const rootRegistryPath = "./registry" -type directoryReadme struct { +type readme struct { FilePath string RawText string } @@ -38,16 +38,16 @@ type contributorFrontmatterWithFilePath struct { FilePath string } -var _ error = workflowPhaseError{} +var _ error = validationPhaseError{} -type workflowPhaseError struct { +type validationPhaseError struct { Phase string Errors []error } -func (wpe workflowPhaseError) Error() string { - msg := fmt.Sprintf("Error during %q phase of README validation:", wpe.Phase) - for _, e := range wpe.Errors { +func (vpe validationPhaseError) Error() string { + msg := fmt.Sprintf("Error during %q phase of README validation:", vpe.Phase) + for _, e := range vpe.Errors { msg += fmt.Sprintf("\n- %v", e) } msg += "\n" @@ -383,12 +383,12 @@ func validateContributorYaml(yml contributorFrontmatterWithFilePath) []error { return problems } -func parseContributorFiles(readmeEntries []directoryReadme) ( +func parseContributorFiles(readmeEntries []readme) ( map[string]contributorFrontmatterWithFilePath, error, ) { frontmatterByUsername := map[string]contributorFrontmatterWithFilePath{} - yamlParsingErrors := workflowPhaseError{ + yamlParsingErrors := validationPhaseError{ Phase: "YAML parsing", } for _, rm := range readmeEntries { @@ -434,7 +434,7 @@ func parseContributorFiles(readmeEntries []directoryReadme) ( } employeeGithubGroups := map[string][]string{} - yamlValidationErrors := workflowPhaseError{ + yamlValidationErrors := validationPhaseError{ Phase: "Raw YAML Validation", } for _, yml := range frontmatterByUsername { @@ -475,13 +475,13 @@ func parseContributorFiles(readmeEntries []directoryReadme) ( return frontmatterByUsername, nil } -func aggregateReadmeFiles() ([]directoryReadme, error) { +func aggregateContributorReadmeFiles() ([]readme, error) { dirEntries, err := os.ReadDir(rootRegistryPath) if err != nil { return nil, err } - allReadmeFiles := []directoryReadme{} + allReadmeFiles := []readme{} problems := []error{} for _, e := range dirEntries { dirPath := path.Join(rootRegistryPath, e.Name()) @@ -502,14 +502,14 @@ func aggregateReadmeFiles() ([]directoryReadme, error) { problems = append(problems, err) continue } - allReadmeFiles = append(allReadmeFiles, directoryReadme{ + allReadmeFiles = append(allReadmeFiles, readme{ FilePath: readmePath, RawText: string(rmBytes), }) } if len(problems) != 0 { - return nil, workflowPhaseError{ + return nil, validationPhaseError{ Phase: "FileSystem reading", Errors: problems, } @@ -552,7 +552,7 @@ func validateRelativeUrls( if len(problems) == 0 { return nil } - return workflowPhaseError{ + return validationPhaseError{ Phase: "Relative URL validation", Errors: problems, } @@ -560,9 +560,9 @@ func validateRelativeUrls( func main() { log.Println("Starting README validation") - allReadmeFiles, err := aggregateReadmeFiles() + allReadmeFiles, err := aggregateContributorReadmeFiles() if err != nil { - panic(err) + log.Panic(err) } log.Printf("Processing %d README files\n", len(allReadmeFiles)) From e035f1fca3eb8f51c5c7cffaf1c1809b2bc9482f Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Wed, 9 Apr 2025 15:26:22 +0000 Subject: [PATCH 18/30] chore: beef up CI --- .../workflows/{validate-readme-files.yaml => ci.yaml} | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) rename .github/workflows/{validate-readme-files.yaml => ci.yaml} (66%) diff --git a/.github/workflows/validate-readme-files.yaml b/.github/workflows/ci.yaml similarity index 66% rename from .github/workflows/validate-readme-files.yaml rename to .github/workflows/ci.yaml index ed6c22d1..4c1a7d6b 100644 --- a/.github/workflows/validate-readme-files.yaml +++ b/.github/workflows/ci.yaml @@ -1,8 +1,11 @@ -name: Validate README files +name: CI on: pull_request: - branches: - - main + branches: [main] +# Cancel in-progress runs for pull requests when developers push new changes +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: validate-contributors: runs-on: ubuntu-latest From 3b9c01ea6c3d3bc49739eee1d1e2502d41d22e63 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Wed, 9 Apr 2025 16:07:58 +0000 Subject: [PATCH 19/30] fix: ensure relative avatars keep small scope --- scripts/validate-contributor-readmes/main.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts/validate-contributor-readmes/main.go b/scripts/validate-contributor-readmes/main.go index 0a00f995..6cc3ed7d 100644 --- a/scripts/validate-contributor-readmes/main.go +++ b/scripts/validate-contributor-readmes/main.go @@ -414,7 +414,7 @@ func parseContributorFiles(readmeEntries []readme) ( contributorProfileFrontmatter: yml, } - if prev, conflict := frontmatterByUsername[processed.GithubUsername]; conflict { + if prev, isConflict := frontmatterByUsername[processed.GithubUsername]; isConflict { yamlParsingErrors.Errors = append( yamlParsingErrors.Errors, fmt.Errorf( @@ -534,6 +534,17 @@ func validateRelativeUrls( continue } + if strings.HasPrefix(*con.AvatarUrl, "..") { + problems = append( + problems, + fmt.Errorf( + "%q: relative avatar URLs cannot be placed outside a user's namespaced directory", + con.FilePath, + ), + ) + continue + } + absolutePath := strings.TrimSuffix(con.FilePath, "README.md") + *con.AvatarUrl _, err := os.ReadFile(absolutePath) From bc4bbdaa07b19f0efe32fea246d22fd6edbab81f Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Wed, 9 Apr 2025 19:25:48 +0000 Subject: [PATCH 20/30] fix: remove unnecessary matrix --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4c1a7d6b..d654a6a5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,9 +16,9 @@ jobs: steps: - name: Check out code uses: actions/checkout@v4 - - name: Set up Go ${{ matrix.go-version }} + - name: Set up Go uses: actions/setup-go@v5 with: - go-version: ${{ matrix.go-version }} + go-version: "1.23.2" - name: Validate run: go run ./scripts/validate-contributor-readmes/main.go From abf9815a846f371a9e92b3840f70dd69bae1ec63 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Thu, 10 Apr 2025 20:40:54 +0000 Subject: [PATCH 21/30] fix: update static files --- .github/workflows/ci.yaml | 4 ---- registry/nataindata/README.md | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d654a6a5..4f9c9f95 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,10 +9,6 @@ concurrency: jobs: validate-contributors: runs-on: ubuntu-latest - strategy: - fail-fast: true - matrix: - go-version: ["1.23.x"] steps: - name: Check out code uses: actions/checkout@v4 diff --git a/registry/nataindata/README.md b/registry/nataindata/README.md index 5f291817..ddc5095f 100644 --- a/registry/nataindata/README.md +++ b/registry/nataindata/README.md @@ -3,5 +3,5 @@ display_name: Nataindata bio: Data engineer github: nataindata website: https://www.nataindata.com -status: partner +status: community --- From affc5063ca5edddd0ea0380716e354b5c836d0ea Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Thu, 10 Apr 2025 21:01:58 +0000 Subject: [PATCH 22/30] refactor: split validation function into smaller pieces --- scripts/validate-contributor-readmes/main.go | 573 ++++++++++--------- 1 file changed, 302 insertions(+), 271 deletions(-) diff --git a/scripts/validate-contributor-readmes/main.go b/scripts/validate-contributor-readmes/main.go index 6cc3ed7d..8a3a13ad 100644 --- a/scripts/validate-contributor-readmes/main.go +++ b/scripts/validate-contributor-readmes/main.go @@ -22,10 +22,12 @@ type readme struct { } type contributorProfileFrontmatter struct { - DisplayName string `yaml:"display_name"` - Bio string `yaml:"bio"` - GithubUsername string `yaml:"github"` - AvatarUrl *string `yaml:"avatar"` // Script assumes that if value is nil, the Registry site build step will backfill the value with the user's GitHub avatar URL + DisplayName string `yaml:"display_name"` + Bio string `yaml:"bio"` + GithubUsername string `yaml:"github"` + // Script assumes that if value is nil, the Registry site build step will + // backfill the value with the user's GitHub avatar URL + AvatarURL *string `yaml:"avatar"` LinkedinURL *string `yaml:"linkedin"` WebsiteURL *string `yaml:"website"` SupportEmail *string `yaml:"support_email"` @@ -89,300 +91,329 @@ func extractFrontmatter(readmeText string) (string, error) { return fm, nil } -func validateContributorYaml(yml contributorFrontmatterWithFilePath) []error { - // This function needs to aggregate a bunch of different problems, rather - // than stopping at the first one found, so using code blocks to section off - // logic for different fields +// A validation function for verifying one specific aspect of a contributor's +// frontmatter content. Each function should be able to return ALL data +// violations that apply to the function's area of concern, rather than +// returning the first error found +type contributorValidationFunc = func(fm contributorFrontmatterWithFilePath) []error + +func validateContributorGithubUsername(fm contributorFrontmatterWithFilePath) []error { problems := []error{} - // Using a bunch of closures to group validations for each field and add - // support for ending validations for a group early. The alternatives were - // making a bunch of functions in the top-level that would only be used - // once, or using goto statements, which would've made refactoring fragile + if fm.GithubUsername == "" { + problems = append( + problems, + fmt.Errorf( + "missing GitHub username for %q", + fm.FilePath, + ), + ) + return problems + } - // GitHub Username - func() { - if yml.GithubUsername == "" { - problems = append( - problems, - fmt.Errorf( - "missing GitHub username for %q", - yml.FilePath, - ), - ) - return - } + lower := strings.ToLower(fm.GithubUsername) + if uriSafe := url.PathEscape(lower); uriSafe != lower { + problems = append( + problems, + fmt.Errorf( + "gitHub username %q (%q) is not a valid URL path segment", + fm.GithubUsername, + fm.FilePath, + ), + ) + } - lower := strings.ToLower(yml.GithubUsername) - if uriSafe := url.PathEscape(lower); uriSafe != lower { - problems = append( - problems, - fmt.Errorf( - "gitHub username %q (%q) is not a valid URL path segment", - yml.GithubUsername, - yml.FilePath, - ), - ) - } - }() + return problems +} - // Company GitHub - func() { - if yml.EmployerGithubUsername == nil { - return - } +func validateContributorEmployerGithubUsername(fm contributorFrontmatterWithFilePath) []error { + if fm.EmployerGithubUsername == nil { + return nil + } - if *yml.EmployerGithubUsername == "" { - problems = append( - problems, - fmt.Errorf( - "company_github field is defined but has empty value for %q", - yml.FilePath, - ), - ) - return - } + problems := []error{} - lower := strings.ToLower(*yml.EmployerGithubUsername) - if uriSafe := url.PathEscape(lower); uriSafe != lower { - problems = append( - problems, - fmt.Errorf( - "gitHub company username %q (%q) is not a valid URL path segment", - *yml.EmployerGithubUsername, - yml.FilePath, - ), - ) - } + if *fm.EmployerGithubUsername == "" { + problems = append( + problems, + fmt.Errorf( + "company_github field is defined but has empty value for %q", + fm.FilePath, + ), + ) + return problems + } - if *yml.EmployerGithubUsername == yml.GithubUsername { - problems = append( - problems, - fmt.Errorf( - "cannot list own GitHub name (%q) as employer (%q)", - yml.GithubUsername, - yml.FilePath, - ), - ) - } - }() + lower := strings.ToLower(*fm.EmployerGithubUsername) + if uriSafe := url.PathEscape(lower); uriSafe != lower { + problems = append( + problems, + fmt.Errorf( + "gitHub company username %q (%q) is not a valid URL path segment", + *fm.EmployerGithubUsername, + fm.FilePath, + ), + ) + } - // Display name - func() { - if yml.DisplayName == "" { - problems = append( - problems, - fmt.Errorf( - "GitHub user %q (%q) is missing display name", - yml.GithubUsername, - yml.FilePath, - ), - ) - } + if *fm.EmployerGithubUsername == fm.GithubUsername { + problems = append( + problems, + fmt.Errorf( + "cannot list own GitHub name (%q) as employer (%q)", + fm.GithubUsername, + fm.FilePath, + ), + ) + } - }() + return problems +} - // LinkedIn URL - func() { - if yml.LinkedinURL == nil { - return - } +func validateContributorDisplayName(fm contributorFrontmatterWithFilePath) []error { + problems := []error{} + if fm.DisplayName == "" { + problems = append( + problems, + fmt.Errorf( + "GitHub user %q (%q) is missing display name", + fm.GithubUsername, + fm.FilePath, + ), + ) + } - if _, err := url.ParseRequestURI(*yml.LinkedinURL); err != nil { - problems = append( - problems, - fmt.Errorf( - "linkedIn URL %q (%q) is not valid: %v", - *yml.LinkedinURL, - yml.FilePath, - err, - ), - ) - } - }() + return problems +} - // Email - func() { - if yml.SupportEmail == nil { - return - } +func validateContributorLinkedinURL(fm contributorFrontmatterWithFilePath) []error { + if fm.LinkedinURL == nil { + return nil + } - // Can't 100% validate that this is correct without actually sending - // an email, and especially with some contributors being individual - // developers, we don't want to do that on every single run of the CI - // pipeline. Best we can do is verify the general structure - username, server, ok := strings.Cut(*yml.SupportEmail, "@") - if !ok { - problems = append( - problems, - fmt.Errorf( - "email address %q (%q) is missing @ symbol", - *yml.LinkedinURL, - yml.FilePath, - ), - ) - return - } + problems := []error{} + if _, err := url.ParseRequestURI(*fm.LinkedinURL); err != nil { + problems = append( + problems, + fmt.Errorf( + "linkedIn URL %q (%q) is not valid: %v", + *fm.LinkedinURL, + fm.FilePath, + err, + ), + ) + } - if username == "" { - problems = append( - problems, - fmt.Errorf( - "email address %q (%q) is missing username", - *yml.LinkedinURL, - yml.FilePath, - ), - ) - } + return problems +} - domain, tld, ok := strings.Cut(server, ".") - if !ok { - problems = append( - problems, - fmt.Errorf( - "email address %q (%q) is missing period for server segment", - *yml.LinkedinURL, - yml.FilePath, - ), - ) - return - } +func validateContributorEmail(fm contributorFrontmatterWithFilePath) []error { + if fm.SupportEmail == nil { + return nil + } - if domain == "" { - problems = append( - problems, - fmt.Errorf( - "email address %q (%q) is missing domain", - *yml.LinkedinURL, - yml.FilePath, - ), - ) - } + problems := []error{} - if tld == "" { - problems = append( - problems, - fmt.Errorf( - "email address %q (%q) is missing top-level domain", - *yml.LinkedinURL, - yml.FilePath, - ), - ) - } + // Can't 100% validate that this is correct without actually sending + // an email, and especially with some contributors being individual + // developers, we don't want to do that on every single run of the CI + // pipeline. Best we can do is verify the general structure + username, server, ok := strings.Cut(*fm.SupportEmail, "@") + if !ok { + problems = append( + problems, + fmt.Errorf( + "email address %q (%q) is missing @ symbol", + *fm.LinkedinURL, + fm.FilePath, + ), + ) + return problems + } - if strings.Contains(*yml.SupportEmail, "?") { - problems = append( - problems, - fmt.Errorf( - "email for %q is not allowed to contain search parameters", - yml.FilePath, - ), - ) - } - }() + if username == "" { + problems = append( + problems, + fmt.Errorf( + "email address %q (%q) is missing username", + *fm.LinkedinURL, + fm.FilePath, + ), + ) + } - // Website - func() { - if yml.WebsiteURL == nil { - return - } + domain, tld, ok := strings.Cut(server, ".") + if !ok { + problems = append( + problems, + fmt.Errorf( + "email address %q (%q) is missing period for server segment", + *fm.LinkedinURL, + fm.FilePath, + ), + ) + return problems + } - if _, err := url.ParseRequestURI(*yml.WebsiteURL); err != nil { - problems = append( - problems, - fmt.Errorf( - "LinkedIn URL %q (%q) is not valid: %v", - *yml.WebsiteURL, - yml.FilePath, - err, - ), - ) - } - }() + if domain == "" { + problems = append( + problems, + fmt.Errorf( + "email address %q (%q) is missing domain", + *fm.LinkedinURL, + fm.FilePath, + ), + ) + } - // Contributor status - func() { - if yml.ContributorStatus == nil { - return - } + if tld == "" { + problems = append( + problems, + fmt.Errorf( + "email address %q (%q) is missing top-level domain", + *fm.LinkedinURL, + fm.FilePath, + ), + ) + } - validStatuses := []string{"official", "partner", "community"} - if !slices.Contains(validStatuses, *yml.ContributorStatus) { - problems = append( - problems, - fmt.Errorf( - "contributor status %q (%q) is not valid", - *yml.ContributorStatus, - yml.FilePath, - ), - ) - } - }() + if strings.Contains(*fm.SupportEmail, "?") { + problems = append( + problems, + fmt.Errorf( + "email for %q is not allowed to contain search parameters", + fm.FilePath, + ), + ) + } - // Avatar URL - can't validate the image actually leads to a valid resource - // in a pure function, but can at least catch obvious problems - func() { - if yml.AvatarUrl == nil { - return - } + return problems +} - if *yml.AvatarUrl == "" { - problems = append( - problems, - fmt.Errorf( - "avatar URL for %q must be omitted or non-empty string", - yml.FilePath, - ), - ) - return - } +func validateContributorWebsite(fm contributorFrontmatterWithFilePath) []error { + if fm.WebsiteURL == nil { + return nil + } - // Have to use .Parse instead of .ParseRequestURI because this is the - // one field that's allowed to be a relative URL - if _, err := url.Parse(*yml.AvatarUrl); err != nil { - problems = append( - problems, - fmt.Errorf( - "error %q (%q) is not a valid relative or absolute URL", - *yml.AvatarUrl, - yml.FilePath, - ), - ) - } + problems := []error{} + if _, err := url.ParseRequestURI(*fm.WebsiteURL); err != nil { + problems = append( + problems, + fmt.Errorf( + "LinkedIn URL %q (%q) is not valid: %v", + *fm.WebsiteURL, + fm.FilePath, + err, + ), + ) + } - if strings.Contains(*yml.AvatarUrl, "?") { - problems = append( - problems, - fmt.Errorf( - "avatar URL for %q is not allowed to contain search parameters", - yml.FilePath, - ), - ) - } + return problems +} - supportedFileFormats := []string{".png", ".jpeg", ".jpg", ".gif", ".svg"} - matched := false - for _, ff := range supportedFileFormats { - matched = strings.HasSuffix(*yml.AvatarUrl, ff) - if matched { - break - } - } - if !matched { - problems = append( - problems, - fmt.Errorf( - "avatar URL for %q does not end in a supported file format: [%s]", - yml.FilePath, - strings.Join(supportedFileFormats, ", "), - ), - ) +func validateContributorStatus(fm contributorFrontmatterWithFilePath) []error { + if fm.ContributorStatus == nil { + return nil + } + + problems := []error{} + validStatuses := []string{"official", "partner", "community"} + if !slices.Contains(validStatuses, *fm.ContributorStatus) { + problems = append( + problems, + fmt.Errorf( + "contributor status %q (%q) is not valid", + *fm.ContributorStatus, + fm.FilePath, + ), + ) + } + + return problems +} + +// Can't validate the image actually leads to a valid resource in a pure +// function, but can at least catch obvious problems +func validateContributorAvatarURL(fm contributorFrontmatterWithFilePath) []error { + if fm.AvatarURL == nil { + return nil + } + + problems := []error{} + if *fm.AvatarURL == "" { + problems = append( + problems, + fmt.Errorf( + "avatar URL for %q must be omitted or non-empty string", + fm.FilePath, + ), + ) + return problems + } + + // Have to use .Parse instead of .ParseRequestURI because this is the + // one field that's allowed to be a relative URL + if _, err := url.Parse(*fm.AvatarURL); err != nil { + problems = append( + problems, + fmt.Errorf( + "error %q (%q) is not a valid relative or absolute URL", + *fm.AvatarURL, + fm.FilePath, + ), + ) + } + + if strings.Contains(*fm.AvatarURL, "?") { + problems = append( + problems, + fmt.Errorf( + "avatar URL for %q is not allowed to contain search parameters", + fm.FilePath, + ), + ) + } + + supportedFileFormats := []string{".png", ".jpeg", ".jpg", ".gif", ".svg"} + matched := false + for _, ff := range supportedFileFormats { + matched = strings.HasSuffix(*fm.AvatarURL, ff) + if matched { + break } - }() + } + if !matched { + problems = append( + problems, + fmt.Errorf( + "avatar URL for %q does not end in a supported file format: [%s]", + fm.FilePath, + strings.Join(supportedFileFormats, ", "), + ), + ) + } return problems } +func validateContributorYaml(yml contributorFrontmatterWithFilePath) []error { + validationFuncs := []contributorValidationFunc{ + validateContributorGithubUsername, + validateContributorEmployerGithubUsername, + validateContributorDisplayName, + validateContributorLinkedinURL, + validateContributorEmail, + validateContributorWebsite, + validateContributorStatus, + validateContributorAvatarURL, + } + allProblems := []error{} + for _, fn := range validationFuncs { + allProblems = append(allProblems, fn(yml)...) + } + return allProblems +} + func parseContributorFiles(readmeEntries []readme) ( map[string]contributorFrontmatterWithFilePath, error, @@ -526,15 +557,15 @@ func validateRelativeUrls( problems := []error{} for _, con := range contributors { - if con.AvatarUrl == nil { + if con.AvatarURL == nil { continue } - if isRelativeUrl := strings.HasPrefix(*con.AvatarUrl, ".") || - strings.HasPrefix(*con.AvatarUrl, "/"); !isRelativeUrl { + if isRelativeURL := strings.HasPrefix(*con.AvatarURL, ".") || + strings.HasPrefix(*con.AvatarURL, "/"); !isRelativeURL { continue } - if strings.HasPrefix(*con.AvatarUrl, "..") { + if strings.HasPrefix(*con.AvatarURL, "..") { problems = append( problems, fmt.Errorf( @@ -546,14 +577,14 @@ func validateRelativeUrls( } absolutePath := strings.TrimSuffix(con.FilePath, "README.md") + - *con.AvatarUrl + *con.AvatarURL _, err := os.ReadFile(absolutePath) if err != nil { problems = append( problems, fmt.Errorf( "relative avatar path %q for %q does not point to image in file system", - *con.AvatarUrl, + *con.AvatarURL, con.FilePath, ), ) From 65fb7bcffb7d426e55006ff24f28903d6e7c8275 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Fri, 11 Apr 2025 13:44:15 +0000 Subject: [PATCH 23/30] refactor: standardize how errors are defined --- scripts/validate-contributor-readmes/main.go | 82 +++++++++++--------- 1 file changed, 45 insertions(+), 37 deletions(-) diff --git a/scripts/validate-contributor-readmes/main.go b/scripts/validate-contributor-readmes/main.go index 8a3a13ad..b9f5da8b 100644 --- a/scripts/validate-contributor-readmes/main.go +++ b/scripts/validate-contributor-readmes/main.go @@ -1,3 +1,8 @@ +// This package is for validating all contributors within the main Registry +// directory. It validates that it has nothing but sub-directories, and that +// each sub-directory has a README.md file. Each of those files must then +// describe a specific contributor. The contents of these files will be parsed +// by the Registry site build step, to be displayed in the Registry site's UI. package main import ( @@ -104,7 +109,7 @@ func validateContributorGithubUsername(fm contributorFrontmatterWithFilePath) [] problems = append( problems, fmt.Errorf( - "missing GitHub username for %q", + "%q: missing GitHub username", fm.FilePath, ), ) @@ -116,9 +121,9 @@ func validateContributorGithubUsername(fm contributorFrontmatterWithFilePath) [] problems = append( problems, fmt.Errorf( - "gitHub username %q (%q) is not a valid URL path segment", - fm.GithubUsername, + "%q: gitHub username %q is not a valid URL path segment", fm.FilePath, + fm.GithubUsername, ), ) } @@ -137,7 +142,7 @@ func validateContributorEmployerGithubUsername(fm contributorFrontmatterWithFile problems = append( problems, fmt.Errorf( - "company_github field is defined but has empty value for %q", + "%q: company_github field is defined but has empty value", fm.FilePath, ), ) @@ -149,9 +154,9 @@ func validateContributorEmployerGithubUsername(fm contributorFrontmatterWithFile problems = append( problems, fmt.Errorf( - "gitHub company username %q (%q) is not a valid URL path segment", - *fm.EmployerGithubUsername, + "%q: gitHub company username %q is not a valid URL path segment", fm.FilePath, + *fm.EmployerGithubUsername, ), ) } @@ -160,9 +165,9 @@ func validateContributorEmployerGithubUsername(fm contributorFrontmatterWithFile problems = append( problems, fmt.Errorf( - "cannot list own GitHub name (%q) as employer (%q)", - fm.GithubUsername, + "%q: cannot list own GitHub name (%q) as employer", fm.FilePath, + fm.GithubUsername, ), ) } @@ -176,9 +181,9 @@ func validateContributorDisplayName(fm contributorFrontmatterWithFilePath) []err problems = append( problems, fmt.Errorf( - "GitHub user %q (%q) is missing display name", - fm.GithubUsername, + "%q: GitHub user %q is missing display name", fm.FilePath, + fm.GithubUsername, ), ) } @@ -196,7 +201,7 @@ func validateContributorLinkedinURL(fm contributorFrontmatterWithFilePath) []err problems = append( problems, fmt.Errorf( - "linkedIn URL %q (%q) is not valid: %v", + "%q: linkedIn URL %q is not valid: %v", *fm.LinkedinURL, fm.FilePath, err, @@ -223,9 +228,9 @@ func validateContributorEmail(fm contributorFrontmatterWithFilePath) []error { problems = append( problems, fmt.Errorf( - "email address %q (%q) is missing @ symbol", - *fm.LinkedinURL, + "%q: email address %q is missing @ symbol", fm.FilePath, + *fm.LinkedinURL, ), ) return problems @@ -235,9 +240,9 @@ func validateContributorEmail(fm contributorFrontmatterWithFilePath) []error { problems = append( problems, fmt.Errorf( - "email address %q (%q) is missing username", - *fm.LinkedinURL, + "%q: email address %q is missing username", fm.FilePath, + *fm.LinkedinURL, ), ) } @@ -247,9 +252,9 @@ func validateContributorEmail(fm contributorFrontmatterWithFilePath) []error { problems = append( problems, fmt.Errorf( - "email address %q (%q) is missing period for server segment", - *fm.LinkedinURL, + "%q: email address %q is missing period for server segment", fm.FilePath, + *fm.LinkedinURL, ), ) return problems @@ -259,9 +264,9 @@ func validateContributorEmail(fm contributorFrontmatterWithFilePath) []error { problems = append( problems, fmt.Errorf( - "email address %q (%q) is missing domain", - *fm.LinkedinURL, + "%q: email address %q is missing domain", fm.FilePath, + *fm.LinkedinURL, ), ) } @@ -270,9 +275,9 @@ func validateContributorEmail(fm contributorFrontmatterWithFilePath) []error { problems = append( problems, fmt.Errorf( - "email address %q (%q) is missing top-level domain", - *fm.LinkedinURL, + "%q: email address %q is missing top-level domain", fm.FilePath, + *fm.LinkedinURL, ), ) } @@ -281,7 +286,7 @@ func validateContributorEmail(fm contributorFrontmatterWithFilePath) []error { problems = append( problems, fmt.Errorf( - "email for %q is not allowed to contain search parameters", + "%q: email is not allowed to contain search parameters", fm.FilePath, ), ) @@ -300,9 +305,9 @@ func validateContributorWebsite(fm contributorFrontmatterWithFilePath) []error { problems = append( problems, fmt.Errorf( - "LinkedIn URL %q (%q) is not valid: %v", - *fm.WebsiteURL, + "%q: LinkedIn URL %q is not valid: %v", fm.FilePath, + *fm.WebsiteURL, err, ), ) @@ -322,9 +327,9 @@ func validateContributorStatus(fm contributorFrontmatterWithFilePath) []error { problems = append( problems, fmt.Errorf( - "contributor status %q (%q) is not valid", - *fm.ContributorStatus, + "%q: contributor status %q is not valid", fm.FilePath, + *fm.ContributorStatus, ), ) } @@ -344,7 +349,7 @@ func validateContributorAvatarURL(fm contributorFrontmatterWithFilePath) []error problems = append( problems, fmt.Errorf( - "avatar URL for %q must be omitted or non-empty string", + "%q: avatar URL must be omitted or non-empty string", fm.FilePath, ), ) @@ -357,9 +362,9 @@ func validateContributorAvatarURL(fm contributorFrontmatterWithFilePath) []error problems = append( problems, fmt.Errorf( - "error %q (%q) is not a valid relative or absolute URL", - *fm.AvatarURL, + "%q: URL %q is not a valid relative or absolute URL", fm.FilePath, + *fm.AvatarURL, ), ) } @@ -368,7 +373,7 @@ func validateContributorAvatarURL(fm contributorFrontmatterWithFilePath) []error problems = append( problems, fmt.Errorf( - "avatar URL for %q is not allowed to contain search parameters", + "%q: avatar URL is not allowed to contain search parameters", fm.FilePath, ), ) @@ -383,11 +388,14 @@ func validateContributorAvatarURL(fm contributorFrontmatterWithFilePath) []error } } if !matched { + segments := strings.Split(*fm.AvatarURL, ".") + fileExtension := segments[len(segments)-1] problems = append( problems, fmt.Errorf( - "avatar URL for %q does not end in a supported file format: [%s]", + "%q: avatar URL '.%s' does not end in a supported file format: [%s]", fm.FilePath, + fileExtension, strings.Join(supportedFileFormats, ", "), ), ) @@ -427,7 +435,7 @@ func parseContributorFiles(readmeEntries []readme) ( if err != nil { yamlParsingErrors.Errors = append( yamlParsingErrors.Errors, - fmt.Errorf("failed to parse %q: %v", rm.FilePath, err), + fmt.Errorf("%q: failed to parse: %v", rm.FilePath, err), ) continue } @@ -436,7 +444,7 @@ func parseContributorFiles(readmeEntries []readme) ( if err := yaml.Unmarshal([]byte(fm), &yml); err != nil { yamlParsingErrors.Errors = append( yamlParsingErrors.Errors, - fmt.Errorf("failed to parse %q: %v", rm.FilePath, err), + fmt.Errorf("%q: failed to parse: %v", rm.FilePath, err), ) continue } @@ -449,10 +457,10 @@ func parseContributorFiles(readmeEntries []readme) ( yamlParsingErrors.Errors = append( yamlParsingErrors.Errors, fmt.Errorf( - "GitHub name conflict for %q for files %q and %q", + "%q: GitHub name %s conflicts with field defined in %q", + processed.FilePath, processed.GithubUsername, prev.FilePath, - processed.FilePath, ), ) continue @@ -583,9 +591,9 @@ func validateRelativeUrls( problems = append( problems, fmt.Errorf( - "relative avatar path %q for %q does not point to image in file system", - *con.AvatarURL, + "%q: relative avatar path %q does not point to image in file system", con.FilePath, + *con.AvatarURL, ), ) } From 96fa5d4157e175f9614e66daaa1c1a7a80ff7b11 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Fri, 11 Apr 2025 16:55:09 +0000 Subject: [PATCH 24/30] refactor: apply majority of feedback --- scripts/validate-contributor-readmes/main.go | 363 ++++++------------- 1 file changed, 103 insertions(+), 260 deletions(-) diff --git a/scripts/validate-contributor-readmes/main.go b/scripts/validate-contributor-readmes/main.go index b9f5da8b..bab52089 100644 --- a/scripts/validate-contributor-readmes/main.go +++ b/scripts/validate-contributor-readmes/main.go @@ -53,10 +53,14 @@ type validationPhaseError struct { } func (vpe validationPhaseError) Error() string { - msg := fmt.Sprintf("Error during %q phase of README validation:", vpe.Phase) + validationStrs := []string{} for _, e := range vpe.Errors { - msg += fmt.Sprintf("\n- %v", e) + validationStrs = append(validationStrs, fmt.Sprintf("- %v", e)) } + slices.Sort(validationStrs) + + msg := fmt.Sprintf("Error during %q phase of README validation:", vpe.Phase) + msg += strings.Join(validationStrs, "\n") msg += "\n" return msg @@ -96,124 +100,67 @@ func extractFrontmatter(readmeText string) (string, error) { return fm, nil } -// A validation function for verifying one specific aspect of a contributor's -// frontmatter content. Each function should be able to return ALL data -// violations that apply to the function's area of concern, rather than -// returning the first error found -type contributorValidationFunc = func(fm contributorFrontmatterWithFilePath) []error - -func validateContributorGithubUsername(fm contributorFrontmatterWithFilePath) []error { - problems := []error{} - - if fm.GithubUsername == "" { - problems = append( - problems, - fmt.Errorf( - "%q: missing GitHub username", - fm.FilePath, - ), - ) - return problems +func validateContributorGithubUsername(githubUsername string) error { + if githubUsername == "" { + return errors.New("missing GitHub username") } - lower := strings.ToLower(fm.GithubUsername) + lower := strings.ToLower(githubUsername) if uriSafe := url.PathEscape(lower); uriSafe != lower { - problems = append( - problems, - fmt.Errorf( - "%q: gitHub username %q is not a valid URL path segment", - fm.FilePath, - fm.GithubUsername, - ), - ) + return fmt.Errorf("gitHub username %q is not a valid URL path segment", githubUsername) } - return problems + return nil } -func validateContributorEmployerGithubUsername(fm contributorFrontmatterWithFilePath) []error { - if fm.EmployerGithubUsername == nil { +func validateContributorEmployerGithubUsername( + employerGithubUsername *string, + githubUsername string, +) []error { + if employerGithubUsername == nil { return nil } problems := []error{} - - if *fm.EmployerGithubUsername == "" { - problems = append( - problems, - fmt.Errorf( - "%q: company_github field is defined but has empty value", - fm.FilePath, - ), - ) + if *employerGithubUsername == "" { + problems = append(problems, errors.New("company_github field is defined but has empty value")) return problems } - lower := strings.ToLower(*fm.EmployerGithubUsername) + lower := strings.ToLower(*employerGithubUsername) if uriSafe := url.PathEscape(lower); uriSafe != lower { - problems = append( - problems, - fmt.Errorf( - "%q: gitHub company username %q is not a valid URL path segment", - fm.FilePath, - *fm.EmployerGithubUsername, - ), - ) - } - - if *fm.EmployerGithubUsername == fm.GithubUsername { - problems = append( - problems, - fmt.Errorf( - "%q: cannot list own GitHub name (%q) as employer", - fm.FilePath, - fm.GithubUsername, - ), - ) + problems = append(problems, fmt.Errorf("gitHub company username %q is not a valid URL path segment", *employerGithubUsername)) + } + + if *employerGithubUsername == githubUsername { + problems = append(problems, fmt.Errorf("cannot list own GitHub name (%q) as employer", githubUsername)) } return problems } -func validateContributorDisplayName(fm contributorFrontmatterWithFilePath) []error { - problems := []error{} - if fm.DisplayName == "" { - problems = append( - problems, - fmt.Errorf( - "%q: GitHub user %q is missing display name", - fm.FilePath, - fm.GithubUsername, - ), - ) +func validateContributorDisplayName(displayName string) error { + if displayName == "" { + return fmt.Errorf("missing display_name") } - return problems + return nil } -func validateContributorLinkedinURL(fm contributorFrontmatterWithFilePath) []error { - if fm.LinkedinURL == nil { +func validateContributorLinkedinURL(linkedinURL *string) error { + if linkedinURL == nil { return nil } - problems := []error{} - if _, err := url.ParseRequestURI(*fm.LinkedinURL); err != nil { - problems = append( - problems, - fmt.Errorf( - "%q: linkedIn URL %q is not valid: %v", - *fm.LinkedinURL, - fm.FilePath, - err, - ), - ) + if _, err := url.ParseRequestURI(*linkedinURL); err != nil { + return fmt.Errorf("linkedIn URL %q is not valid: %v", *linkedinURL, err) } - return problems + return nil } -func validateContributorEmail(fm contributorFrontmatterWithFilePath) []error { - if fm.SupportEmail == nil { +func validateContributorSupportEmail(email *string) []error { + if email == nil { return nil } @@ -223,202 +170,131 @@ func validateContributorEmail(fm contributorFrontmatterWithFilePath) []error { // an email, and especially with some contributors being individual // developers, we don't want to do that on every single run of the CI // pipeline. Best we can do is verify the general structure - username, server, ok := strings.Cut(*fm.SupportEmail, "@") + username, server, ok := strings.Cut(*email, "@") if !ok { - problems = append( - problems, - fmt.Errorf( - "%q: email address %q is missing @ symbol", - fm.FilePath, - *fm.LinkedinURL, - ), - ) + problems = append(problems, fmt.Errorf("email address %q is missing @ symbol", *email)) return problems } if username == "" { - problems = append( - problems, - fmt.Errorf( - "%q: email address %q is missing username", - fm.FilePath, - *fm.LinkedinURL, - ), - ) + problems = append(problems, fmt.Errorf("email address %q is missing username", *email)) } domain, tld, ok := strings.Cut(server, ".") if !ok { - problems = append( - problems, - fmt.Errorf( - "%q: email address %q is missing period for server segment", - fm.FilePath, - *fm.LinkedinURL, - ), - ) + problems = append(problems, fmt.Errorf("email address %q is missing period for server segment", *email)) return problems } if domain == "" { - problems = append( - problems, - fmt.Errorf( - "%q: email address %q is missing domain", - fm.FilePath, - *fm.LinkedinURL, - ), - ) + problems = append(problems, fmt.Errorf("email address %q is missing domain", *email)) } - if tld == "" { - problems = append( - problems, - fmt.Errorf( - "%q: email address %q is missing top-level domain", - fm.FilePath, - *fm.LinkedinURL, - ), - ) - } - - if strings.Contains(*fm.SupportEmail, "?") { - problems = append( - problems, - fmt.Errorf( - "%q: email is not allowed to contain search parameters", - fm.FilePath, - ), - ) + problems = append(problems, fmt.Errorf("email address %q is missing top-level domain", *email)) + } + if strings.Contains(*email, "?") { + problems = append(problems, errors.New("email is not allowed to contain search parameters")) } return problems } -func validateContributorWebsite(fm contributorFrontmatterWithFilePath) []error { - if fm.WebsiteURL == nil { +func validateContributorWebsite(websiteURL *string) error { + if websiteURL == nil { return nil } - problems := []error{} - if _, err := url.ParseRequestURI(*fm.WebsiteURL); err != nil { - problems = append( - problems, - fmt.Errorf( - "%q: LinkedIn URL %q is not valid: %v", - fm.FilePath, - *fm.WebsiteURL, - err, - ), - ) + if _, err := url.ParseRequestURI(*websiteURL); err != nil { + return fmt.Errorf("linkedIn URL %q is not valid: %v", *websiteURL, err) } - return problems + return nil } -func validateContributorStatus(fm contributorFrontmatterWithFilePath) []error { - if fm.ContributorStatus == nil { +func validateContributorStatus(status *string) error { + if status == nil { return nil } - problems := []error{} validStatuses := []string{"official", "partner", "community"} - if !slices.Contains(validStatuses, *fm.ContributorStatus) { - problems = append( - problems, - fmt.Errorf( - "%q: contributor status %q is not valid", - fm.FilePath, - *fm.ContributorStatus, - ), - ) + if !slices.Contains(validStatuses, *status) { + return fmt.Errorf("contributor status %q is not valid", *status) } - return problems + return nil } // Can't validate the image actually leads to a valid resource in a pure // function, but can at least catch obvious problems -func validateContributorAvatarURL(fm contributorFrontmatterWithFilePath) []error { - if fm.AvatarURL == nil { +func validateContributorAvatarURL(avatarURL *string) []error { + if avatarURL == nil { return nil } problems := []error{} - if *fm.AvatarURL == "" { - problems = append( - problems, - fmt.Errorf( - "%q: avatar URL must be omitted or non-empty string", - fm.FilePath, - ), - ) + if *avatarURL == "" { + problems = append(problems, errors.New("avatar URL must be omitted or non-empty string")) return problems } // Have to use .Parse instead of .ParseRequestURI because this is the // one field that's allowed to be a relative URL - if _, err := url.Parse(*fm.AvatarURL); err != nil { - problems = append( - problems, - fmt.Errorf( - "%q: URL %q is not a valid relative or absolute URL", - fm.FilePath, - *fm.AvatarURL, - ), - ) - } - - if strings.Contains(*fm.AvatarURL, "?") { - problems = append( - problems, - fmt.Errorf( - "%q: avatar URL is not allowed to contain search parameters", - fm.FilePath, - ), - ) + if _, err := url.Parse(*avatarURL); err != nil { + problems = append(problems, fmt.Errorf("URL %q is not a valid relative or absolute URL", *avatarURL)) + } + if strings.Contains(*avatarURL, "?") { + problems = append(problems, errors.New("avatar URL is not allowed to contain search parameters")) } supportedFileFormats := []string{".png", ".jpeg", ".jpg", ".gif", ".svg"} matched := false for _, ff := range supportedFileFormats { - matched = strings.HasSuffix(*fm.AvatarURL, ff) + matched = strings.HasSuffix(*avatarURL, ff) if matched { break } } if !matched { - segments := strings.Split(*fm.AvatarURL, ".") + segments := strings.Split(*avatarURL, ".") fileExtension := segments[len(segments)-1] - problems = append( - problems, - fmt.Errorf( - "%q: avatar URL '.%s' does not end in a supported file format: [%s]", - fm.FilePath, - fileExtension, - strings.Join(supportedFileFormats, ", "), - ), - ) + problems = append(problems, fmt.Errorf("avatar URL '.%s' does not end in a supported file format: [%s]", fileExtension, strings.Join(supportedFileFormats, ", "))) } return problems } func validateContributorYaml(yml contributorFrontmatterWithFilePath) []error { - validationFuncs := []contributorValidationFunc{ - validateContributorGithubUsername, - validateContributorEmployerGithubUsername, - validateContributorDisplayName, - validateContributorLinkedinURL, - validateContributorEmail, - validateContributorWebsite, - validateContributorStatus, - validateContributorAvatarURL, - } allProblems := []error{} - for _, fn := range validationFuncs { - allProblems = append(allProblems, fn(yml)...) + addFilePath := func(err error) error { + return fmt.Errorf("%q: %v", yml.FilePath, err) + } + + if err := validateContributorGithubUsername(yml.GithubUsername); err != nil { + allProblems = append(allProblems, addFilePath(err)) + } + if err := validateContributorDisplayName(yml.DisplayName); err != nil { + allProblems = append(allProblems, addFilePath(err)) + } + if err := validateContributorLinkedinURL(yml.LinkedinURL); err != nil { + allProblems = append(allProblems, addFilePath(err)) + } + if err := validateContributorWebsite(yml.WebsiteURL); err != nil { + allProblems = append(allProblems, addFilePath(err)) + } + if err := validateContributorStatus(yml.ContributorStatus); err != nil { + allProblems = append(allProblems, addFilePath(err)) + } + + for _, err := range validateContributorEmployerGithubUsername(yml.EmployerGithubUsername, yml.GithubUsername) { + allProblems = append(allProblems, addFilePath(err)) + } + for _, err := range validateContributorSupportEmail(yml.SupportEmail) { + allProblems = append(allProblems, addFilePath(err)) + } + for _, err := range validateContributorAvatarURL(yml.AvatarURL) { + allProblems = append(allProblems, addFilePath(err)) } + return allProblems } @@ -454,15 +330,7 @@ func parseContributorFiles(readmeEntries []readme) ( } if prev, isConflict := frontmatterByUsername[processed.GithubUsername]; isConflict { - yamlParsingErrors.Errors = append( - yamlParsingErrors.Errors, - fmt.Errorf( - "%q: GitHub name %s conflicts with field defined in %q", - processed.FilePath, - processed.GithubUsername, - prev.FilePath, - ), - ) + yamlParsingErrors.Errors = append(yamlParsingErrors.Errors, fmt.Errorf("%q: GitHub name %s conflicts with field defined in %q", processed.FilePath, processed.GithubUsername, prev.FilePath)) continue } @@ -497,15 +365,7 @@ func parseContributorFiles(readmeEntries []readme) ( if _, found := frontmatterByUsername[companyName]; found { continue } - yamlValidationErrors.Errors = append( - yamlValidationErrors.Errors, - fmt.Errorf( - "company %q does not exist in %q directory but is referenced by these profiles: [%s]", - companyName, - rootRegistryPath, - strings.Join(group, ", "), - ), - ) + yamlValidationErrors.Errors = append(yamlValidationErrors.Errors, fmt.Errorf("company %q does not exist in %q directory but is referenced by these profiles: [%s]", companyName, rootRegistryPath, strings.Join(group, ", "))) } if len(yamlValidationErrors.Errors) != 0 { return nil, yamlValidationErrors @@ -525,13 +385,7 @@ func aggregateContributorReadmeFiles() ([]readme, error) { for _, e := range dirEntries { dirPath := path.Join(rootRegistryPath, e.Name()) if !e.IsDir() { - problems = append( - problems, - fmt.Errorf( - "Detected non-directory file %q at base of main Registry directory", - dirPath, - ), - ) + problems = append(problems, fmt.Errorf("detected non-directory file %q at base of main Registry directory", dirPath)) continue } @@ -565,6 +419,8 @@ func validateRelativeUrls( problems := []error{} for _, con := range contributors { + // If the avatar URL is missing, we'll just assume that the Registry + // site build step will take care of filling in the data properly if con.AvatarURL == nil { continue } @@ -574,13 +430,7 @@ func validateRelativeUrls( } if strings.HasPrefix(*con.AvatarURL, "..") { - problems = append( - problems, - fmt.Errorf( - "%q: relative avatar URLs cannot be placed outside a user's namespaced directory", - con.FilePath, - ), - ) + problems = append(problems, fmt.Errorf("%q: relative avatar URLs cannot be placed outside a user's namespaced directory", con.FilePath)) continue } @@ -588,14 +438,7 @@ func validateRelativeUrls( *con.AvatarURL _, err := os.ReadFile(absolutePath) if err != nil { - problems = append( - problems, - fmt.Errorf( - "%q: relative avatar path %q does not point to image in file system", - con.FilePath, - *con.AvatarURL, - ), - ) + problems = append(problems, fmt.Errorf("%q: relative avatar path %q does not point to image in file system", con.FilePath, *con.AvatarURL)) } } From 5c45642e4be79777792fa5d1957b89a542b8eb2d Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Fri, 11 Apr 2025 17:19:19 +0000 Subject: [PATCH 25/30] refactor: split off another function --- scripts/validate-contributor-readmes/main.go | 73 ++++++++++---------- 1 file changed, 36 insertions(+), 37 deletions(-) diff --git a/scripts/validate-contributor-readmes/main.go b/scripts/validate-contributor-readmes/main.go index bab52089..13b7f35e 100644 --- a/scripts/validate-contributor-readmes/main.go +++ b/scripts/validate-contributor-readmes/main.go @@ -298,59 +298,55 @@ func validateContributorYaml(yml contributorFrontmatterWithFilePath) []error { return allProblems } +func parseContributor(rm readme) (contributorFrontmatterWithFilePath, error) { + fm, err := extractFrontmatter(rm.RawText) + if err != nil { + return contributorFrontmatterWithFilePath{}, fmt.Errorf("%q: failed to parse frontmatter: %v", rm.FilePath, err) + } + + yml := contributorProfileFrontmatter{} + if err := yaml.Unmarshal([]byte(fm), &yml); err != nil { + return contributorFrontmatterWithFilePath{}, fmt.Errorf("%q: failed to parse: %v", rm.FilePath, err) + } + + return contributorFrontmatterWithFilePath{ + FilePath: rm.FilePath, + contributorProfileFrontmatter: yml, + }, nil +} + func parseContributorFiles(readmeEntries []readme) ( map[string]contributorFrontmatterWithFilePath, error, ) { frontmatterByUsername := map[string]contributorFrontmatterWithFilePath{} - yamlParsingErrors := validationPhaseError{ - Phase: "YAML parsing", - } + yamlParsingErrors := []error{} for _, rm := range readmeEntries { - fm, err := extractFrontmatter(rm.RawText) + fm, err := parseContributor(rm) if err != nil { - yamlParsingErrors.Errors = append( - yamlParsingErrors.Errors, - fmt.Errorf("%q: failed to parse: %v", rm.FilePath, err), - ) + yamlParsingErrors = append(yamlParsingErrors, err) continue } - yml := contributorProfileFrontmatter{} - if err := yaml.Unmarshal([]byte(fm), &yml); err != nil { - yamlParsingErrors.Errors = append( - yamlParsingErrors.Errors, - fmt.Errorf("%q: failed to parse: %v", rm.FilePath, err), - ) + if prev, isConflict := frontmatterByUsername[fm.GithubUsername]; isConflict { + yamlParsingErrors = append(yamlParsingErrors, fmt.Errorf("%q: GitHub name %s conflicts with field defined in %q", fm.FilePath, fm.GithubUsername, prev.FilePath)) continue } - processed := contributorFrontmatterWithFilePath{ - FilePath: rm.FilePath, - contributorProfileFrontmatter: yml, - } - - if prev, isConflict := frontmatterByUsername[processed.GithubUsername]; isConflict { - yamlParsingErrors.Errors = append(yamlParsingErrors.Errors, fmt.Errorf("%q: GitHub name %s conflicts with field defined in %q", processed.FilePath, processed.GithubUsername, prev.FilePath)) - continue - } - - frontmatterByUsername[processed.GithubUsername] = processed + frontmatterByUsername[fm.GithubUsername] = fm } - if len(yamlParsingErrors.Errors) != 0 { - return nil, yamlParsingErrors + if len(yamlParsingErrors) != 0 { + return nil, validationPhaseError{ + Phase: "YAML parsing", + Errors: yamlParsingErrors, + } } employeeGithubGroups := map[string][]string{} - yamlValidationErrors := validationPhaseError{ - Phase: "Raw YAML Validation", - } + yamlValidationErrors := []error{} for _, yml := range frontmatterByUsername { errors := validateContributorYaml(yml) if len(errors) > 0 { - yamlValidationErrors.Errors = append( - yamlValidationErrors.Errors, - errors..., - ) + yamlValidationErrors = append(yamlValidationErrors, errors...) continue } @@ -365,10 +361,13 @@ func parseContributorFiles(readmeEntries []readme) ( if _, found := frontmatterByUsername[companyName]; found { continue } - yamlValidationErrors.Errors = append(yamlValidationErrors.Errors, fmt.Errorf("company %q does not exist in %q directory but is referenced by these profiles: [%s]", companyName, rootRegistryPath, strings.Join(group, ", "))) + yamlValidationErrors = append(yamlValidationErrors, fmt.Errorf("company %q does not exist in %q directory but is referenced by these profiles: [%s]", companyName, rootRegistryPath, strings.Join(group, ", "))) } - if len(yamlValidationErrors.Errors) != 0 { - return nil, yamlValidationErrors + if len(yamlValidationErrors) != 0 { + return nil, validationPhaseError{ + Phase: "Raw YAML Validation", + Errors: yamlValidationErrors, + } } return frontmatterByUsername, nil From ffd9861e03489b71732441c307f3b9764d324758 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Mon, 14 Apr 2025 13:37:17 +0000 Subject: [PATCH 26/30] refactor: extract pseudo-constants --- .gitignore | 3 ++ scripts/validate-contributor-readmes/main.go | 55 +++++++++++--------- 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index 1170717c..7d2bfb25 100644 --- a/.gitignore +++ b/.gitignore @@ -134,3 +134,6 @@ dist .yarn/build-state.yml .yarn/install-state.gz .pnp.* + +# Script output +/validate-contributor-readmes diff --git a/scripts/validate-contributor-readmes/main.go b/scripts/validate-contributor-readmes/main.go index 13b7f35e..dd496058 100644 --- a/scripts/validate-contributor-readmes/main.go +++ b/scripts/validate-contributor-readmes/main.go @@ -21,9 +21,14 @@ import ( const rootRegistryPath = "./registry" +var ( + validContributorStatuses = []string{"official", "partner", "community"} + supportedAvatarFileFormats = []string{".png", ".jpeg", ".jpg", ".gif", ".svg"} +) + type readme struct { - FilePath string - RawText string + filePath string + rawText string } type contributorProfileFrontmatter struct { @@ -48,18 +53,18 @@ type contributorFrontmatterWithFilePath struct { var _ error = validationPhaseError{} type validationPhaseError struct { - Phase string - Errors []error + phase string + errors []error } func (vpe validationPhaseError) Error() string { validationStrs := []string{} - for _, e := range vpe.Errors { + for _, e := range vpe.errors { validationStrs = append(validationStrs, fmt.Sprintf("- %v", e)) } slices.Sort(validationStrs) - msg := fmt.Sprintf("Error during %q phase of README validation:", vpe.Phase) + msg := fmt.Sprintf("Error during %q phase of README validation:", vpe.phase) msg += strings.Join(validationStrs, "\n") msg += "\n" @@ -193,7 +198,7 @@ func validateContributorSupportEmail(email *string) []error { problems = append(problems, fmt.Errorf("email address %q is missing top-level domain", *email)) } if strings.Contains(*email, "?") { - problems = append(problems, errors.New("email is not allowed to contain search parameters")) + problems = append(problems, errors.New("email is not allowed to contain query parameters")) } return problems @@ -216,8 +221,7 @@ func validateContributorStatus(status *string) error { return nil } - validStatuses := []string{"official", "partner", "community"} - if !slices.Contains(validStatuses, *status) { + if !slices.Contains(validContributorStatuses, *status) { return fmt.Errorf("contributor status %q is not valid", *status) } @@ -246,9 +250,8 @@ func validateContributorAvatarURL(avatarURL *string) []error { problems = append(problems, errors.New("avatar URL is not allowed to contain search parameters")) } - supportedFileFormats := []string{".png", ".jpeg", ".jpg", ".gif", ".svg"} matched := false - for _, ff := range supportedFileFormats { + for _, ff := range supportedAvatarFileFormats { matched = strings.HasSuffix(*avatarURL, ff) if matched { break @@ -257,7 +260,7 @@ func validateContributorAvatarURL(avatarURL *string) []error { if !matched { segments := strings.Split(*avatarURL, ".") fileExtension := segments[len(segments)-1] - problems = append(problems, fmt.Errorf("avatar URL '.%s' does not end in a supported file format: [%s]", fileExtension, strings.Join(supportedFileFormats, ", "))) + problems = append(problems, fmt.Errorf("avatar URL '.%s' does not end in a supported file format: [%s]", fileExtension, strings.Join(supportedAvatarFileFormats, ", "))) } return problems @@ -299,18 +302,18 @@ func validateContributorYaml(yml contributorFrontmatterWithFilePath) []error { } func parseContributor(rm readme) (contributorFrontmatterWithFilePath, error) { - fm, err := extractFrontmatter(rm.RawText) + fm, err := extractFrontmatter(rm.rawText) if err != nil { - return contributorFrontmatterWithFilePath{}, fmt.Errorf("%q: failed to parse frontmatter: %v", rm.FilePath, err) + return contributorFrontmatterWithFilePath{}, fmt.Errorf("%q: failed to parse frontmatter: %v", rm.filePath, err) } yml := contributorProfileFrontmatter{} if err := yaml.Unmarshal([]byte(fm), &yml); err != nil { - return contributorFrontmatterWithFilePath{}, fmt.Errorf("%q: failed to parse: %v", rm.FilePath, err) + return contributorFrontmatterWithFilePath{}, fmt.Errorf("%q: failed to parse: %v", rm.filePath, err) } return contributorFrontmatterWithFilePath{ - FilePath: rm.FilePath, + FilePath: rm.filePath, contributorProfileFrontmatter: yml, }, nil } @@ -336,8 +339,8 @@ func parseContributorFiles(readmeEntries []readme) ( } if len(yamlParsingErrors) != 0 { return nil, validationPhaseError{ - Phase: "YAML parsing", - Errors: yamlParsingErrors, + phase: "YAML parsing", + errors: yamlParsingErrors, } } @@ -365,8 +368,8 @@ func parseContributorFiles(readmeEntries []readme) ( } if len(yamlValidationErrors) != 0 { return nil, validationPhaseError{ - Phase: "Raw YAML Validation", - Errors: yamlValidationErrors, + phase: "Raw YAML Validation", + errors: yamlValidationErrors, } } @@ -395,15 +398,15 @@ func aggregateContributorReadmeFiles() ([]readme, error) { continue } allReadmeFiles = append(allReadmeFiles, readme{ - FilePath: readmePath, - RawText: string(rmBytes), + filePath: readmePath, + rawText: string(rmBytes), }) } if len(problems) != 0 { return nil, validationPhaseError{ - Phase: "FileSystem reading", - Errors: problems, + phase: "FileSystem reading", + errors: problems, } } @@ -445,8 +448,8 @@ func validateRelativeUrls( return nil } return validationPhaseError{ - Phase: "Relative URL validation", - Errors: problems, + phase: "Relative URL validation", + errors: problems, } } From bdf9c5f51b80f096600ae3c6b432b93351ec7877 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Mon, 14 Apr 2025 13:45:41 +0000 Subject: [PATCH 27/30] refactor: update namespacing --- scripts/validate-contributor-readmes/main.go | 87 ++++++++++---------- 1 file changed, 42 insertions(+), 45 deletions(-) diff --git a/scripts/validate-contributor-readmes/main.go b/scripts/validate-contributor-readmes/main.go index dd496058..d83a7c8b 100644 --- a/scripts/validate-contributor-readmes/main.go +++ b/scripts/validate-contributor-readmes/main.go @@ -45,9 +45,9 @@ type contributorProfileFrontmatter struct { ContributorStatus *string `yaml:"status"` } -type contributorFrontmatterWithFilePath struct { - contributorProfileFrontmatter - FilePath string +type contributorProfile struct { + frontmatter contributorProfileFrontmatter + filePath string } var _ error = validationPhaseError{} @@ -266,76 +266,73 @@ func validateContributorAvatarURL(avatarURL *string) []error { return problems } -func validateContributorYaml(yml contributorFrontmatterWithFilePath) []error { +func validateContributorYaml(yml contributorProfile) []error { allProblems := []error{} addFilePath := func(err error) error { - return fmt.Errorf("%q: %v", yml.FilePath, err) + return fmt.Errorf("%q: %v", yml.filePath, err) } - if err := validateContributorGithubUsername(yml.GithubUsername); err != nil { + if err := validateContributorGithubUsername(yml.frontmatter.GithubUsername); err != nil { allProblems = append(allProblems, addFilePath(err)) } - if err := validateContributorDisplayName(yml.DisplayName); err != nil { + if err := validateContributorDisplayName(yml.frontmatter.DisplayName); err != nil { allProblems = append(allProblems, addFilePath(err)) } - if err := validateContributorLinkedinURL(yml.LinkedinURL); err != nil { + if err := validateContributorLinkedinURL(yml.frontmatter.LinkedinURL); err != nil { allProblems = append(allProblems, addFilePath(err)) } - if err := validateContributorWebsite(yml.WebsiteURL); err != nil { + if err := validateContributorWebsite(yml.frontmatter.WebsiteURL); err != nil { allProblems = append(allProblems, addFilePath(err)) } - if err := validateContributorStatus(yml.ContributorStatus); err != nil { + if err := validateContributorStatus(yml.frontmatter.ContributorStatus); err != nil { allProblems = append(allProblems, addFilePath(err)) } - for _, err := range validateContributorEmployerGithubUsername(yml.EmployerGithubUsername, yml.GithubUsername) { + for _, err := range validateContributorEmployerGithubUsername(yml.frontmatter.EmployerGithubUsername, yml.frontmatter.GithubUsername) { allProblems = append(allProblems, addFilePath(err)) } - for _, err := range validateContributorSupportEmail(yml.SupportEmail) { + for _, err := range validateContributorSupportEmail(yml.frontmatter.SupportEmail) { allProblems = append(allProblems, addFilePath(err)) } - for _, err := range validateContributorAvatarURL(yml.AvatarURL) { + for _, err := range validateContributorAvatarURL(yml.frontmatter.AvatarURL) { allProblems = append(allProblems, addFilePath(err)) } return allProblems } -func parseContributor(rm readme) (contributorFrontmatterWithFilePath, error) { +func parseContributorProfile(rm readme) (contributorProfile, error) { fm, err := extractFrontmatter(rm.rawText) if err != nil { - return contributorFrontmatterWithFilePath{}, fmt.Errorf("%q: failed to parse frontmatter: %v", rm.filePath, err) + return contributorProfile{}, fmt.Errorf("%q: failed to parse frontmatter: %v", rm.filePath, err) } yml := contributorProfileFrontmatter{} if err := yaml.Unmarshal([]byte(fm), &yml); err != nil { - return contributorFrontmatterWithFilePath{}, fmt.Errorf("%q: failed to parse: %v", rm.filePath, err) + return contributorProfile{}, fmt.Errorf("%q: failed to parse: %v", rm.filePath, err) } - return contributorFrontmatterWithFilePath{ - FilePath: rm.filePath, - contributorProfileFrontmatter: yml, + return contributorProfile{ + filePath: rm.filePath, + frontmatter: yml, }, nil } -func parseContributorFiles(readmeEntries []readme) ( - map[string]contributorFrontmatterWithFilePath, - error, -) { - frontmatterByUsername := map[string]contributorFrontmatterWithFilePath{} +func parseContributorFiles(readmeEntries []readme) (map[string]contributorProfile, error) { + profilesByUsername := map[string]contributorProfile{} yamlParsingErrors := []error{} for _, rm := range readmeEntries { - fm, err := parseContributor(rm) + p, err := parseContributorProfile(rm) if err != nil { yamlParsingErrors = append(yamlParsingErrors, err) continue } - if prev, isConflict := frontmatterByUsername[fm.GithubUsername]; isConflict { - yamlParsingErrors = append(yamlParsingErrors, fmt.Errorf("%q: GitHub name %s conflicts with field defined in %q", fm.FilePath, fm.GithubUsername, prev.FilePath)) + if prev, alreadyExists := profilesByUsername[p.frontmatter.GithubUsername]; alreadyExists { + yamlParsingErrors = append(yamlParsingErrors, fmt.Errorf("%q: GitHub name %s conflicts with field defined in %q", p.filePath, p.frontmatter.GithubUsername, prev.filePath)) continue } - frontmatterByUsername[fm.GithubUsername] = fm + profilesByUsername[p.frontmatter.GithubUsername] = p } if len(yamlParsingErrors) != 0 { return nil, validationPhaseError{ @@ -346,22 +343,22 @@ func parseContributorFiles(readmeEntries []readme) ( employeeGithubGroups := map[string][]string{} yamlValidationErrors := []error{} - for _, yml := range frontmatterByUsername { - errors := validateContributorYaml(yml) + for _, p := range profilesByUsername { + errors := validateContributorYaml(p) if len(errors) > 0 { yamlValidationErrors = append(yamlValidationErrors, errors...) continue } - if yml.EmployerGithubUsername != nil { - employeeGithubGroups[*yml.EmployerGithubUsername] = append( - employeeGithubGroups[*yml.EmployerGithubUsername], - yml.GithubUsername, + if p.frontmatter.EmployerGithubUsername != nil { + employeeGithubGroups[*p.frontmatter.EmployerGithubUsername] = append( + employeeGithubGroups[*p.frontmatter.EmployerGithubUsername], + p.frontmatter.GithubUsername, ) } } for companyName, group := range employeeGithubGroups { - if _, found := frontmatterByUsername[companyName]; found { + if _, found := profilesByUsername[companyName]; found { continue } yamlValidationErrors = append(yamlValidationErrors, fmt.Errorf("company %q does not exist in %q directory but is referenced by these profiles: [%s]", companyName, rootRegistryPath, strings.Join(group, ", "))) @@ -373,7 +370,7 @@ func parseContributorFiles(readmeEntries []readme) ( } } - return frontmatterByUsername, nil + return profilesByUsername, nil } func aggregateContributorReadmeFiles() ([]readme, error) { @@ -414,7 +411,7 @@ func aggregateContributorReadmeFiles() ([]readme, error) { } func validateRelativeUrls( - contributors map[string]contributorFrontmatterWithFilePath, + contributors map[string]contributorProfile, ) error { // This function only validates relative avatar URLs for now, but it can be // beefed up to validate more in the future @@ -423,24 +420,24 @@ func validateRelativeUrls( for _, con := range contributors { // If the avatar URL is missing, we'll just assume that the Registry // site build step will take care of filling in the data properly - if con.AvatarURL == nil { + if con.frontmatter.AvatarURL == nil { continue } - if isRelativeURL := strings.HasPrefix(*con.AvatarURL, ".") || - strings.HasPrefix(*con.AvatarURL, "/"); !isRelativeURL { + if isRelativeURL := strings.HasPrefix(*con.frontmatter.AvatarURL, ".") || + strings.HasPrefix(*con.frontmatter.AvatarURL, "/"); !isRelativeURL { continue } - if strings.HasPrefix(*con.AvatarURL, "..") { - problems = append(problems, fmt.Errorf("%q: relative avatar URLs cannot be placed outside a user's namespaced directory", con.FilePath)) + if strings.HasPrefix(*con.frontmatter.AvatarURL, "..") { + problems = append(problems, fmt.Errorf("%q: relative avatar URLs cannot be placed outside a user's namespaced directory", con.filePath)) continue } - absolutePath := strings.TrimSuffix(con.FilePath, "README.md") + - *con.AvatarURL + absolutePath := strings.TrimSuffix(con.filePath, "README.md") + + *con.frontmatter.AvatarURL _, err := os.ReadFile(absolutePath) if err != nil { - problems = append(problems, fmt.Errorf("%q: relative avatar path %q does not point to image in file system", con.FilePath, *con.AvatarURL)) + problems = append(problems, fmt.Errorf("%q: relative avatar path %q does not point to image in file system", con.filePath, *con.frontmatter.AvatarURL)) } } From 39b264a7f979d4df90b13f2a9681611031d77313 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Mon, 14 Apr 2025 13:48:19 +0000 Subject: [PATCH 28/30] refactor: split up package boundaries --- .../main.go => contributors/contributors.go} | 35 ----------------- scripts/contributors/main.go | 39 +++++++++++++++++++ 2 files changed, 39 insertions(+), 35 deletions(-) rename scripts/{validate-contributor-readmes/main.go => contributors/contributors.go} (92%) create mode 100644 scripts/contributors/main.go diff --git a/scripts/validate-contributor-readmes/main.go b/scripts/contributors/contributors.go similarity index 92% rename from scripts/validate-contributor-readmes/main.go rename to scripts/contributors/contributors.go index d83a7c8b..671d3710 100644 --- a/scripts/validate-contributor-readmes/main.go +++ b/scripts/contributors/contributors.go @@ -1,15 +1,9 @@ -// This package is for validating all contributors within the main Registry -// directory. It validates that it has nothing but sub-directories, and that -// each sub-directory has a README.md file. Each of those files must then -// describe a specific contributor. The contents of these files will be parsed -// by the Registry site build step, to be displayed in the Registry site's UI. package main import ( "bufio" "errors" "fmt" - "log" "net/url" "os" "path" @@ -449,32 +443,3 @@ func validateRelativeUrls( errors: problems, } } - -func main() { - log.Println("Starting README validation") - allReadmeFiles, err := aggregateContributorReadmeFiles() - if err != nil { - log.Panic(err) - } - - log.Printf("Processing %d README files\n", len(allReadmeFiles)) - contributors, err := parseContributorFiles(allReadmeFiles) - if err != nil { - log.Panic(err) - } - log.Printf( - "Processed %d README files as valid contributor profiles", - len(contributors), - ) - - err = validateRelativeUrls(contributors) - if err != nil { - log.Panic(err) - } - log.Println("All relative URLs for READMEs are valid") - - log.Printf( - "Processed all READMEs in the %q directory\n", - rootRegistryPath, - ) -} diff --git a/scripts/contributors/main.go b/scripts/contributors/main.go new file mode 100644 index 00000000..90913185 --- /dev/null +++ b/scripts/contributors/main.go @@ -0,0 +1,39 @@ +// This package is for validating all contributors within the main Registry +// directory. It validates that it has nothing but sub-directories, and that +// each sub-directory has a README.md file. Each of those files must then +// describe a specific contributor. The contents of these files will be parsed +// by the Registry site build step, to be displayed in the Registry site's UI. +package main + +import ( + "log" +) + +func main() { + log.Println("Starting README validation") + allReadmeFiles, err := aggregateContributorReadmeFiles() + if err != nil { + log.Panic(err) + } + + log.Printf("Processing %d README files\n", len(allReadmeFiles)) + contributors, err := parseContributorFiles(allReadmeFiles) + log.Printf( + "Processed %d README files as valid contributor profiles", + len(contributors), + ) + if err != nil { + log.Panic(err) + } + + err = validateRelativeUrls(contributors) + if err != nil { + log.Panic(err) + } + log.Println("All relative URLs for READMEs are valid") + + log.Printf( + "Processed all READMEs in the %q directory\n", + rootRegistryPath, + ) +} From a2c246ea06dad0cb7153e6b885bb93cdcf17b752 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Mon, 14 Apr 2025 13:52:40 +0000 Subject: [PATCH 29/30] refactor: split out error func --- scripts/contributors/contributors.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/scripts/contributors/contributors.go b/scripts/contributors/contributors.go index 671d3710..02823f26 100644 --- a/scripts/contributors/contributors.go +++ b/scripts/contributors/contributors.go @@ -260,36 +260,37 @@ func validateContributorAvatarURL(avatarURL *string) []error { return problems } +func addFilePathToError(filePath string, err error) error { + return fmt.Errorf("%q: %v", filePath, err) +} + func validateContributorYaml(yml contributorProfile) []error { allProblems := []error{} - addFilePath := func(err error) error { - return fmt.Errorf("%q: %v", yml.filePath, err) - } if err := validateContributorGithubUsername(yml.frontmatter.GithubUsername); err != nil { - allProblems = append(allProblems, addFilePath(err)) + allProblems = append(allProblems, addFilePathToError(yml.filePath, err)) } if err := validateContributorDisplayName(yml.frontmatter.DisplayName); err != nil { - allProblems = append(allProblems, addFilePath(err)) + allProblems = append(allProblems, addFilePathToError(yml.filePath, err)) } if err := validateContributorLinkedinURL(yml.frontmatter.LinkedinURL); err != nil { - allProblems = append(allProblems, addFilePath(err)) + allProblems = append(allProblems, addFilePathToError(yml.filePath, err)) } if err := validateContributorWebsite(yml.frontmatter.WebsiteURL); err != nil { - allProblems = append(allProblems, addFilePath(err)) + allProblems = append(allProblems, addFilePathToError(yml.filePath, err)) } if err := validateContributorStatus(yml.frontmatter.ContributorStatus); err != nil { - allProblems = append(allProblems, addFilePath(err)) + allProblems = append(allProblems, addFilePathToError(yml.filePath, err)) } for _, err := range validateContributorEmployerGithubUsername(yml.frontmatter.EmployerGithubUsername, yml.frontmatter.GithubUsername) { - allProblems = append(allProblems, addFilePath(err)) + allProblems = append(allProblems, addFilePathToError(yml.filePath, err)) } for _, err := range validateContributorSupportEmail(yml.frontmatter.SupportEmail) { - allProblems = append(allProblems, addFilePath(err)) + allProblems = append(allProblems, addFilePathToError(yml.filePath, err)) } for _, err := range validateContributorAvatarURL(yml.frontmatter.AvatarURL) { - allProblems = append(allProblems, addFilePath(err)) + allProblems = append(allProblems, addFilePathToError(yml.filePath, err)) } return allProblems From 50d651c2f96f615c4f38d5b504523fdaf602782c Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Mon, 14 Apr 2025 13:57:02 +0000 Subject: [PATCH 30/30] fix: update CI step --- .github/workflows/ci.yaml | 6 ++++-- .gitignore | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4f9c9f95..c27a1529 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,5 +16,7 @@ jobs: uses: actions/setup-go@v5 with: go-version: "1.23.2" - - name: Validate - run: go run ./scripts/validate-contributor-readmes/main.go + - name: Validate contributors + run: go build ./scripts/contributors && ./contributors + - name: Remove build file artifact + run: rm ./contributors diff --git a/.gitignore b/.gitignore index 7d2bfb25..5f109fd9 100644 --- a/.gitignore +++ b/.gitignore @@ -136,4 +136,4 @@ dist .pnp.* # Script output -/validate-contributor-readmes +/contributors