Add rules to check loading of URLs in package index

internal/rule/ruleconfiguration/ruleconfiguration.go

@@ -2921,4 +2921,84 @@ var configurations = []Type{
 		ErrorModes:       []rulemode.Type{rulemode.Default},
 		RuleFunction:     rulefunction.PackageIndexFormat,
 	},
+	{
+		ProjectType:      projecttype.PackageIndex,
+		SuperprojectType: projecttype.All,
+		Category:         "data",
+		Subcategory:      "package",
+		ID:               "ID003",
+		Brief:            "dead packages[].websiteURL",
+		Description:      "",
+		MessageTemplate:  "Unable to load the packages[].websiteURL URL for package(s): {{.}}",
+		DisableModes:     nil,
+		EnableModes:      []rulemode.Type{rulemode.Default},
+		InfoModes:        nil,
+		WarningModes:     []rulemode.Type{rulemode.Permissive},
+		ErrorModes:       []rulemode.Type{rulemode.Default},
+		RuleFunction:     rulefunction.PackageIndexPackagesWebsiteURLDeadLink,
+	},
+	{
+		ProjectType:      projecttype.PackageIndex,
+		SuperprojectType: projecttype.All,
+		Category:         "data",
+		Subcategory:      "package",
+		ID:               "ID004",
+		Brief:            "dead packages[].help.online",
+		Description:      "",
+		MessageTemplate:  "Unable to load the packages[].help.online URL for package(s): {{.}}",
+		DisableModes:     nil,
+		EnableModes:      []rulemode.Type{rulemode.Default},
+		InfoModes:        nil,
+		WarningModes:     []rulemode.Type{rulemode.Permissive},
+		ErrorModes:       []rulemode.Type{rulemode.Default},
+		RuleFunction:     rulefunction.PackageIndexPackagesHelpOnlineDeadLink,
+	},
+	{
+		ProjectType:      projecttype.PackageIndex,
+		SuperprojectType: projecttype.All,
+		Category:         "data",
+		Subcategory:      "platform",
+		ID:               "ID005",
+		Brief:            "dead packages[].platforms[].help.online",
+		Description:      "",
+		MessageTemplate:  "Unable to load the packages[].platforms[].help.online URL for platforms(s): {{.}}",
+		DisableModes:     nil,
+		EnableModes:      []rulemode.Type{rulemode.Default},
+		InfoModes:        nil,
+		WarningModes:     []rulemode.Type{rulemode.Permissive},
+		ErrorModes:       []rulemode.Type{rulemode.Default},
+		RuleFunction:     rulefunction.PackageIndexPackagesPlatformsHelpOnlineDeadLink,
+	},
+	{
+		ProjectType:      projecttype.PackageIndex,
+		SuperprojectType: projecttype.All,
+		Category:         "data",
+		Subcategory:      "platform",
+		ID:               "ID006",
+		Brief:            "dead packages[].platforms[].url",
+		Description:      "",
+		MessageTemplate:  "Unable to load the packages[].platforms[].url URL for platforms(s): {{.}}",
+		DisableModes:     nil,
+		EnableModes:      []rulemode.Type{rulemode.Default},
+		InfoModes:        nil,
+		WarningModes:     []rulemode.Type{rulemode.Permissive},
+		ErrorModes:       []rulemode.Type{rulemode.Default},
+		RuleFunction:     rulefunction.PackageIndexPackagesPlatformsURLDeadLink,
+	},
+	{
+		ProjectType:      projecttype.PackageIndex,
+		SuperprojectType: projecttype.All,
+		Category:         "data",
+		Subcategory:      "tool",
+		ID:               "ID007",
+		Brief:            "dead packages[].tools[].systems[].url",
+		Description:      "",
+		MessageTemplate:  "Unable to load the packages[].tools[].systems[].url URL for tools(s): {{.}}",
+		DisableModes:     nil,
+		EnableModes:      []rulemode.Type{rulemode.Default},
+		InfoModes:        nil,
+		WarningModes:     []rulemode.Type{rulemode.Permissive},
+		ErrorModes:       []rulemode.Type{rulemode.Default},
+		RuleFunction:     rulefunction.PackageIndexPackagesToolsSystemsURLDeadLink,
+	},
 }

internal/rule/rulefunction/library.go

@@ -19,7 +19,6 @@ package rulefunction
 
 import (
 	"fmt"
-	"net/http"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -1010,17 +1009,12 @@ func LibraryPropertiesUrlFieldDeadLink() (result ruleresult.Type, output string)
 		return ruleresult.NotRun, "Field not present"
 	}
 
-	logrus.Tracef("Checking URL: %s", url)
-	httpResponse, err := http.Head(url)
+	err := checkURL(url)
 	if err != nil {
 		return ruleresult.Fail, err.Error()
 	}
 
-	if httpResponse.StatusCode == http.StatusOK {
-		return ruleresult.Pass, ""
-	}
-
-	return ruleresult.Fail, httpResponse.Status
+	return ruleresult.Pass, ""
 }
 
 // LibraryPropertiesArchitecturesFieldMissing checks for missing library.properties "architectures" field.

internal/rule/rulefunction/packageindex.go

@@ -16,6 +16,8 @@
 package rulefunction
 
 import (
+	"strings"
+
 	"github.com/arduino/arduino-lint/internal/project/packageindex"
 	"github.com/arduino/arduino-lint/internal/project/projectdata"
 	"github.com/arduino/arduino-lint/internal/rule/ruleresult"
@@ -83,3 +85,168 @@ func PackageIndexFormat() (result ruleresult.Type, output string) {
 
 	return ruleresult.Pass, ""
 }
+
+// PackageIndexPackagesWebsiteURLDeadLink checks for dead links in packages[].websiteURL.
+func PackageIndexPackagesWebsiteURLDeadLink() (result ruleresult.Type, output string) {
+	if projectdata.PackageIndexLoadError() != nil {
+		return ruleresult.NotRun, "Error loading package index"
+	}
+
+	nonCompliantIDs := []string{}
+	for _, data := range projectdata.PackageIndexPackages() {
+		url, ok := data.Object["websiteURL"].(string)
+		if !ok {
+			continue
+		}
+
+		if url == "" {
+			continue
+		}
+
+		if checkURL(url) == nil {
+			continue
+		}
+
+		nonCompliantIDs = append(nonCompliantIDs, data.ID)
+	}
+
+	if len(nonCompliantIDs) > 0 {
+		return ruleresult.Fail, strings.Join(nonCompliantIDs, ", ")
+	}
+
+	return ruleresult.Pass, ""
+}
+
+// PackageIndexPackagesHelpOnlineDeadLink checks for dead links in packages[].help.online.
+func PackageIndexPackagesHelpOnlineDeadLink() (result ruleresult.Type, output string) {
+	if projectdata.PackageIndexLoadError() != nil {
+		return ruleresult.NotRun, "Error loading package index"
+	}
+
+	nonCompliantIDs := []string{}
+	for _, data := range projectdata.PackageIndexPackages() {
+		help, ok := data.Object["help"].(map[string]interface{})
+		if !ok {
+			continue
+		}
+
+		url, ok := help["online"].(string)
+		if !ok {
+			continue
+		}
+
+		if url == "" {
+			continue
+		}
+
+		if checkURL(url) == nil {
+			continue
+		}
+
+		nonCompliantIDs = append(nonCompliantIDs, data.ID)
+	}
+
+	if len(nonCompliantIDs) > 0 {
+		return ruleresult.Fail, strings.Join(nonCompliantIDs, ", ")
+	}
+
+	return ruleresult.Pass, ""
+}
+
+// PackageIndexPackagesPlatformsHelpOnlineDeadLink checks for dead links in packages[].platforms[].help.online.
+func PackageIndexPackagesPlatformsHelpOnlineDeadLink() (result ruleresult.Type, output string) {
+	if projectdata.PackageIndexLoadError() != nil {
+		return ruleresult.NotRun, "Error loading package index"
+	}
+
+	nonCompliantIDs := []string{}
+	for _, data := range projectdata.PackageIndexPlatforms() {
+		help, ok := data.Object["help"].(map[string]interface{})
+		if !ok {
+			continue
+		}
+
+		url, ok := help["online"].(string)
+		if !ok {
+			continue
+		}
+
+		if url == "" {
+			continue
+		}
+
+		if checkURL(url) == nil {
+			continue
+		}
+
+		nonCompliantIDs = append(nonCompliantIDs, data.ID)
+	}
+
+	if len(nonCompliantIDs) > 0 {
+		return ruleresult.Fail, strings.Join(nonCompliantIDs, ", ")
+	}
+
+	return ruleresult.Pass, ""
+}
+
+// PackageIndexPackagesPlatformsURLDeadLink checks for dead links in packages[].platforms[].url.
+func PackageIndexPackagesPlatformsURLDeadLink() (result ruleresult.Type, output string) {
+	if projectdata.PackageIndexLoadError() != nil {
+		return ruleresult.NotRun, "Error loading package index"
+	}
+
+	nonCompliantIDs := []string{}
+	for _, data := range projectdata.PackageIndexPlatforms() {
+		url, ok := data.Object["url"].(string)
+		if !ok {
+			continue
+		}
+
+		if url == "" {
+			continue
+		}
+
+		if checkURL(url) == nil {
+			continue
+		}
+
+		nonCompliantIDs = append(nonCompliantIDs, data.ID)
+	}
+
+	if len(nonCompliantIDs) > 0 {
+		return ruleresult.Fail, strings.Join(nonCompliantIDs, ", ")
+	}
+
+	return ruleresult.Pass, ""
+}
+
+// PackageIndexPackagesToolsSystemsURLDeadLink checks for dead links in packages[].tools[].systems[].url.
+func PackageIndexPackagesToolsSystemsURLDeadLink() (result ruleresult.Type, output string) {
+	if projectdata.PackageIndexLoadError() != nil {
+		return ruleresult.NotRun, "Error loading package index"
+	}
+
+	nonCompliantIDs := []string{}
+	for _, data := range projectdata.PackageIndexSystems() {
+		url, ok := data.Object["url"].(string)
+		if !ok {
+			continue
+		}
+
+		if url == "" {
+			continue
+		}
+
+		if checkURL(url) == nil {
+			continue
+		}
+
+		nonCompliantIDs = append(nonCompliantIDs, data.ID)
+	}
+
+	if len(nonCompliantIDs) > 0 {
+		return ruleresult.Fail, strings.Join(nonCompliantIDs, ", ")
+	}
+
+	return ruleresult.Pass, ""
+}

internal/rule/rulefunction/packageindex_test.go

@@ -110,3 +110,54 @@ func TestPackageIndexFormat(t *testing.T) {
 
 	checkPackageIndexRuleFunction(PackageIndexFormat, testTables, t)
 }
+
+func TestPackageIndexPackagesWebsiteURLDeadLink(t *testing.T) {
+	testTables := []packageIndexRuleFunctionTestTable{
+		{"Invalid JSON", "invalid-JSON", ruleresult.NotRun, ""},
+		{"Dead URLs", "packages-websiteurl-dead", ruleresult.Fail, "^myboard1, myboard2$"},
+		{"Invalid URL", "packages-websiteurl-invalid", ruleresult.Fail, "^myboard$"},
+		{"Valid URL", "valid-package-index", ruleresult.Pass, ""},
+	}
+
+	checkPackageIndexRuleFunction(PackageIndexPackagesWebsiteURLDeadLink, testTables, t)
+}
+
+func TestPackageIndexPackagesHelpOnlineDeadLink(t *testing.T) {
+	testTables := []packageIndexRuleFunctionTestTable{
+		{"Invalid JSON", "invalid-JSON", ruleresult.NotRun, ""},
+		{"Dead URLs", "packages-help-online-dead", ruleresult.Fail, "^myboard1, myboard2$"},
+		{"Valid URL", "valid-package-index", ruleresult.Pass, ""},
+	}
+
+	checkPackageIndexRuleFunction(PackageIndexPackagesHelpOnlineDeadLink, testTables, t)
+}
+
+func TestPackageIndexPackagesPlatformsHelpOnlineDeadLink(t *testing.T) {
+	testTables := []packageIndexRuleFunctionTestTable{
+		{"Invalid JSON", "invalid-JSON", ruleresult.NotRun, ""},
+		{"Dead URLs", "packages-platforms-help-online-dead", ruleresult.Fail, "^myboard:avr@1\\.0\\.0, myboard:samd@1\\.0\\.0$"},
+		{"Valid URL", "valid-package-index", ruleresult.Pass, ""},
+	}
+
+	checkPackageIndexRuleFunction(PackageIndexPackagesPlatformsHelpOnlineDeadLink, testTables, t)
+}
+
+func TestPackageIndexPackagesPlatformsURLDeadLink(t *testing.T) {
+	testTables := []packageIndexRuleFunctionTestTable{
+		{"Invalid JSON", "invalid-JSON", ruleresult.NotRun, ""},
+		{"Dead URLs", "packages-platforms-url-dead", ruleresult.Fail, "^myboard:avr@1\\.0\\.0, myboard:samd@1\\.0\\.0$"},
+		{"Valid URL", "valid-package-index", ruleresult.Pass, ""},
+	}
+
+	checkPackageIndexRuleFunction(PackageIndexPackagesPlatformsURLDeadLink, testTables, t)
+}
+
+func TestPackageIndexPackagesToolsSystemsURLDeadLink(t *testing.T) {
+	testTables := []packageIndexRuleFunctionTestTable{
+		{"Invalid JSON", "invalid-JSON", ruleresult.NotRun, ""},
+		{"Dead URLs", "packages-tools-systems-url-dead", ruleresult.Fail, "^myboard:CMSIS@4\\.0\\.0-atmel - arm-linux-gnueabihf, myboard:CMSIS@4\\.0\\.0-atmel - i686-mingw32$"},
+		{"Valid URL", "valid-package-index", ruleresult.Pass, ""},
+	}
+
+	checkPackageIndexRuleFunction(PackageIndexPackagesToolsSystemsURLDeadLink, testTables, t)
+}

internal/rule/rulefunction/rulefunction.go

@@ -19,13 +19,15 @@ package rulefunction
 import (
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"regexp"
 	"strings"
 
 	"github.com/arduino/arduino-lint/internal/project/projectdata"
 	"github.com/arduino/arduino-lint/internal/project/sketch"
 	"github.com/arduino/arduino-lint/internal/rule/ruleresult"
 	"github.com/arduino/go-paths-helper"
+	"github.com/sirupsen/logrus"
 )
 
 // Type is the function signature for the rule functions.
@@ -156,3 +158,17 @@ func isValidJSON(path *paths.Path) bool {
 	}
 	return json.Valid(data)
 }
+
+func checkURL(url string) error {
+	logrus.Tracef("Checking URL: %s", url)
+	response, err := http.Head(url)
+	if err != nil {
+		return err
+	}
+
+	if response.StatusCode != http.StatusOK {
+		return fmt.Errorf("%s", response.Status)
+	}
+
+	return nil
+}