Add github feed for releases

pkg/config/scheduledfeed.go

@@ -17,6 +17,7 @@ import (
 	"github.com/ossf/package-feeds/pkg/events"
 	"github.com/ossf/package-feeds/pkg/feeds"
 	"github.com/ossf/package-feeds/pkg/feeds/crates"
+	"github.com/ossf/package-feeds/pkg/feeds/github"
 	"github.com/ossf/package-feeds/pkg/feeds/goproxy"
 	"github.com/ossf/package-feeds/pkg/feeds/npm"
 	"github.com/ossf/package-feeds/pkg/feeds/nuget"
@@ -161,6 +162,8 @@ func (fc FeedConfig) ToFeed(eventHandler *events.Handler) (feeds.ScheduledFeed,
 	switch fc.Type {
 	case crates.FeedName:
 		return crates.New(fc.Options, eventHandler)
+	case github.FeedName:
+		return github.New(fc.Options)
 	case goproxy.FeedName:
 		return goproxy.New(fc.Options)
 	case npm.FeedName:

pkg/feeds/feed.go

@@ -3,6 +3,7 @@ package feeds
 import (
 	"errors"
 	"fmt"
+	"sort"
 	"time"
 )
 
@@ -72,3 +73,10 @@ func ApplyCutoff(pkgs []*Package, cutoff time.Time) []*Package {
 func (err UnsupportedOptionError) Error() string {
 	return fmt.Sprintf("unsupported option `%v` supplied to %v feed", err.Option, err.Feed)
 }
+
+func SortPackages(pkgs []*Package) {
+	// Ensure packages are sorted by CreatedDate in order of most recent
+	sort.SliceStable(pkgs, func(i, j int) bool {
+		return pkgs[j].CreatedDate.Before(pkgs[i].CreatedDate)
+	})
+}

pkg/feeds/github/README.md

@@ -0,0 +1,15 @@
+# Github Feed
+
+This feed allows polling of releases from specific github repositories. This feed **requires** a list of repositories to be specified.
+
+## Configuration options
+
+The `packages` field is required by the github feed.
+
+```
+feeds:
+- type: github
+  options:
+    packages:
+    - "ossf/package-feeds"
+```

pkg/feeds/github/github.go

@@ -0,0 +1,117 @@
+package github
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/ossf/package-feeds/pkg/feeds"
+)
+
+const (
+	FeedName = "github"
+)
+
+var (
+	urlFormatString = "https://api.github.com/repos/%v/releases?per_page=%v"
+	httpClient      = &http.Client{
+		Timeout: 10 * time.Second,
+	}
+	releasesPerQuery           = 20
+	errPackageOptionsUnset     = errors.New("github feed requires packages to be configured as a feed option")
+	errMinimumPackagesRequired = errors.New("github feed requires a minimum of 1 package supplied under options")
+)
+
+type Feed struct {
+	repositories []string
+	options      feeds.FeedOptions
+}
+
+func New(feedOptions feeds.FeedOptions) (*Feed, error) {
+	if feedOptions.Packages == nil {
+		return nil, errPackageOptionsUnset
+	}
+	if len(*feedOptions.Packages) == 0 {
+		return nil, errMinimumPackagesRequired
+	}
+	return &Feed{
+		repositories: *feedOptions.Packages,
+		options:      feedOptions,
+	}, nil
+}
+
+func (feed Feed) GetFeedOptions() feeds.FeedOptions {
+	return feed.options
+}
+
+func (feed Feed) GetName() string {
+	return "github"
+}
+
+type releaseList []*release
+
+type release struct {
+	TagName string `json:"tag_name"`
+
+	// TODO: Add optional filter of Draft/Prerelease
+	Draft      bool `json:"draft"`
+	Prerelease bool `json:"prerelease"`
+
+	PublishedAt time.Time `json:"published_at"`
+}
+
+func fetchReleases(repository string) (releaseList, error) {
+	releases := releaseList{}
+	resp, err := httpClient.Get(fmt.Sprintf(urlFormatString, repository, releasesPerQuery))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	err = json.NewDecoder(resp.Body).Decode(&releases)
+	if err != nil {
+		return nil, err
+	}
+	return releases, nil
+}
+
+func (feed Feed) Latest(cutoff time.Time) ([]*feeds.Package, []error) {
+	pkgs := []*feeds.Package{}
+	pkgChannel := make(chan []*feeds.Package)
+	errs := []error{}
+	errChannel := make(chan error)
+
+	for _, repo := range feed.repositories {
+		go func(repo string) {
+			repoReleases, err := fetchReleases(repo)
+			if err != nil {
+				errChannel <- err
+				return
+			}
+			pkgChannel <- releasesToPackages(repo, repoReleases)
+		}(repo)
+	}
+
+	for i := 0; i < len(feed.repositories); i++ {
+		select {
+		case pkgSlice := <-pkgChannel:
+			pkgs = append(pkgs, pkgSlice...)
+		case err := <-errChannel:
+			errs = append(errs, err)
+		}
+	}
+	// Sort packages to ensure deterministic ordering.
+	feeds.SortPackages(pkgs)
+
+	pkgs = feeds.ApplyCutoff(pkgs, cutoff)
+	return pkgs, errs
+}
+
+func releasesToPackages(repo string, releases releaseList) []*feeds.Package {
+	pkgs := []*feeds.Package{}
+	for _, release := range releases {
+		pkgs = append(pkgs, feeds.NewPackage(release.PublishedAt, repo, release.TagName, FeedName))
+	}
+	return pkgs
+}

pkg/feeds/github/github_test.go

@@ -0,0 +1,127 @@
+package github
+
+import (
+	"errors"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/ossf/package-feeds/pkg/feeds"
+	testutils "github.com/ossf/package-feeds/pkg/utils/test"
+)
+
+func TestGithubLatest(t *testing.T) {
+	t.Parallel()
+
+	handlers := map[string]testutils.HTTPHandlerFunc{
+		"/repos/fooOrg/bar/releases": barResponse,
+		"/repos/fooOrg/baz/releases": bazResponse,
+	}
+	srv := testutils.HTTPServerMock(handlers)
+
+	urlFormatString = srv.URL + "/repos/%v/releases?per_page=%v"
+
+	packages := []string{
+		"fooOrg/bar",
+		"fooOrg/baz",
+	}
+
+	cutoff := time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)
+	feed, err := New(feeds.FeedOptions{Packages: &packages})
+	if err != nil {
+		t.Fatalf("Unexpected error during feed creation: %v", err)
+	}
+
+	pkgs, errs := feed.Latest(cutoff)
+	if len(errs) > 0 {
+		t.Fatalf("Failed to poll latest packages from feed: %v", errs[0])
+	}
+	if len(pkgs) != 6 {
+		t.Fatalf("Polling feed did not return the expected number of packages")
+	}
+	expectedVersions := []string{"2.1.0", "1.1.0", "2.0.5", "1.0.5", "1.0.0", "2.0.0"}
+	expectedRepos := []string{"fooOrg/baz", "fooOrg/bar", "fooOrg/baz", "fooOrg/bar", "fooOrg/bar", "fooOrg/baz"}
+
+	for i := range pkgs {
+		if pkgs[i].Version != expectedVersions[i] {
+			t.Errorf("Unexpected version %v found when expecting %v", pkgs[i].Version, expectedVersions[i])
+		}
+		if pkgs[i].Type != FeedName {
+			t.Errorf("Type set incorrectly for feed type")
+		}
+		if pkgs[i].Name != expectedRepos[i] {
+			t.Errorf("Unexpected name %v found when expecting %v", pkgs[i].Name, expectedRepos[i])
+		}
+	}
+}
+
+func TestConfigurationErrors(t *testing.T) {
+	t.Parallel()
+
+	_, err := New(feeds.FeedOptions{})
+	if !errors.Is(err, errPackageOptionsUnset) {
+		t.Fatalf("Expected to fail due to missing packages option")
+	}
+
+	packages := &[]string{}
+	_, err = New(feeds.FeedOptions{Packages: packages})
+	if !errors.Is(err, errMinimumPackagesRequired) {
+		t.Fatalf("Expected to fail due to missing packages option")
+	}
+}
+
+func barResponse(w http.ResponseWriter, r *http.Request) {
+	_, err := w.Write([]byte(`
+[
+	{
+		"tag_name": "1.1.0",
+		"draft": false,
+		"prerelease": false,
+		"published_at": "2021-02-04T10:22:41Z"
+	},
+	{
+		"tag_name": "1.0.5",
+		"draft": false,
+		"prerelease": false,
+		"published_at": "2021-01-04T10:22:41Z"
+	},
+	{
+		"tag_name": "1.0.0",
+		"draft": false,
+		"prerelease": false,
+		"published_at": "2021-01-02T10:22:41Z"
+	}
+]
+`))
+	if err != nil {
+		http.Error(w, testutils.UnexpectedWriteError(err), http.StatusInternalServerError)
+	}
+}
+
+func bazResponse(w http.ResponseWriter, r *http.Request) {
+	_, err := w.Write([]byte(`
+[
+	{
+		"tag_name": "2.1.0",
+		"draft": false,
+		"prerelease": false,
+		"published_at": "2021-04-05T10:22:41Z"
+	},
+	{
+		"tag_name": "2.0.5",
+		"draft": false,
+		"prerelease": false,
+		"published_at": "2021-02-01T10:22:41Z"
+	},
+	{
+		"tag_name": "2.0.0",
+		"draft": false,
+		"prerelease": false,
+		"published_at": "2021-01-01T10:22:41Z"
+	}
+]
+`))
+	if err != nil {
+		http.Error(w, testutils.UnexpectedWriteError(err), http.StatusInternalServerError)
+	}
+}

pkg/feeds/lossy_logging.go

@@ -1,8 +1,6 @@
 package feeds
 
 import (
-	"sort"
-
 	log "github.com/sirupsen/logrus"
 
 	"github.com/ossf/package-feeds/pkg/events"
@@ -32,10 +30,7 @@ func (lfa *LossyFeedAlerter) ProcessPackages(feed string, packages []*Package) {
 	pkgs := make([]*Package, len(packages))
 	copy(pkgs, packages)
 
-	// Ensure packages are sorted by CreatedDate in order of most recent
-	sort.SliceStable(pkgs, func(i, j int) bool {
-		return pkgs[j].CreatedDate.Before(pkgs[i].CreatedDate)
-	})
+	SortPackages(pkgs)
 
 	previousPackages, ok := lfa.previousPackages[feed]
 	nonZeroResults := len(pkgs) > 0 && len(previousPackages) > 0