Add Kube YAML parser/reader

go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/spf13/afero v1.2.2
 	github.com/stretchr/testify v1.7.0
 	github.com/xeipuuv/gojsonschema v1.2.0
+	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.21.3
 	k8s.io/apimachinery v0.21.3
 	k8s.io/client-go v0.21.3

go.sum

@@ -748,8 +748,9 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

pkg/devfile/generator/generators.go

@@ -2,6 +2,7 @@ package generator
 
 import (
 	"fmt"
+
 	v1 "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
 	"github.com/devfile/library/pkg/devfile/parser"
 	"github.com/devfile/library/pkg/devfile/parser/data/v2/common"

pkg/devfile/parser/reader.go

@@ -0,0 +1,107 @@
+package parser
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/devfile/library/pkg/testingutil/filesystem"
+	"github.com/devfile/library/pkg/util"
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v3"
+	k8yaml "sigs.k8s.io/yaml"
+
+	routev1 "github.com/openshift/api/route/v1"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+)
+
+// YamlSrc specifies the src of the yaml in either Path, URL or Data format
+type YamlSrc struct {
+	// Path is a relative or absolute yaml path.
+	Path string
+	// URL is the URL address of the specific yaml.
+	URL string
+	// Data is the yaml content in []byte format.
+	Data []byte
+}
+
+// ReadKubernetesYaml reads a yaml Kubernetes file from either the Path, URL or Data provided.
+// It returns Deployments, Services, Routes resources as the primary Kubernetes resources.
+// Other Kubernetes resources are returned as []byte type. Consumers interested in other Kubernetes resources
+// are expected to Unmarshal it to the struct of the respective resource.
+func ReadKubernetesYaml(src YamlSrc, fs filesystem.Filesystem) ([]appsv1.Deployment, []corev1.Service, []routev1.Route, [][]byte, error) {
+
+	var data []byte
+	var err error
+
+	if src.URL != "" {
+		data, err = util.DownloadFileInMemory(src.URL)
+		if err != nil {
+			return nil, nil, nil, nil, errors.Wrapf(err, "failed to download file %q", src.URL)
+		}
+	} else if src.Path != "" {
+		absPath, err := util.GetAbsPath(src.Path)
+		if err != nil {
+			return nil, nil, nil, nil, err
+		}
+		data, err = fs.ReadFile(absPath)
+		if err != nil {
+			return nil, nil, nil, nil, errors.Wrapf(err, "failed to read yaml from path %q", src.Path)
+		}
+	} else if len(src.Data) > 0 {
+		data = src.Data
+	}
+
+	var values []interface{}
+	dec := yaml.NewDecoder(bytes.NewReader(data))
+	for {
+		var value interface{}
+		err = dec.Decode(&value)
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, nil, nil, nil, err
+		}
+		values = append(values, value)
+	}
+
+	var deployments []appsv1.Deployment
+	var services []corev1.Service
+	var routes []routev1.Route
+	var otherResources [][]byte
+
+	for _, value := range values {
+		var deployment appsv1.Deployment
+		var service corev1.Service
+		var route routev1.Route
+
+		byteData, err := k8yaml.Marshal(value)
+		if err != nil {
+			return nil, nil, nil, nil, err
+		}
+
+		kubernetesMap := value.(map[string]interface{})
+		kind := kubernetesMap["kind"]
+
+		switch kind {
+		case "Deployment":
+			err = k8yaml.Unmarshal(byteData, &deployment)
+			deployments = append(deployments, deployment)
+		case "Service":
+			err = k8yaml.Unmarshal(byteData, &service)
+			services = append(services, service)
+		case "Route":
+			err = k8yaml.Unmarshal(byteData, &route)
+			routes = append(routes, route)
+		default:
+			otherResources = append(otherResources, byteData)
+		}
+
+		if err != nil {
+			return nil, nil, nil, nil, err
+		}
+	}
+
+	return deployments, services, routes, otherResources, nil
+}

pkg/devfile/parser/reader_test.go

@@ -0,0 +1,152 @@
+package parser
+
+import (
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/devfile/library/pkg/testingutil/filesystem"
+	"github.com/devfile/library/pkg/util"
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	k8yaml "sigs.k8s.io/yaml"
+)
+
+func TestReadKubernetesYaml(t *testing.T) {
+	const serverIP = "127.0.0.1:9080"
+	var data []byte
+
+	fs := filesystem.DefaultFs{}
+	absPath, err := util.GetAbsPath("../../../tests/yamls/resources.yaml")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	data, err = fs.ReadFile(absPath)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	// Mocking the YAML file endpoint on a very basic level
+	testServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, err = w.Write(data)
+		if err != nil {
+			t.Errorf("Unexpected error while writing data: %v", err)
+		}
+	}))
+	// create a listener with the desired port.
+	l, err := net.Listen("tcp", serverIP)
+	if err != nil {
+		t.Errorf("Unexpected error while creating listener: %v", err)
+		return
+	}
+
+	// NewUnstartedServer creates a listener. Close that listener and replace
+	// with the one we created.
+	testServer.Listener.Close()
+	testServer.Listener = l
+
+	testServer.Start()
+	defer testServer.Close()
+
+	badData := append(data, 59)
+
+	tests := []struct {
+		name                string
+		src                 YamlSrc
+		fs                  filesystem.Filesystem
+		wantErr             bool
+		wantDeploymentNames []string
+		wantServiceNames    []string
+		wantRouteNames      []string
+		wantOtherNames      []string
+	}{
+		{
+			name: "Read the YAML from the URL",
+			src: YamlSrc{
+				URL: "http://" + serverIP,
+			},
+			fs:                  filesystem.DefaultFs{},
+			wantDeploymentNames: []string{"deploy-sample"},
+			wantServiceNames:    []string{"service-sample"},
+			wantRouteNames:      []string{"route-sample"},
+			wantOtherNames:      []string{"pvc-sample"},
+		},
+		{
+			name: "Read the YAML from the Path",
+			src: YamlSrc{
+				Path: "../../../tests/yamls/resources.yaml",
+			},
+			fs:                  filesystem.DefaultFs{},
+			wantDeploymentNames: []string{"deploy-sample"},
+			wantServiceNames:    []string{"service-sample"},
+			wantRouteNames:      []string{"route-sample"},
+			wantOtherNames:      []string{"pvc-sample"},
+		},
+		{
+			name: "Read the YAML from the Data",
+			src: YamlSrc{
+				Data: data,
+			},
+			fs:                  filesystem.DefaultFs{},
+			wantDeploymentNames: []string{"deploy-sample"},
+			wantServiceNames:    []string{"service-sample"},
+			wantRouteNames:      []string{"route-sample"},
+			wantOtherNames:      []string{"pvc-sample"},
+		},
+		{
+			name: "Bad URL",
+			src: YamlSrc{
+				URL: "http://badurl",
+			},
+			fs:      filesystem.DefaultFs{},
+			wantErr: true,
+		},
+		{
+			name: "Bad Path",
+			src: YamlSrc{
+				Path: "$%^&",
+			},
+			fs:      filesystem.DefaultFs{},
+			wantErr: true,
+		},
+		{
+			name: "Bad Data",
+			src: YamlSrc{
+				Data: badData,
+			},
+			fs:      filesystem.DefaultFs{},
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			deployments, services, routes, others, err := ReadKubernetesYaml(tt.src, tt.fs)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("unexpected error: %v", err)
+				return
+			}
+			for _, deploy := range deployments {
+				assert.Contains(t, tt.wantDeploymentNames, deploy.Name)
+			}
+			for _, svc := range services {
+				assert.Contains(t, tt.wantServiceNames, svc.Name)
+			}
+			for _, route := range routes {
+				assert.Contains(t, tt.wantRouteNames, route.Name)
+			}
+			for _, other := range others {
+				pvc := corev1.PersistentVolumeClaim{}
+				err = k8yaml.Unmarshal(other, &pvc)
+				if err != nil {
+					t.Error(err)
+				}
+				assert.Contains(t, tt.wantOtherNames, pvc.Name)
+			}
+		})
+	}
+}