diff --git a/.gitignore b/.gitignore index 537346e0e..a183c7b28 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ out/ .run* manifest.json -ami-id \ No newline at end of file +ami-id +Pulumi*.yaml diff --git a/Makefile b/Makefile index 7ad6e2c41..13244538f 100644 --- a/Makefile +++ b/Makefile @@ -7,11 +7,12 @@ TKN_IMG ?= quay.io/redhat-developer/mapt:v${VERSION}-tkn # Go and compilation related variables GOPATH ?= $(shell go env GOPATH) BUILD_DIR ?= out -SOURCE_DIRS = cmd pkg test +SOURCE_DIRS = cmd pkg +SOURCES := $(shell find . -name "*.go" -not -path "./vendor/*") # https://golang.org/cmd/link/ # LDFLAGS := $(VERSION_VARIABLES) -extldflags='-static' ${GO_EXTRA_LDFLAGS} LDFLAGS := $(VERSION_VARIABLES) ${GO_EXTRA_LDFLAGS} -GCFLAGS := all=-N -l +GCFLAGS := all=-N -l TOOLS_DIR := tools include tools/tools.mk @@ -33,12 +34,12 @@ check: build test lint .PHONY: install install: $(SOURCES) - go install -ldflags="$(LDFLAGS)" $(GO_EXTRA_BUILDFLAGS) ./cmd + go install -ldflags="$(LDFLAGS)" $(GO_EXTRA_BUILDFLAGS) ./cmd/mapt $(BUILD_DIR)/mapt: $(SOURCES) - GOOS=linux GOARCH=amd64 go build -gcflags="$(GCFLAGS)" -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/mapt $(GO_EXTRA_BUILDFLAGS) ./cmd + GOOS=linux GOARCH=amd64 go build -gcflags="$(GCFLAGS)" -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/mapt $(GO_EXTRA_BUILDFLAGS) ./cmd/mapt -.PHONY: build +.PHONY: build build: $(BUILD_DIR)/mapt .PHONY: test @@ -77,4 +78,4 @@ tkn-push: install-out-of-tree-tools -f tkn/infra-aws-mac.yaml \ -f tkn/infra-aws-rhel.yaml \ -f tkn/infra-aws-windows-server.yaml \ - -f tkn/infra-azure-windows-desktop.yaml \ No newline at end of file + -f tkn/infra-azure-windows-desktop.yaml diff --git a/README.md b/README.md index aa3c469c1..d61012d72 100644 --- a/README.md +++ b/README.md @@ -31,4 +31,19 @@ Features: * a airgap * s spot * p proxy -* v vpn \ No newline at end of file +* v vpn + +## Github Self hosted runner + +`mapt` can setup a deployed machine as a Self Hosted runner on most of the Platform and Provider combinations +it supports. + +Use the following flags with `mapt create` command: + +``` +--install-ghactions-runner Install and setup Github Actions runner in the instance +--ghactions-runner-name Name for the Github Actions Runner +--ghactions-runner-repo Full URL of the repository where the Github Actions Runner should be registered +--ghactions-runner-token Token needed for registering the Github Actions Runner token +``` + diff --git a/cmd/main.go b/cmd/main.go deleted file mode 100644 index 48e479ca8..000000000 --- a/cmd/main.go +++ /dev/null @@ -1,7 +0,0 @@ -package main - -import "github.com/redhat-developer/mapt/cmd/cmd" - -func main() { - cmd.Execute() -} diff --git a/cmd/cmd/aws/aws.go b/cmd/mapt/cmd/aws/aws.go similarity index 89% rename from cmd/cmd/aws/aws.go rename to cmd/mapt/cmd/aws/aws.go index 42d23cf6e..28766b91a 100644 --- a/cmd/cmd/aws/aws.go +++ b/cmd/mapt/cmd/aws/aws.go @@ -1,7 +1,7 @@ package aws import ( - "github.com/redhat-developer/mapt/cmd/cmd/aws/hosts" + "github.com/redhat-developer/mapt/cmd/mapt/cmd/aws/hosts" "github.com/spf13/cobra" "github.com/spf13/viper" ) diff --git a/cmd/cmd/aws/hosts/constans.go b/cmd/mapt/cmd/aws/hosts/constans.go similarity index 100% rename from cmd/cmd/aws/hosts/constans.go rename to cmd/mapt/cmd/aws/hosts/constans.go diff --git a/cmd/cmd/aws/hosts/fedora.go b/cmd/mapt/cmd/aws/hosts/fedora.go similarity index 97% rename from cmd/cmd/aws/hosts/fedora.go rename to cmd/mapt/cmd/aws/hosts/fedora.go index 918fc6a1b..77928ac4f 100644 --- a/cmd/cmd/aws/hosts/fedora.go +++ b/cmd/mapt/cmd/aws/hosts/fedora.go @@ -1,7 +1,7 @@ package hosts import ( - params "github.com/redhat-developer/mapt/cmd/cmd/constants" + params "github.com/redhat-developer/mapt/cmd/mapt/cmd/constants" maptContext "github.com/redhat-developer/mapt/pkg/manager/context" "github.com/redhat-developer/mapt/pkg/provider/aws/action/fedora" "github.com/redhat-developer/mapt/pkg/util/logging" diff --git a/cmd/cmd/aws/hosts/mac.go b/cmd/mapt/cmd/aws/hosts/mac.go similarity index 98% rename from cmd/cmd/aws/hosts/mac.go rename to cmd/mapt/cmd/aws/hosts/mac.go index 15f5633b8..6fbe2e463 100644 --- a/cmd/cmd/aws/hosts/mac.go +++ b/cmd/mapt/cmd/aws/hosts/mac.go @@ -1,7 +1,7 @@ package hosts import ( - params "github.com/redhat-developer/mapt/cmd/cmd/constants" + params "github.com/redhat-developer/mapt/cmd/mapt/cmd/constants" maptContext "github.com/redhat-developer/mapt/pkg/manager/context" "github.com/redhat-developer/mapt/pkg/provider/aws/action/mac" "github.com/redhat-developer/mapt/pkg/util/logging" diff --git a/cmd/cmd/aws/hosts/rhel.go b/cmd/mapt/cmd/aws/hosts/rhel.go similarity index 77% rename from cmd/cmd/aws/hosts/rhel.go rename to cmd/mapt/cmd/aws/hosts/rhel.go index 92ed84ac3..41abc2dc6 100644 --- a/cmd/cmd/aws/hosts/rhel.go +++ b/cmd/mapt/cmd/aws/hosts/rhel.go @@ -1,9 +1,10 @@ package hosts import ( - params "github.com/redhat-developer/mapt/cmd/cmd/constants" + params "github.com/redhat-developer/mapt/cmd/mapt/cmd/constants" maptContext "github.com/redhat-developer/mapt/pkg/manager/context" "github.com/redhat-developer/mapt/pkg/provider/aws/action/rhel" + "github.com/redhat-developer/mapt/pkg/util/ghactions" "github.com/redhat-developer/mapt/pkg/util/logging" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -61,18 +62,30 @@ func getRHELCreate() *cobra.Command { viper.GetString(params.ConnectionDetailsOutput), viper.GetStringMapString(params.Tags)) + // Initialize gh actions runner if needed + if viper.IsSet(params.InstallGHActionsRunner) { + err := ghactions.InitGHRunnerArgs(viper.GetString(params.GHActionsRunnerToken), + viper.GetString(params.GHActionsRunnerName), + viper.GetString(params.GHActionsRunnerRepo)) + if err != nil { + logging.Error(err) + } + } + // Run create if err := rhel.Create( &rhel.Request{ - Prefix: "main", - Version: viper.GetString(rhelVersion), - Arch: viper.GetString(rhelArch), - VMType: viper.GetStringSlice(vmTypes), - SubsUsername: viper.GetString(subsUsername), - SubsUserpass: viper.GetString(subsUserpass), - ProfileSNC: viper.IsSet(profileSNC), - Spot: viper.IsSet(spot), - Airgap: viper.IsSet(airgap)}); err != nil { + Prefix: "main", + Version: viper.GetString(rhelVersion), + Arch: viper.GetString(rhelArch), + VMType: viper.GetStringSlice(vmTypes), + SubsUsername: viper.GetString(subsUsername), + SubsUserpass: viper.GetString(subsUserpass), + ProfileSNC: viper.IsSet(profileSNC), + Spot: viper.IsSet(spot), + Airgap: viper.IsSet(airgap), + SetupGHActionsRunner: viper.GetBool(params.InstallGHActionsRunner), + }); err != nil { logging.Error(err) } return nil @@ -89,6 +102,7 @@ func getRHELCreate() *cobra.Command { flagSet.Bool(airgap, false, airgapDesc) flagSet.Bool(spot, false, spotDesc) flagSet.Bool(profileSNC, false, profileSNCDesc) + flagSet.AddFlagSet(params.GetGHActionsFlagset()) c.PersistentFlags().AddFlagSet(flagSet) // if err := c.MarkFlagRequired(subsUsername); err != nil { // logging.Error(err) diff --git a/cmd/cmd/aws/hosts/windows.go b/cmd/mapt/cmd/aws/hosts/windows.go similarity index 77% rename from cmd/cmd/aws/hosts/windows.go rename to cmd/mapt/cmd/aws/hosts/windows.go index a10e0b789..3397e8da8 100644 --- a/cmd/cmd/aws/hosts/windows.go +++ b/cmd/mapt/cmd/aws/hosts/windows.go @@ -1,9 +1,10 @@ package hosts import ( - params "github.com/redhat-developer/mapt/cmd/cmd/constants" + params "github.com/redhat-developer/mapt/cmd/mapt/cmd/constants" maptContext "github.com/redhat-developer/mapt/pkg/manager/context" "github.com/redhat-developer/mapt/pkg/provider/aws/action/windows" + "github.com/redhat-developer/mapt/pkg/util/ghactions" "github.com/redhat-developer/mapt/pkg/util/logging" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -61,17 +62,29 @@ func getWindowsCreate() *cobra.Command { viper.GetString(params.ConnectionDetailsOutput), viper.GetStringMapString(params.Tags)) + // Initialize gh actions runner if needed + if viper.IsSet(params.InstallGHActionsRunner) { + err := ghactions.InitGHRunnerArgs(viper.GetString(params.GHActionsRunnerToken), + viper.GetString(params.GHActionsRunnerName), + viper.GetString(params.GHActionsRunnerRepo)) + if err != nil { + logging.Error(err) + } + } + // Run create if err := windows.Create( &windows.Request{ - Prefix: "main", - AMIName: viper.GetString(amiName), - AMIUser: viper.GetString(amiUsername), - AMIOwner: viper.GetString(amiOwner), - AMILang: viper.GetString(amiLang), - AMIKeepCopy: viper.IsSet(amiKeepCopy), - Spot: viper.IsSet(spot), - Airgap: viper.IsSet(airgap)}); err != nil { + Prefix: "main", + AMIName: viper.GetString(amiName), + AMIUser: viper.GetString(amiUsername), + AMIOwner: viper.GetString(amiOwner), + AMILang: viper.GetString(amiLang), + AMIKeepCopy: viper.IsSet(amiKeepCopy), + Spot: viper.IsSet(spot), + Airgap: viper.IsSet(airgap), + SetupGHActionsRunner: viper.IsSet(params.InstallGHActionsRunner), + }); err != nil { logging.Error(err) } return nil @@ -87,6 +100,7 @@ func getWindowsCreate() *cobra.Command { flagSet.Bool(airgap, false, airgapDesc) flagSet.Bool(spot, false, spotDesc) flagSet.Bool(amiKeepCopy, false, amiKeepCopyDesc) + flagSet.AddFlagSet(params.GetGHActionsFlagset()) c.PersistentFlags().AddFlagSet(flagSet) return c } diff --git a/cmd/cmd/aws/replica/replica.go b/cmd/mapt/cmd/aws/replica/replica.go similarity index 96% rename from cmd/cmd/aws/replica/replica.go rename to cmd/mapt/cmd/aws/replica/replica.go index 45c4ba934..c6873c6d7 100644 --- a/cmd/cmd/aws/replica/replica.go +++ b/cmd/mapt/cmd/aws/replica/replica.go @@ -1,7 +1,7 @@ package replica import ( - params "github.com/redhat-developer/mapt/cmd/cmd/constants" + params "github.com/redhat-developer/mapt/cmd/mapt/cmd/constants" amireplication "github.com/redhat-developer/mapt/pkg/provider/aws/modules/ami" "github.com/redhat-developer/mapt/pkg/util/logging" "github.com/spf13/cobra" diff --git a/cmd/cmd/azure/azure.go b/cmd/mapt/cmd/azure/azure.go similarity index 87% rename from cmd/cmd/azure/azure.go rename to cmd/mapt/cmd/azure/azure.go index 5a3d39c9a..5a56f84fa 100644 --- a/cmd/cmd/azure/azure.go +++ b/cmd/mapt/cmd/azure/azure.go @@ -1,7 +1,7 @@ package azure import ( - "github.com/redhat-developer/mapt/cmd/cmd/azure/hosts" + "github.com/redhat-developer/mapt/cmd/mapt/cmd/azure/hosts" "github.com/spf13/cobra" "github.com/spf13/viper" ) diff --git a/cmd/cmd/azure/hosts/windows.go b/cmd/mapt/cmd/azure/hosts/windows.go similarity index 80% rename from cmd/cmd/azure/hosts/windows.go rename to cmd/mapt/cmd/azure/hosts/windows.go index 436cf9571..654bfba8b 100644 --- a/cmd/cmd/azure/hosts/windows.go +++ b/cmd/mapt/cmd/azure/hosts/windows.go @@ -3,10 +3,11 @@ package hosts import ( "fmt" - params "github.com/redhat-developer/mapt/cmd/cmd/constants" + params "github.com/redhat-developer/mapt/cmd/mapt/cmd/constants" maptContext "github.com/redhat-developer/mapt/pkg/manager/context" azureWindows "github.com/redhat-developer/mapt/pkg/provider/azure/action/windows" spotprice "github.com/redhat-developer/mapt/pkg/provider/azure/module/spot-price" + "github.com/redhat-developer/mapt/pkg/util/ghactions" "github.com/redhat-developer/mapt/pkg/util/logging" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -74,6 +75,7 @@ func getCreate() *cobra.Command { viper.GetString(params.ConnectionDetailsOutput), viper.GetStringMapString(params.Tags)) + // ParseEvictionRate var spotToleranceValue = spotprice.DefaultEvictionRate if viper.IsSet(paramSpotTolerance) { var ok bool @@ -83,19 +85,30 @@ func getCreate() *cobra.Command { return fmt.Errorf("%s is not a valid spot tolerance value", viper.GetString(paramSpotTolerance)) } } - // ParseEvictionRate + + // Initialize gh actions runner if needed + if viper.IsSet(params.InstallGHActionsRunner) { + err := ghactions.InitGHRunnerArgs(viper.GetString(params.GHActionsRunnerToken), + viper.GetString(params.GHActionsRunnerName), + viper.GetString(params.GHActionsRunnerRepo)) + if err != nil { + logging.Error(err) + } + } + if err := azureWindows.Create( &azureWindows.WindowsRequest{ - Prefix: "", - Location: viper.GetString(paramLocation), - VMSize: viper.GetString(paramVMSize), - Version: viper.GetString(paramVersion), - Feature: viper.GetString(paramFeature), - Username: viper.GetString(paramUsername), - AdminUsername: viper.GetString(paramAdminUsername), - Profiles: viper.GetStringSlice(paramProfile), - Spot: viper.IsSet(paramSpot), - SpotTolerance: spotToleranceValue}); err != nil { + Prefix: viper.GetString(params.ProjectName), + Location: viper.GetString(paramLocation), + VMSize: viper.GetString(paramVMSize), + Version: viper.GetString(paramVersion), + Feature: viper.GetString(paramFeature), + Username: viper.GetString(paramUsername), + AdminUsername: viper.GetString(paramAdminUsername), + Profiles: viper.GetStringSlice(paramProfile), + SetupGHActionsRunner: viper.IsSet(params.InstallGHActionsRunner), + Spot: viper.IsSet(paramSpot), + SpotTolerance: spotToleranceValue}); err != nil { logging.Error(err) } return nil @@ -113,6 +126,7 @@ func getCreate() *cobra.Command { flagSet.StringSliceP(paramProfile, "", []string{}, paramProfileDesc) flagSet.Bool(paramSpot, false, paramSpotDesc) flagSet.StringP(paramSpotTolerance, "", defaultSpotTolerance, paramSpotToleranceDesc) + flagSet.AddFlagSet(params.GetGHActionsFlagset()) c.PersistentFlags().AddFlagSet(flagSet) return c } diff --git a/cmd/cmd/constants/constants.go b/cmd/mapt/cmd/constants/constants.go similarity index 66% rename from cmd/cmd/constants/constants.go rename to cmd/mapt/cmd/constants/constants.go index 4a5be3eb3..2a51a936e 100644 --- a/cmd/cmd/constants/constants.go +++ b/cmd/mapt/cmd/constants/constants.go @@ -1,5 +1,7 @@ package constants +import "github.com/spf13/pflag" + const ( ProjectName string = "project-name" ProjectNameDesc string = "project name to identify the instance of the stack" @@ -29,7 +31,25 @@ const ( AMISourceRegionDesc string = "region for the ami to be copied worldwide" Tags string = "tags" TagsDesc string = "tags to add on each resource (--tags name1=value1,name2=value2)" + InstallGHActionsRunnerDesc string = "Install and setup Github Actions runner in the instance" + GHActionsRunnerTokenDesc string = "Token needed for registering the Github Actions Runner token" + GHActionsRunnerNameDesc string = "Name for the Github Actions Runner" + GHActionsRunnerRepoDesc string = "Full URL of the repository where the Github Actions Runner should be registered" CreateCmdName string = "create" DestroyCmdName string = "destroy" + + InstallGHActionsRunner string = "install-ghactions-runner" + GHActionsRunnerToken string = "ghactions-runner-token" + GHActionsRunnerName string = "ghactions-runner-name" + GHActionsRunnerRepo string = "ghactions-runner-repo" ) + +func GetGHActionsFlagset() *pflag.FlagSet { + flagSet := pflag.NewFlagSet(CreateCmdName, pflag.ExitOnError) + flagSet.Bool(InstallGHActionsRunner, false, InstallGHActionsRunnerDesc) + flagSet.StringP(GHActionsRunnerToken, "", "", GHActionsRunnerTokenDesc) + flagSet.StringP(GHActionsRunnerName, "", "", GHActionsRunnerNameDesc) + flagSet.StringP(GHActionsRunnerRepo, "", "", GHActionsRunnerRepoDesc) + return flagSet +} diff --git a/cmd/cmd/root.go b/cmd/mapt/cmd/root.go similarity index 85% rename from cmd/cmd/root.go rename to cmd/mapt/cmd/root.go index 6d4c57d5f..6b1c04703 100644 --- a/cmd/cmd/root.go +++ b/cmd/mapt/cmd/root.go @@ -7,9 +7,9 @@ import ( "path/filepath" "strings" - "github.com/redhat-developer/mapt/cmd/cmd/aws" - "github.com/redhat-developer/mapt/cmd/cmd/azure" - params "github.com/redhat-developer/mapt/cmd/cmd/constants" + "github.com/redhat-developer/mapt/cmd/mapt/cmd/aws" + "github.com/redhat-developer/mapt/cmd/mapt/cmd/azure" + params "github.com/redhat-developer/mapt/cmd/mapt/cmd/constants" "github.com/redhat-developer/mapt/pkg/util/logging" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -17,8 +17,8 @@ import ( const ( commandName = "mapt" - descriptionShort = "PoC for pulumi" - descriptionLong = "PoC for pulumi" + descriptionShort = "Multi Architecture Provisioning Tool" + descriptionLong = "MAPT is a tool for creating pre-configured machines (baremetal or VMs) on cloud providers" defaultErrorExitCode = 1 ) diff --git a/cmd/mapt/main.go b/cmd/mapt/main.go new file mode 100644 index 000000000..d8384931d --- /dev/null +++ b/cmd/mapt/main.go @@ -0,0 +1,7 @@ +package main + +import "github.com/redhat-developer/mapt/cmd/mapt/cmd" + +func main() { + cmd.Execute() +} diff --git a/docs/self-hosted-runner.md b/docs/self-hosted-runner.md new file mode 100644 index 000000000..4dad6c617 --- /dev/null +++ b/docs/self-hosted-runner.md @@ -0,0 +1,58 @@ +# Overview + +This feature of `mapt` allows to setup hosts deployed by it as a Github Self Hosted Runner, which can then be directly used for running github actions jobs. +It benefits from all the existing features that `mapt` already provides, allowing to create self hosted runners that can be used for different QE scenarios. + +## Providers and Platforms + +Currently it allow to create self hosted runners on AWS (Windows Server, RHEL) and Azure (Windows Desktop) + +### Prerequisite + +To register a Self Hosted Runner for a repository or a Github organization, the runner program needs a registration token, which can be obtained by requesting the +Github API. + +* [Information for requesting a token to register a runner for an Organization](https://docs.github.com/en/rest/actions/self-hosted-runners#create-a-registration-token-for-an-organization) +* [Information for requesting a token to register a runner for a repository](https://docs.github.com/en/rest/actions/self-hosted-runners#create-a-registration-token-for-a-repository) + +After obtaining the token we can invoke `mapt` with it to deploy a VM as a Self hosted runner. + +For example to add a runner to this repository, we can use the following `curl` command to request a token: + +``` +% curl -L \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer " \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/redhat-developer/mapt/actions/runners/registration-token +``` +The Response from this `POST` request will be: + +``` +{ + "token": "ACDZL3QXEIC73UXBDGSEYEI", + "expires_at": "2024-07-12T19:01:48.478+05:30" +} +``` + +### Operations + +After getting the required token, we need to also decide what we are going to call this runner, the desired name can be passed to the `mapt` command using the +`--ghactions-runner-name` flag. + +The full URL of the repository or the github organization also needs to be passed using the `--ghactions-runner-repo` flag. + +To deploy a Windows runner on the Azure provider, we can use the following command: + +``` +% mapt azure windows create --spot \ + --install-ghactions-runner \ + --ghcations-runner-token="ACDZL3QXEIC73UXBDGSEYEI" \ + --ghcations-runner-name "az-win-11" \ + --ghcations-runner-repo "https://github.com/redhat-developer/mapt" \ + --project-name mapt-windows-azure \ + --backed-url file:///Users/tester/workspace \ + --conn-details-output /Users/tester/workspace/conn-details +``` + diff --git a/pkg/provider/aws/action/rhel/cloud-config-base b/pkg/provider/aws/action/rhel/cloud-config-base index 6809e8dc2..817d04265 100644 --- a/pkg/provider/aws/action/rhel/cloud-config-base +++ b/pkg/provider/aws/action/rhel/cloud-config-base @@ -5,4 +5,12 @@ rh_subscription: auto-attach: true runcmd: - while fuser /var/lib/rpm/.rpm.lock > /dev/null 2>&1 ; do sleep 1 ; done - - dnf install -y podman \ No newline at end of file + - dnf install -y podman +{{ if .InstallActionsRunner }} - sudo -u {{ .Username }} bash -c /opt/install-ghrunner.sh{{ end }} +{{ if .InstallActionsRunner }}write_files: + # Github actions runner installation + - content: | + {{ .ActionsRunnerSnippet }} + path: /opt/install-ghrunner.sh + permissions: '0755' +{{ end }} diff --git a/pkg/provider/aws/action/rhel/cloud-config-snc b/pkg/provider/aws/action/rhel/cloud-config-snc index 9f4a6a0f2..6dcc22d94 100644 --- a/pkg/provider/aws/action/rhel/cloud-config-snc +++ b/pkg/provider/aws/action/rhel/cloud-config-snc @@ -25,3 +25,11 @@ runcmd: - echo "user.max_user_namespaces=28633" | tee -a /etc/sysctl.d/userns.conf - sysctl -p /etc/sysctl.d/userns.conf - dnf upgrade -y curl openssl +{{ if .InstallActionsRunner }} - sudo -u {{ .Username }} bash -c /opt/install-ghrunner.sh {{ end }} +{{ if .InstallActionsRunner }}write_files: + # Github actions runner installation + - content: | + {{ .ActionsRunnerSnippet }} + path: /opt/install-ghrunner.sh + permissions: '0755' +{{ end }} diff --git a/pkg/provider/aws/action/rhel/rhel.go b/pkg/provider/aws/action/rhel/rhel.go index 0b43e128e..ab05c2446 100644 --- a/pkg/provider/aws/action/rhel/rhel.go +++ b/pkg/provider/aws/action/rhel/rhel.go @@ -25,6 +25,7 @@ import ( "github.com/redhat-developer/mapt/pkg/provider/util/output" "github.com/redhat-developer/mapt/pkg/util" "github.com/redhat-developer/mapt/pkg/util/file" + "github.com/redhat-developer/mapt/pkg/util/ghactions" resourcesUtil "github.com/redhat-developer/mapt/pkg/util/resources" ) @@ -41,6 +42,8 @@ type Request struct { ProfileSNC bool Spot bool Airgap bool + // setup as github actions runner + SetupGHActionsRunner bool // internal management // For airgap scenario there is an orchestation of // a phase with connectivity on the machine (allowing bootstraping) @@ -56,6 +59,8 @@ type userDataValues struct { SubscriptionUsername string SubscriptionPassword string Username string + InstallActionsRunner bool + ActionsRunnerSnippet string } //go:embed cloud-config-base @@ -274,7 +279,9 @@ func (r *Request) getUserdata() (pulumi.StringPtrInput, error) { userDataValues{ r.SubsUsername, r.SubsUserpass, - amiUserDefault}, + amiUserDefault, + r.SetupGHActionsRunner, + ghactions.GetActionRunnerSnippetLinux()}, resourcesUtil.GetResourceName( r.Prefix, awsRHELDedicatedID, "userdata"), templateConfig) diff --git a/pkg/provider/aws/action/windows/bootstrap.ps1 b/pkg/provider/aws/action/windows/bootstrap.ps1 index e2214b5ee..3a530572d 100644 --- a/pkg/provider/aws/action/windows/bootstrap.ps1 +++ b/pkg/provider/aws/action/windows/bootstrap.ps1 @@ -19,8 +19,13 @@ Invoke-Command -ScriptBlock { New-Item -Path C:\Users\{{.Username}}\.ssh -Name " # Set back disable unnecrypted communications Set-Item .\allowunencrypted $false +# Install github-actions-runner if needed +{{ if .InstallActionsRunner }}$ghToken = {{ .RunnerToken }} + {{- .ActionsRunnerSnippet }} +{{ end }} + netsh advfirewall firewall add rule name="Open SSH Port 22" dir=in action=allow protocol=TCP localport=22 remoteip=any # Restart computer to have the ssh connection available with setup from this script Start-Process powershell -verb runas -ArgumentList "Restart-Computer -Force" - \ No newline at end of file + diff --git a/pkg/provider/aws/action/windows/windows.go b/pkg/provider/aws/action/windows/windows.go index 7f9b05f35..b98e5572d 100644 --- a/pkg/provider/aws/action/windows/windows.go +++ b/pkg/provider/aws/action/windows/windows.go @@ -28,6 +28,7 @@ import ( "github.com/redhat-developer/mapt/pkg/provider/util/security" "github.com/redhat-developer/mapt/pkg/util" "github.com/redhat-developer/mapt/pkg/util/file" + "github.com/redhat-developer/mapt/pkg/util/ghactions" resourcesUtil "github.com/redhat-developer/mapt/pkg/util/resources" ) @@ -44,6 +45,8 @@ type Request struct { // Features Spot bool Airgap bool + // setup as github actions runner + SetupGHActionsRunner bool // internal management // For airgap scenario there is an orchestation of // a phase with connectivity on the machine (allowing bootstraping) @@ -56,9 +59,12 @@ type Request struct { } type userDataValues struct { - Username string - Password string - AuthorizedKey string + Username string + Password string + AuthorizedKey string + InstallActionsRunner bool + ActionsRunnerSnippet string + RunnerToken string } //go:embed bootstrap.ps1 @@ -318,14 +324,16 @@ func (r *Request) getUserdata(ctx *pulumi.Context, func(args []interface{}) (string, error) { password := args[0].(string) authorizedKey := args[1].(string) - userdata, err := file.Template( - userDataValues{ - r.AMIUser, - password, - authorizedKey}, - resourcesUtil.GetResourceName( - r.Prefix, awsWindowsDedicatedID, "userdatas"), - string(BootstrapScript[:])) + tmplName := resourcesUtil.GetResourceName(r.Prefix, awsWindowsDedicatedID, "userdatas") + udv := userDataValues{ + r.AMIUser, + password, + authorizedKey, + r.SetupGHActionsRunner, + ghactions.GetActionRunnerSnippetWin(), + ghactions.GetToken(), + } + userdata, err := file.Template(udv, tmplName, string(BootstrapScript[:])) if err != nil { return "", err } diff --git a/pkg/provider/azure/action/windows/rhqp-ci-setup.ps1 b/pkg/provider/azure/action/windows/rhqp-ci-setup.ps1 index 4ab7736a7..5c5f8f21d 100644 --- a/pkg/provider/azure/action/windows/rhqp-ci-setup.ps1 +++ b/pkg/provider/azure/action/windows/rhqp-ci-setup.ps1 @@ -7,6 +7,8 @@ param( $hostname, [Parameter(Mandatory,HelpMessage='authorizedkey for ssh private key for the user')] $authorizedKey, + [Parameter(HelpMessage='token for the github actions runner')] + $ghToken, [switch]$crcProfile=$false ) # Create local user @@ -124,6 +126,12 @@ if ($crcProfile) { New-LocalGroup -Name "crc-users" Add-LocalGroupMember -Group "crc-users" -Member $user } + +# Install github-actions-runner if needed +{{ if .InstallActionsRunner }} + {{- .ActionsRunnerSnippet }} +{{ end }} + # Restart computer to have the ssh connection available with setup from this script Start-Process powershell -verb runas -ArgumentList "Restart-Computer -Force" diff --git a/pkg/provider/azure/action/windows/windows.go b/pkg/provider/azure/action/windows/windows.go index 3dabdaa95..b1435eeba 100644 --- a/pkg/provider/azure/action/windows/windows.go +++ b/pkg/provider/azure/action/windows/windows.go @@ -21,6 +21,8 @@ import ( "github.com/redhat-developer/mapt/pkg/provider/util/output" "github.com/redhat-developer/mapt/pkg/provider/util/security" "github.com/redhat-developer/mapt/pkg/util" + "github.com/redhat-developer/mapt/pkg/util/file" + "github.com/redhat-developer/mapt/pkg/util/ghactions" "github.com/redhat-developer/mapt/pkg/util/logging" resourcesUtil "github.com/redhat-developer/mapt/pkg/util/resources" "golang.org/x/exp/slices" @@ -40,6 +42,13 @@ type WindowsRequest struct { Spot bool SpotTolerance spotprice.EvictionRate Profiles []string + // setup as github actions runner + SetupGHActionsRunner bool +} + +type ghActionsRunnerData struct { + InstallActionsRunner bool + ActionsRunnerSnippet string } func Create(r *WindowsRequest) (err error) { @@ -315,18 +324,20 @@ func (r *WindowsRequest) postInitSetup(ctx *pulumi.Context, rg *resources.Resour return nil, nil, err } // the post script command will be generated based on generated data as parameters - setupCommand := pulumi.All(userPasswd.Result, privateKey.PublicKeyOpenssh, vm.OsProfile.ComputerName()).ApplyT( + setupCommand := pulumi.All(userPasswd.Result, privateKey.PublicKeyOpenssh, vm.OsProfile.ComputerName(), ghactions.GetToken()).ApplyT( func(args []interface{}) string { password := args[0].(string) authorizedKey := args[1].(string) hostname := args[2].(*string) + token := args[3].(string) return fmt.Sprintf( - "powershell -ExecutionPolicy Unrestricted -File %s %s -userPass \"%s\" -user %s -hostname %s -authorizedKey \"%s\"", + "powershell -ExecutionPolicy Unrestricted -File %s %s -userPass \"%s\" -user %s -hostname %s -ghToken \"%s\" -authorizedKey \"%s\"", scriptName, r.profilesAsParams(), password, r.Username, *hostname, + token, authorizedKey, ) }).(pulumi.StringOutput) @@ -386,13 +397,23 @@ func (r *WindowsRequest) uploadScript(ctx *pulumi.Context, if err != nil { return nil, err } + + data := ghActionsRunnerData{ + r.SetupGHActionsRunner, + ghactions.GetActionRunnerSnippetWin(), + } + ciSetupScript, err := file.Template(data, "ciSetupScript", string(RHQPCISetupScript)) + if err != nil { + return nil, err + } + return storage.NewBlob(ctx, resourcesUtil.GetResourceName(r.Prefix, azureWindowsDesktopID, "bl"), &storage.BlobArgs{ AccountName: sa.Name, ContainerName: c.Name, ResourceGroupName: rg.Name, - Source: pulumi.NewStringAsset(string(RHQPCISetupScript)), + Source: pulumi.NewStringAsset(ciSetupScript), BlobName: pulumi.String(scriptName), }) } diff --git a/pkg/util/ghactions/runner.go b/pkg/util/ghactions/runner.go new file mode 100644 index 000000000..2119adf09 --- /dev/null +++ b/pkg/util/ghactions/runner.go @@ -0,0 +1,84 @@ +package ghactions + +import ( + "fmt" + + "github.com/pkg/errors" +) + +type RunnerArgs struct { + Token string + RepoURL string + Name string +} + +const runnerVersion = "2.317.0" + +var ( + args *RunnerArgs + + runnerBaseURLWin = "https://github.com/actions/runner/releases/download/v%s/actions-runner-win-x64-%s.zip" + runnerBaseURLLinux = "https://github.com/actions/runner/releases/download/v%s/actions-runner-linux-x64-%s.tar.gz" +) + +func getRunnerDownloadURLWin() string { + return fmt.Sprintf(runnerBaseURLWin, runnerVersion, runnerVersion) +} + +func getRunnerDownloadURLLinux() string { + return fmt.Sprintf(runnerBaseURLLinux, runnerVersion, runnerVersion) +} + +func InitGHRunnerArgs(token, name, repoURL string) error { + if token == "" || name == "" || repoURL == "" { + return errors.New("All args are required and must have non-empty values") + } + args = &RunnerArgs{ + Token: token, + RepoURL: repoURL, + Name: name, + } + return nil +} + +func GetToken() string { + if (args == &RunnerArgs{}) { + return "" + } + return args.Token +} + +// $ghToken needs to be set externally before use; it is defined in the platform specific setup scripts +// for aws this is defined in the script and for azure it is passed as an arg to the setup script +const WindowsActionsRunnerInstallSnippet string = `New-Item -Path C:\actions-runner -Type Directory ; cd C:\actions-runner +Invoke-WebRequest -Uri %s -OutFile actions-runner-win.zip +Add-Type -AssemblyName System.IO.Compression.FileSystem ; +[System.IO.Compression.ZipFile]::ExtractToDirectory("$PWD\actions-runner-win.zip", "$PWD") +./config.cmd --token $ghToken --url %s --name %s --unattended --runasservice --replace` + +// whitespace at the start is required since this is expanded in a cloud-init yaml file +// to start as service need to relable the runsvc.sh file on rhel: https://github.com/actions/runner/issues/3222 +const LinuxActionsRunnerInstallSnippet string = ` mkdir ~/actions-runner && cd ~/actions-runner` + "\n" + + ` curl -o actions-runner-linux.tar.gz -L %s` + "\n" + + ` tar xzf ./actions-runner-linux.tar.gz` + "\n" + + ` sudo ./bin/installdependencies.sh` + "\n" + + ` ./config.sh --token %s --url %s --name %s --unattended --replace` + "\n" + + ` sudo ./svc.sh install` + "\n" + + ` chcon system_u:object_r:usr_t:s0 $(pwd)/runsvc.sh` + "\n" + + ` sudo ./svc.sh start` + +func GetActionRunnerSnippetWin() string { + if (args == &RunnerArgs{}) { + return "" + } + return fmt.Sprintf(WindowsActionsRunnerInstallSnippet, + getRunnerDownloadURLWin(), args.RepoURL, args.Name) +} + +func GetActionRunnerSnippetLinux() string { + if (args == &RunnerArgs{}) { + return "" + } + return fmt.Sprintf(LinuxActionsRunnerInstallSnippet, + getRunnerDownloadURLLinux(), args.Token, args.RepoURL, args.Name) +} diff --git a/pkg/util/logging/logging.go b/pkg/util/logging/logging.go index a0ec9dfa0..4a5824070 100644 --- a/pkg/util/logging/logging.go +++ b/pkg/util/logging/logging.go @@ -48,7 +48,10 @@ func InitLogrus(basePath, fileName string) { // logrus.SetOutput(io.MultiWriter(logfile, os.Stdout)) logrus.SetOutput(io.MultiWriter(os.Stdout)) logrus.SetLevel(logrus.DebugLevel) - logrus.SetFormatter(&logrus.JSONFormatter{}) + logrus.SetFormatter(&logrus.TextFormatter{ + ForceColors: true, + DisableTimestamp: true, + }) for k, v := range logrus.StandardLogger().Hooks { originalHooks[k] = v