Merge pull request #74 from buildkite-plugins/SUP-3346-env-metadata-attributes

SUP-3346 Add the env and metadata attributes

Author: mcncl

README.md

@@ -46,6 +46,7 @@ steps:
   - label: "Triggering pipelines"
     plugins:
       - monorepo-diff#v1.4.0:
+          diff: "git diff --name-only HEAD~1"
           watch:
             - path: "**/*"
               skip_path: "folder/file"
@@ -101,7 +102,32 @@ steps:
 
 ⚠️ Warning : The user has to explictly state the paths they want to monitor or use wildcards. For instance if a user, is only watching path `app/` changes made to `app/bin` will not trigger the configuration. This is because the subfolder `/bin` was not specified.
 
-### `diff` (optional)
+**Example**
+
+```yaml
+steps:
+  - label: "Triggering pipelines with plugin"
+    plugins:
+      - monorepo-diff#v1.4.0:
+          watch:
+            - path: test/.buildkite/
+              config: # Required [trigger step configuration]
+                trigger: test-pipeline # Required [trigger pipeline slug]
+            - path:
+                - app/
+                - app/bin/service/
+              config:
+                trigger: "data-generator"
+                label: ":package: Generate data"
+                build:
+                  meta_data:
+                    release-version: "1.1"
+```
+
+- When changes are detected in the path `test/.buildkite/` it triggers the pipeline `test-pipeline`
+- If the changes are made to either `app/` or `app/bin/service/` it triggers the pipeline `data-generator`
+
+#### `diff` (optional)
 
 This will run the script provided to determine the folder changes.
 Depending on your use case, you may want to determine the point where the branch occurs
@@ -140,7 +166,24 @@ LATEST_BUILT_TAG=$(git describe --tags --match foo-service-* --abbrev=0)
 git diff --name-only "$LATEST_TAG"
 ```
 
-### `interpolation` (optional)
+**Example**
+
+```yaml
+steps:
+  - label: "Triggering pipelines"
+    plugins:
+      - monorepo-diff#v1.4.0:
+          diff: "git diff --name-only HEAD~1"
+          watch:
+            - path: "bar-service/"
+              config:
+                command: "echo deploy-bar"
+            - path: "foo-service/"
+              config:
+                trigger: "deploy-foo-service"
+```
+
+#### `interpolation` (optional)
 
 This controls the pipeline interpolation on upload, and defaults to `true`.
 If set to `false` it adds `--no-interpolation` to the `buildkite pipeline upload`,
@@ -157,6 +200,7 @@ steps:
   - label: "Triggering pipelines"
     plugins:
       - monorepo-diff#v1.4.0:
+          diff: "git diff --name-only HEAD~1"
           watch:
             - path: "bar-service/"
               config:
@@ -178,6 +222,7 @@ steps:
   - label: "Triggering pipelines"
     plugins:
       - monorepo-diff#v1.4.0:
+          diff: "git diff --name-only HEAD~1"
           watch:
             - path: "foo-service/"
               config:
@@ -199,6 +244,7 @@ steps:
   - label: "Triggering pipelines"
     plugins:
       - monorepo-diff#v1.4.0:
+          diff: "git diff --name-only HEAD~1"
           log_level: "debug" # defaults to "info"
           watch:
             - path: "foo-service/"

main_test.go

@@ -12,12 +12,13 @@ func TestMain(m *testing.M) {
 	log.SetLevel(log.DebugLevel)
 
 	// set some env variables for using in tests
-	os.Setenv("BUILDKITE_COMMIT", "123")
-	os.Setenv("BUILDKITE_MESSAGE", "fix: temp file not correctly deleted")
-	os.Setenv("BUILDKITE_BRANCH", "go-rewrite")
-	os.Setenv("env3", "env-3")
-	os.Setenv("env4", "env-4")
-	os.Setenv("TEST_MODE", "true")
+	// given this is a test we can use `_` for the return; our tests will check values
+	_ = os.Setenv("BUILDKITE_COMMIT", "123")
+	_ = os.Setenv("BUILDKITE_MESSAGE", "fix: temp file not correctly deleted")
+	_ = os.Setenv("BUILDKITE_BRANCH", "go-rewrite")
+	_ = os.Setenv("env3", "env-3")
+	_ = os.Setenv("env4", "env-4")
+	_ = os.Setenv("TEST_MODE", "true")
 
 	run := m.Run()
 

pipeline.go

@@ -66,12 +66,14 @@ func uploadPipeline(plugin Plugin, generatePipeline PipelineGenerator) (string,
 	}
 
 	pipeline, hasSteps, err := generatePipeline(steps, plugin)
-	defer os.Remove(pipeline.Name())
-
 	if err != nil {
-		log.Error(err)
 		return "", []string{}, err
 	}
+	defer func() {
+		if removeErr := os.Remove(pipeline.Name()); removeErr != nil {
+			log.Errorf("Failed to remove temporary pipeline file: %v", removeErr)
+		}
+	}()
 
 	if !hasSteps {
 		// Handle the case where no steps were provided
@@ -96,7 +98,7 @@ func diff(command string) ([]string, error) {
 
 	output, err := executeCommand(
 		env("SHELL", "bash"),
-		[]string{"-c", strings.Replace(command, "\n", " ", -1)},
+		[]string{"-c", strings.ReplaceAll(command, "\n", " ")},
 	)
 	if err != nil {
 		return nil, fmt.Errorf("diff command failed: %v", err)

pipeline_test.go

@@ -12,7 +12,9 @@ import (
 
 func mockGeneratePipeline(steps []Step, plugin Plugin) (*os.File, bool, error) {
 	mockFile, _ := os.Create("pipeline.txt")
-	defer mockFile.Close()
+	defer func() {
+		_ = mockFile.Close()
+	}()
 
 	return mockFile, true, nil
 }
@@ -39,7 +41,9 @@ func TestUploadPipelineCallsBuildkiteAgentCommand(t *testing.T) {
 	assert.Equal(t, []string{"pipeline", "upload", "pipeline.txt"}, args)
 	assert.NoError(t, err)
 
-	agent.CheckAndClose(t)
+	if err := agent.CheckAndClose(t); err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestUploadPipelineCallsBuildkiteAgentCommandWithInterpolation(t *testing.T) {
@@ -64,7 +68,9 @@ func TestUploadPipelineCallsBuildkiteAgentCommandWithInterpolation(t *testing.T)
 	assert.Equal(t, []string{"pipeline", "upload", "pipeline.txt", "--no-interpolation"}, args)
 	assert.NoError(t, err)
 
-	agent.CheckAndClose(t)
+	if err := agent.CheckAndClose(t); err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestUploadPipelineCancelsIfThereIsNoDiffOutput(t *testing.T) {
@@ -298,7 +304,8 @@ func TestPipelinesStepsToTrigger(t *testing.T) {
 					Paths:     []string{"watch-path"},
 					SkipPaths: []string{"watch-path/text.txt"},
 					Step:      Step{Trigger: "service-2"},
-				}},
+				},
+			},
 			Expected: []Step{
 				{Trigger: "service-1"},
 			},
@@ -316,7 +323,8 @@ func TestPipelinesStepsToTrigger(t *testing.T) {
 					Paths:     []string{"*.txt"},
 					SkipPaths: []string{"*.secret.txt"},
 					Step:      Step{Trigger: "service-2"},
-				}},
+				},
+			},
 			Expected: []Step{
 				{Trigger: "service-1"},
 			},
@@ -334,7 +342,8 @@ func TestPipelinesStepsToTrigger(t *testing.T) {
 					Paths:     []string{"**/*.txt"},
 					SkipPaths: []string{"docs/*.txt"},
 					Step:      Step{Trigger: "service-2"},
-				}},
+				},
+			},
 			Expected: []Step{
 				{Trigger: "service-1"},
 			},
@@ -353,7 +362,8 @@ func TestPipelinesStepsToTrigger(t *testing.T) {
 					Paths:     []string{"**/*.txt"},
 					SkipPaths: []string{"docs/*.secret.txt"},
 					Step:      Step{Trigger: "service-2"},
-				}},
+				},
+			},
 			Expected: []Step{
 				{Trigger: "service-1"},
 				{Trigger: "service-2"},
@@ -367,7 +377,8 @@ func TestPipelinesStepsToTrigger(t *testing.T) {
 				{
 					SkipPaths: []string{"docs/*.secret.txt"},
 					Step:      Step{Trigger: "service-1"},
-				}},
+				},
+			},
 			Expected: []Step{},
 		},
 		"step is not included if except path is set": {
@@ -447,7 +458,11 @@ func TestGeneratePipeline(t *testing.T) {
 	pipeline, _, err := generatePipeline(steps, plugin)
 
 	require.NoError(t, err)
-	defer os.Remove(pipeline.Name())
+	defer func() {
+		if err := os.Remove(pipeline.Name()); err != nil {
+			t.Logf("Failed to remove temporary pipeline file: %v", err)
+		}
+	}()
 
 	got, err := os.ReadFile(pipeline.Name())
 	require.NoError(t, err)

plugin.go

@@ -3,6 +3,7 @@ package main
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"net/url"
 	"path"
 	"strings"
@@ -22,6 +23,7 @@ type Plugin struct {
 	Watch         []WatchConfig
 	RawEnv        interface{} `json:"env"`
 	Env           map[string]string
+	Metadata      map[string]string        `json:"meta_data"`
 	RawNotify     []map[string]interface{} `json:"notify" yaml:",omitempty"`
 	Notify        []PluginNotify           `yaml:"notify,omitempty"`
 }
@@ -98,11 +100,12 @@ type Agent map[string]string
 
 // Build is buildkite build definition
 type Build struct {
-	Message string            `yaml:"message,omitempty"`
-	Branch  string            `yaml:"branch,omitempty"`
-	Commit  string            `yaml:"commit,omitempty"`
-	RawEnv  interface{}       `json:"env" yaml:",omitempty"`
-	Env     map[string]string `yaml:"env,omitempty"`
+	Message  string            `yaml:"message,omitempty"`
+	Branch   string            `yaml:"branch,omitempty"`
+	Commit   string            `yaml:"commit,omitempty"`
+	RawEnv   interface{}       `json:"env" yaml:",omitempty"`
+	Env      map[string]string `yaml:"env,omitempty"`
+	Metadata map[string]string `json:"meta_data" yaml:"meta_data,omitempty"`
 	// Notify  []Notify          `yaml:"notify,omitempty"`
 }
 
@@ -129,6 +132,13 @@ func (plugin *Plugin) UnmarshalJSON(data []byte) error {
 	plugin.Env = parseResult
 	plugin.RawEnv = nil
 
+	metaDataParseResult, err := parseMetadata(plugin.Metadata)
+	if err != nil {
+		return errors.New("failed to parse metadata configuration")
+	}
+
+	plugin.Metadata = metaDataParseResult
+
 	setPluginNotify(&plugin.Notify, &plugin.RawNotify)
 
 	for i, p := range plugin.Watch {
@@ -189,6 +199,14 @@ func (plugin *Plugin) UnmarshalJSON(data []byte) error {
 
 		appendEnv(&plugin.Watch[i], plugin.Env)
 
+		// Attempt to parse the metadata after the env's
+		parsedMetadata, err := parseMetadata(plugin.Metadata)
+		if err != nil {
+			return errors.New("failed to parse metadata configuration")
+		}
+
+		appendMetadata(&plugin.Watch[i], parsedMetadata)
+
 		p.RawPath = nil
 		p.RawSkipPath = nil
 	}
@@ -357,6 +375,22 @@ func appendEnv(watch *WatchConfig, env map[string]string) {
 	watch.RawSkipPath = nil
 }
 
+// appends build metadata
+func appendMetadata(watch *WatchConfig, metadata map[string]string) {
+	if len(metadata) == 0 {
+		return
+	}
+	// Only apply metadata to trigger steps, not command steps
+	if watch.Step.Trigger != "" {
+		if watch.Step.Build.Metadata == nil {
+			watch.Step.Build.Metadata = make(map[string]string)
+		}
+		for k, v := range metadata {
+			watch.Step.Build.Metadata[k] = v
+		}
+	}
+}
+
 // parse env in format from env=env-value to map[env] = env-value
 func parseEnv(raw interface{}) (map[string]string, error) {
 	if raw == nil {
@@ -385,6 +419,44 @@ func parseEnv(raw interface{}) (map[string]string, error) {
 	return result, nil
 }
 
+// parse metadata in format from key:value to map[key] = value
+func parseMetadata(raw interface{}) (map[string]string, error) {
+	if raw == nil {
+		return nil, nil
+	}
+
+	switch v := raw.(type) {
+	case map[string]string:
+		return v, nil
+	case map[string]interface{}:
+		result := make(map[string]string)
+		for k, val := range v {
+			result[k] = fmt.Sprintf("%v", val)
+		}
+		return result, nil
+	case []interface{}:
+		result := make(map[string]string)
+		for _, item := range v {
+			str, ok := item.(string)
+			if !ok {
+				continue
+			}
+			split := strings.SplitN(str, ":", 2)
+			key := strings.TrimSpace(split[0])
+			value := ""
+			if len(split) > 1 {
+				value = strings.TrimSpace(split[1])
+			}
+			if key != "" {
+				result[key] = value
+			}
+		}
+		return result, nil
+	default:
+		return nil, errors.New("failed to parse metadata configuration: unknown type")
+	}
+}
+
 func getPluginName(s string) string {
 	ref := s
 	if strings.HasPrefix(ref, "github.com/") && !strings.Contains(ref, "://") {

plugin.yml

@@ -69,6 +69,8 @@ configuration:
             build:
               type: object
               properties:
+                meta_data:
+                  type: array
                 message:
                   type: string
                 commit:
@@ -97,5 +99,4 @@ configuration:
           type: string
   required:
     - watch
-
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite-plugins/buildkite-plugin-linter/master/lib/plugin-yaml-schema.yml