Merge branch 'main' into INS-79-jira-cloud-on-prem-autodetection

Author: shahzadhaider1

README.md

@@ -499,7 +499,7 @@ trufflehog git https://github.com/trufflesecurity/trufflehog.git
 
 ## Configuration
 
-TruffleHog supports defining [custom regex detectors](#regex-detector-alpha)
+TruffleHog supports defining [custom regex detectors](#custom-regex-detector-alpha)
 and multiple sources in a configuration file provided via the `--config` flag.
 The regex detectors can be used with any subcommand, while the sources defined
 in configuration are only for the `multi-scan` subcommand.

pkg/engine/engine.go

@@ -1170,6 +1170,12 @@ func (e *Engine) filterResults(
 	return results
 }
 
+// processResult generates a detectors.ResultWithMetadata from the provided chunk and result and puts it on the results
+// channel, unless the result exists on a line with an ignore tag, in which case no result is generated.
+//
+// CMR: The provided chunk is wrapped in a detectableChunk, but I'm pretty sure that's purely out of convenience
+// (because that's what this function's callers are using when they call this function). We're past detection at this
+// point in the engine, so we should probably refactor that parameter into a less confusing data type.
 func (e *Engine) processResult(
 	ctx context.Context,
 	data detectableChunk,

pkg/engine/engine_test.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/config"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
@@ -26,6 +27,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/verificationcache"
 )
 
 const fakeDetectorKeyword = "fakedetector"
@@ -1318,3 +1320,223 @@ def test_something():
 		})
 	}
 }
+
+type passthroughDetector struct {
+	detectorType detectorspb.DetectorType
+	keywords     []string
+	secret       string
+}
+
+func (p passthroughDetector) FromData(_ aCtx.Context, verify bool, data []byte) ([]detectors.Result, error) {
+	raw := data
+	if p.secret != "" {
+		raw = []byte(p.secret)
+	}
+	return []detectors.Result{
+		{
+			Raw:      raw,
+			Verified: verify,
+		},
+	}, nil
+}
+
+func (p passthroughDetector) Keywords() []string             { return p.keywords }
+func (p passthroughDetector) Type() detectorspb.DetectorType { return p.detectorType }
+func (p passthroughDetector) Description() string            { return "fake detector for testing" }
+
+type passthroughDecoder struct{}
+
+func (p passthroughDecoder) FromChunk(chunk *sources.Chunk) *decoders.DecodableChunk {
+	return &decoders.DecodableChunk{
+		Chunk:       chunk,
+		DecoderType: detectorspb.DecoderType(-1),
+	}
+}
+
+func (p passthroughDecoder) Type() detectorspb.DecoderType { return detectorspb.DecoderType(-1) }
+
+func TestEngine_DetectChunk_UsesVerifyFlag(t *testing.T) {
+	ctx := context.Background()
+
+	// Arrange: Create a minimal engine.
+	e := &Engine{
+		results:           make(chan detectors.ResultWithMetadata, 1),
+		verificationCache: verificationcache.New(nil, &verificationcache.InMemoryMetrics{}),
+	}
+
+	// Arrange: Create a detector match. We can't create one directly, so we have to use a minimal A-H core.
+	ahcore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{passthroughDetector{keywords: []string{"keyword"}}})
+	detectorMatches := ahcore.FindDetectorMatches([]byte("keyword"))
+	require.Len(t, detectorMatches, 1)
+
+	// Arrange: Create a chunk to detect.
+	chunk := detectableChunk{
+		chunk: sources.Chunk{
+			Verify: true,
+		},
+		detector: detectorMatches[0],
+		wgDoneFn: func() {},
+	}
+
+	// Act
+	e.detectChunk(ctx, chunk)
+	close(e.results)
+
+	// Assert: Confirm that a result was generated and that it has the expected verify flag.
+	select {
+	case result := <-e.results:
+		assert.True(t, result.Result.Verified)
+	default:
+		t.Errorf("expected a result but did not get one")
+	}
+}
+
+func TestEngine_ScannerWorker_DetectableChunkHasCorrectVerifyFlag(t *testing.T) {
+	ctx := context.Background()
+
+	// Arrange: Create a minimal engine.
+	detector := &passthroughDetector{keywords: []string{"keyword"}}
+	e := &Engine{
+		AhoCorasickCore:      ahocorasick.NewAhoCorasickCore([]detectors.Detector{detector}),
+		decoders:             []decoders.Decoder{passthroughDecoder{}},
+		detectableChunksChan: make(chan detectableChunk, 1),
+		sourceManager:        sources.NewManager(),
+		verify:               true,
+	}
+
+	// Arrange: Create a chunk to scan.
+	chunk := sources.Chunk{
+		Data:   []byte("keyword"),
+		Verify: true,
+	}
+
+	// Arrange: Enqueue a chunk to be scanned.
+	e.sourceManager.ScanChunk(&chunk)
+
+	// Act
+	go e.scannerWorker(ctx)
+
+	// Assert: Confirm that a chunk was generated and that it has the expected verify flag.
+	select {
+	case chunk := <-e.detectableChunksChan:
+		assert.True(t, chunk.chunk.Verify)
+	case <-time.After(1 * time.Second):
+		t.Errorf("expected a detectableChunk but did not get one")
+	}
+}
+
+func TestEngine_VerificationOverlapWorker_DetectableChunkHasCorrectVerifyFlag(t *testing.T) {
+	ctx := context.Background()
+
+	t.Run("overlap", func(t *testing.T) {
+		// Arrange: Create a minimal engine
+		e := &Engine{
+			detectableChunksChan:          make(chan detectableChunk, 2),
+			results:                       make(chan detectors.ResultWithMetadata, 2),
+			retainFalsePositives:          true,
+			verificationOverlapChunksChan: make(chan verificationOverlapChunk, 2),
+			verify:                        true,
+		}
+
+		// Arrange: Set up a fake detectableChunk processor so that any chunks (incorrectly) sent to
+		// e.detectableChunksChan don't block the test.
+		processedDetectableChunks := make(chan detectableChunk, 2)
+		go func() {
+			for chunk := range e.detectableChunksChan {
+				chunk.wgDoneFn()
+				processedDetectableChunks <- chunk
+			}
+		}()
+
+		// Arrange: Create a chunk to "scan."
+		chunk := sources.Chunk{
+			Data:   []byte("keyword ;oahpow8heg;blaisd"),
+			Verify: true,
+		}
+
+		// Arrange: Create overlapping detector matches. We can't create them directly, so we have to use a minimal A-H
+		// core.
+		ahcore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{
+			passthroughDetector{detectorType: detectorspb.DetectorType(-1), keywords: []string{"keyw"}},
+			passthroughDetector{detectorType: detectorspb.DetectorType(-2), keywords: []string{"keyword"}},
+		})
+		detectorMatches := ahcore.FindDetectorMatches(chunk.Data)
+		require.Len(t, detectorMatches, 2)
+
+		// Arrange: Enqueue a verification overlap chunk
+		e.verificationOverlapChunksChan <- verificationOverlapChunk{
+			chunk:                       chunk,
+			detectors:                   detectorMatches,
+			verificationOverlapWgDoneFn: func() { close(e.verificationOverlapChunksChan) },
+		}
+
+		// Act
+		e.verificationOverlapWorker(ctx)
+		close(e.results)
+		close(e.detectableChunksChan)
+		close(processedDetectableChunks)
+
+		// Assert: Confirm that every generated result is unverified (because overlap detection precluded it).
+		for result := range e.results {
+			assert.False(t, result.Result.Verified)
+		}
+
+		// Assert: Confirm that every generated detectable chunk carries the original Verify flag.
+		// CMR: There should be not be any of these chunks. However, due to what I believe is an unrelated bug, there
+		// are. This test ensures that even in that erroneous case, their Verify flag is correct.
+		for detectableChunk := range processedDetectableChunks {
+			assert.True(t, detectableChunk.chunk.Verify)
+		}
+	})
+	t.Run("no overlap", func(t *testing.T) {
+		// Arrange: Create a minimal engine
+		e := &Engine{
+			detectableChunksChan:          make(chan detectableChunk, 2),
+			retainFalsePositives:          true,
+			verificationOverlapChunksChan: make(chan verificationOverlapChunk, 2),
+			verify:                        true,
+		}
+
+		// Arrange: Set up a fake detectableChunk processor so that any chunks sent to e.detectableChunksChan don't
+		// block the test.
+		processedDetectableChunks := make(chan detectableChunk, 2)
+		go func() {
+			for chunk := range e.detectableChunksChan {
+				chunk.wgDoneFn()
+				processedDetectableChunks <- chunk
+			}
+		}()
+
+		// Arrange: Create a chunk to "scan."
+		chunk := sources.Chunk{
+			Data:   []byte("keyword ;oahpow8heg;blaisd"),
+			Verify: true,
+		}
+
+		// Arrange: Create non-overlapping detector matches. We can't create them directly, so we have to use a minimal
+		// A-H core.
+		ahcore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{
+			passthroughDetector{detectorType: detectorspb.DetectorType(-1), keywords: []string{"keyw"}, secret: "oahpow"},
+			passthroughDetector{detectorType: detectorspb.DetectorType(-2), keywords: []string{"keyword"}, secret: "blaisd"},
+		})
+		detectorMatches := ahcore.FindDetectorMatches(chunk.Data)
+		require.Len(t, detectorMatches, 2)
+
+		// Arrange: Enqueue a verification overlap chunk
+		e.verificationOverlapChunksChan <- verificationOverlapChunk{
+			chunk:                       chunk,
+			detectors:                   detectorMatches,
+			verificationOverlapWgDoneFn: func() { close(e.verificationOverlapChunksChan) },
+		}
+
+		// Act
+		e.verificationOverlapWorker(ctx)
+		close(e.detectableChunksChan)
+		close(processedDetectableChunks)
+
+		// Assert: Confirm that every generated detectable chunk carries the original Verify flag.
+		for detectableChunk := range processedDetectableChunks {
+			assert.True(t, detectableChunk.chunk.Verify)
+		}
+	})
+}