Skip to content

Commit eff52eb

Browse files
DmitriyLewenCopilot
DmitriyLewen
and
Copilot
authored
feat(java): add support remote repositories from settings.xml files (#9708)
Co-authored-by: Copilot <[email protected]>
1 parent fb0593b commit eff52eb

File tree

10 files changed

+714
-153
lines changed

10 files changed

+714
-153
lines changed

‎pkg/dependency/parser/java/pom/parse.go‎

Lines changed: 79 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,10 @@ import (
2929
xio "github.com/aquasecurity/trivy/pkg/x/io"
3030
)
3131

32-
const (
33-
centralURL = "https://repo.maven.apache.org/maven2/"
34-
)
35-
3632
type options struct {
37-
offline bool
38-
releaseRemoteRepos []string
39-
snapshotRemoteRepos []string
33+
offline bool
34+
defaultRepo repository
35+
settingsRepos []repository
4036
}
4137

4238
type option func(*options)
@@ -47,55 +43,71 @@ func WithOffline(offline bool) option {
4743
}
4844
}
4945

50-
func WithReleaseRemoteRepos(repos []string) option {
46+
func WithDefaultRepo(repoURL string, releaseEnabled, snapshotEnabled bool) option {
5147
return func(opts *options) {
52-
opts.releaseRemoteRepos = repos
48+
u, _ := url.Parse(repoURL)
49+
opts.defaultRepo = repository{
50+
url: *u,
51+
releaseEnabled: releaseEnabled,
52+
snapshotEnabled: snapshotEnabled,
53+
}
5354
}
5455
}
5556

56-
func WithSnapshotRemoteRepos(repos []string) option {
57+
func WithSettingsRepos(repoURLs []string, releaseEnabled, snapshotEnabled bool) option {
5758
return func(opts *options) {
58-
opts.snapshotRemoteRepos = repos
59+
opts.settingsRepos = lo.Map(repoURLs, func(repoURL string, _ int) repository {
60+
u, _ := url.Parse(repoURL)
61+
return repository{
62+
url: *u,
63+
releaseEnabled: releaseEnabled,
64+
snapshotEnabled: snapshotEnabled,
65+
}
66+
})
5967
}
6068
}
6169

6270
type Parser struct {
63-
logger *log.Logger
64-
rootPath string
65-
cache pomCache
66-
localRepository string
67-
releaseRemoteRepos []string
68-
snapshotRemoteRepos []string
69-
offline bool
70-
servers []Server
71+
logger *log.Logger
72+
rootPath string
73+
cache pomCache
74+
localRepository string
75+
remoteRepos repositories
76+
offline bool
77+
servers []Server
7178
}
7279

7380
func NewParser(filePath string, opts ...option) *Parser {
7481
o := &options{
75-
offline: false,
76-
releaseRemoteRepos: []string{centralURL}, // Maven doesn't use central repository for snapshot dependencies
77-
}
78-
79-
for _, opt := range opts {
80-
opt(o)
82+
offline: false,
83+
defaultRepo: mavenCentralRepo,
8184
}
8285

8386
s := readSettings()
87+
o.settingsRepos = s.effectiveRepositories()
8488
localRepository := s.LocalRepository
8589
if localRepository == "" {
8690
homeDir, _ := os.UserHomeDir()
8791
localRepository = filepath.Join(homeDir, ".m2", "repository")
8892
}
8993

94+
for _, opt := range opts {
95+
opt(o)
96+
}
97+
98+
remoteRepos := repositories{
99+
defaultRepo: o.defaultRepo,
100+
settings: o.settingsRepos,
101+
}
102+
90103
return &Parser{
91-
logger: log.WithPrefix("pom"),
92-
rootPath: filepath.Clean(filePath),
93-
cache: newPOMCache(),
94-
localRepository: localRepository,
95-
releaseRemoteRepos: o.releaseRemoteRepos,
96-
snapshotRemoteRepos: o.snapshotRemoteRepos,
97-
offline: o.offline,
98-
servers: s.Servers,
104+
logger: log.WithPrefix("pom"),
105+
rootPath: filepath.Clean(filePath),
106+
cache: newPOMCache(),
107+
localRepository: localRepository,
108+
remoteRepos: remoteRepos,
109+
offline: o.offline,
110+
servers: s.Servers,
99111
}
100112
}
101113

@@ -362,9 +374,10 @@ func (p *Parser) analyze(ctx context.Context, pom *pom, opts analysisOptions) (a
362374
opts.exclusions = set.New[string]()
363375
}
364376
// Update remoteRepositories
365-
pomReleaseRemoteRepos, pomSnapshotRemoteRepos := pom.repositories(p.servers)
366-
p.releaseRemoteRepos = lo.Uniq(append(pomReleaseRemoteRepos, p.releaseRemoteRepos...))
367-
p.snapshotRemoteRepos = lo.Uniq(append(pomSnapshotRemoteRepos, p.snapshotRemoteRepos...))
377+
pomRepos := pom.repositories(p.servers)
378+
p.remoteRepos.pom = lo.UniqBy(append(pomRepos, p.remoteRepos.pom...), func(r repository) url.URL {
379+
return r.url
380+
})
368381

369382
// Resolve parent POM
370383
if err := p.resolveParent(ctx, pom); err != nil {
@@ -710,17 +723,19 @@ func (p *Parser) fetchPOMFromRemoteRepositories(ctx context.Context, paths []str
710723
return nil, xerrors.New("offline mode")
711724
}
712725

713-
remoteRepos := p.releaseRemoteRepos
714-
// Maven uses only snapshot repos for snapshot artifacts
715-
if snapshot {
716-
remoteRepos = p.snapshotRemoteRepos
717-
}
726+
// Try all remoteRepositories by following order:
727+
// 1. remoteRepositories from settings.xml
728+
// 2. remoteRepositories from pom.xml
729+
// 3. default remoteRepository (Maven Central for Release repository)
730+
for _, repo := range slices.Concat(p.remoteRepos.settings, p.remoteRepos.pom, []repository{p.remoteRepos.defaultRepo}) {
731+
// Skip Release only repositories for snapshot artifacts and vice versa
732+
if snapshot && !repo.snapshotEnabled || !snapshot && !repo.releaseEnabled {
733+
continue
734+
}
718735

719-
// try all remoteRepositories
720-
for _, repo := range remoteRepos {
721736
repoPaths := slices.Clone(paths) // Clone slice to avoid overwriting last element of `paths`
722737
if snapshot {
723-
pomFileName, err := p.fetchPomFileNameFromMavenMetadata(ctx, repo, repoPaths)
738+
pomFileName, err := p.fetchPomFileNameFromMavenMetadata(ctx, repo.url, repoPaths)
724739
if err != nil {
725740
return nil, xerrors.Errorf("fetch maven-metadata.xml error: %w", err)
726741
}
@@ -729,7 +744,7 @@ func (p *Parser) fetchPOMFromRemoteRepositories(ctx context.Context, paths []str
729744
repoPaths[len(repoPaths)-1] = pomFileName
730745
}
731746
}
732-
fetched, err := p.fetchPOMFromRemoteRepository(ctx, repo, repoPaths)
747+
fetched, err := p.fetchPOMFromRemoteRepository(ctx, repo.url, repoPaths)
733748
if err != nil {
734749
return nil, xerrors.Errorf("fetch repository error: %w", err)
735750
} else if fetched == nil {
@@ -740,12 +755,7 @@ func (p *Parser) fetchPOMFromRemoteRepositories(ctx context.Context, paths []str
740755
return nil, xerrors.Errorf("the POM was not found in remote remoteRepositories")
741756
}
742757

743-
func (p *Parser) remoteRepoRequest(ctx context.Context, repo string, paths []string) (*http.Request, error) {
744-
repoURL, err := url.Parse(repo)
745-
if err != nil {
746-
return nil, xerrors.Errorf("unable to parse URL: %w", err)
747-
}
748-
758+
func (p *Parser) remoteRepoRequest(ctx context.Context, repoURL url.URL, paths []string) (*http.Request, error) {
749759
paths = append([]string{repoURL.Path}, paths...)
750760
repoURL.Path = path.Join(paths...)
751761

@@ -762,14 +772,14 @@ func (p *Parser) remoteRepoRequest(ctx context.Context, repo string, paths []str
762772
}
763773

764774
// fetchPomFileNameFromMavenMetadata fetches `maven-metadata.xml` file to detect file name of pom file.
765-
func (p *Parser) fetchPomFileNameFromMavenMetadata(ctx context.Context, repo string, paths []string) (string, error) {
775+
func (p *Parser) fetchPomFileNameFromMavenMetadata(ctx context.Context, repoURL url.URL, paths []string) (string, error) {
766776
// Overwrite pom file name to `maven-metadata.xml`
767777
mavenMetadataPaths := slices.Clone(paths[:len(paths)-1]) // Clone slice to avoid shadow overwriting last element of `paths`
768778
mavenMetadataPaths = append(mavenMetadataPaths, "maven-metadata.xml")
769779

770-
req, err := p.remoteRepoRequest(ctx, repo, mavenMetadataPaths)
780+
req, err := p.remoteRepoRequest(ctx, repoURL, mavenMetadataPaths)
771781
if err != nil {
772-
p.logger.Debug("Unable to create request", log.String("repo", repo), log.Err(err))
782+
p.logger.Debug("Unable to create request", log.String("repo", repoURL.Redacted()), log.Err(err))
773783
return "", nil
774784
}
775785

@@ -779,14 +789,16 @@ func (p *Parser) fetchPomFileNameFromMavenMetadata(ctx context.Context, repo str
779789
if shouldReturnError(err) {
780790
return "", err
781791
}
782-
p.logger.Debug("Failed to fetch", log.String("url", req.URL.String()), log.Err(err))
783-
return "", nil
784-
} else if resp.StatusCode != http.StatusOK {
785-
p.logger.Debug("Failed to fetch", log.String("url", req.URL.String()), log.Int("statusCode", resp.StatusCode))
792+
p.logger.Debug("Failed to fetch", log.String("url", req.URL.Redacted()), log.Err(err))
786793
return "", nil
787794
}
788795
defer resp.Body.Close()
789796

797+
if resp.StatusCode != http.StatusOK {
798+
p.logger.Debug("Failed to fetch", log.String("url", req.URL.Redacted()), log.Int("statusCode", resp.StatusCode))
799+
return "", nil
800+
}
801+
790802
mavenMetadata, err := parseMavenMetadata(resp.Body)
791803
if err != nil {
792804
return "", xerrors.Errorf("failed to parse maven-metadata.xml file: %w", err)
@@ -803,10 +815,10 @@ func (p *Parser) fetchPomFileNameFromMavenMetadata(ctx context.Context, repo str
803815
return pomFileName, nil
804816
}
805817

806-
func (p *Parser) fetchPOMFromRemoteRepository(ctx context.Context, repo string, paths []string) (*pom, error) {
807-
req, err := p.remoteRepoRequest(ctx, repo, paths)
818+
func (p *Parser) fetchPOMFromRemoteRepository(ctx context.Context, repoURL url.URL, paths []string) (*pom, error) {
819+
req, err := p.remoteRepoRequest(ctx, repoURL, paths)
808820
if err != nil {
809-
p.logger.Debug("Unable to create request", log.String("repo", repo), log.Err(err))
821+
p.logger.Debug("Unable to create request", log.String("repo", repoURL.Redacted()), log.Err(err))
810822
return nil, nil
811823
}
812824

@@ -816,14 +828,16 @@ func (p *Parser) fetchPOMFromRemoteRepository(ctx context.Context, repo string,
816828
if shouldReturnError(err) {
817829
return nil, err
818830
}
819-
p.logger.Debug("Failed to fetch", log.String("url", req.URL.String()), log.Err(err))
820-
return nil, nil
821-
} else if resp.StatusCode != http.StatusOK {
822-
p.logger.Debug("Failed to fetch", log.String("url", req.URL.String()), log.Int("statusCode", resp.StatusCode))
831+
p.logger.Debug("Failed to fetch", log.String("url", req.URL.Redacted()), log.Err(err))
823832
return nil, nil
824833
}
825834
defer resp.Body.Close()
826835

836+
if resp.StatusCode != http.StatusOK {
837+
p.logger.Debug("Failed to fetch", log.String("url", req.URL.Redacted()), log.Int("statusCode", resp.StatusCode))
838+
return nil, nil
839+
}
840+
827841
content, err := parsePom(resp.Body, false)
828842
if err != nil {
829843
return nil, xerrors.Errorf("failed to parse the remote POM: %w", err)

‎pkg/dependency/parser/java/pom/parse_test.go‎

Lines changed: 69 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -110,13 +110,14 @@ var (
110110

111111
func TestPom_Parse(t *testing.T) {
112112
tests := []struct {
113-
name string
114-
inputFile string
115-
local bool
116-
offline bool
117-
want []ftypes.Package
118-
wantDeps []ftypes.Dependency
119-
wantErr string
113+
name string
114+
inputFile string
115+
local bool
116+
enableRepoForSettingsRepo bool // use another repo for repository from settings.xml
117+
offline bool
118+
want []ftypes.Package
119+
wantDeps []ftypes.Dependency
120+
wantErr string
120121
}{
121122
{
122123
name: "local repository",
@@ -326,6 +327,55 @@ func TestPom_Parse(t *testing.T) {
326327
},
327328
},
328329
},
330+
{
331+
name: "multiple repositories are used",
332+
inputFile: filepath.Join("testdata", "happy", "pom.xml"),
333+
local: false,
334+
enableRepoForSettingsRepo: true,
335+
want: []ftypes.Package{
336+
{
337+
ID: "com.example:happy:1.0.0",
338+
Name: "com.example:happy",
339+
Version: "1.0.0",
340+
Licenses: []string{"BSD-3-Clause"},
341+
Relationship: ftypes.RelationshipRoot,
342+
},
343+
{
344+
ID: "org.example:example-api:1.7.30",
345+
Name: "org.example:example-api",
346+
Version: "1.7.30",
347+
Licenses: []string{"Custom License from custom repo"},
348+
Relationship: ftypes.RelationshipDirect,
349+
Locations: ftypes.Locations{
350+
{
351+
StartLine: 32,
352+
EndLine: 36,
353+
},
354+
},
355+
},
356+
{
357+
ID: "org.example:example-runtime:1.0.0",
358+
Name: "org.example:example-runtime",
359+
Version: "1.0.0",
360+
Relationship: ftypes.RelationshipDirect,
361+
Locations: ftypes.Locations{
362+
{
363+
StartLine: 37,
364+
EndLine: 42,
365+
},
366+
},
367+
},
368+
},
369+
wantDeps: []ftypes.Dependency{
370+
{
371+
ID: "com.example:happy:1.0.0",
372+
DependsOn: []string{
373+
"org.example:example-api:1.7.30",
374+
"org.example:example-runtime:1.0.0",
375+
},
376+
},
377+
},
378+
},
329379
{
330380
name: "inherit parent properties",
331381
inputFile: filepath.Join("testdata", "parent-properties", "child", "pom.xml"),
@@ -2206,18 +2256,27 @@ func TestPom_Parse(t *testing.T) {
22062256
require.NoError(t, err)
22072257
defer f.Close()
22082258

2209-
var remoteRepos []string
2259+
var defaultRepo string
2260+
var settingsRepos []string
22102261
if tt.local {
22112262
// for local repository
22122263
t.Setenv("MAVEN_HOME", "testdata/settings/global")
22132264
} else {
22142265
// for remote repository
22152266
h := http.FileServer(http.Dir(filepath.Join("testdata", "repository")))
22162267
ts := httptest.NewServer(h)
2217-
remoteRepos = []string{ts.URL}
2268+
defaultRepo = ts.URL
2269+
2270+
// Enable custom repository to be sure in repository order checking
2271+
if tt.enableRepoForSettingsRepo {
2272+
ch := http.FileServer(http.Dir(filepath.Join("testdata", "repository-for-settings-repo")))
2273+
cts := httptest.NewServer(ch)
2274+
settingsRepos = []string{cts.URL}
2275+
}
22182276
}
22192277

2220-
p := pom.NewParser(tt.inputFile, pom.WithReleaseRemoteRepos(remoteRepos), pom.WithSnapshotRemoteRepos(remoteRepos), pom.WithOffline(tt.offline))
2278+
p := pom.NewParser(tt.inputFile, pom.WithDefaultRepo(defaultRepo, true, true),
2279+
pom.WithSettingsRepos(settingsRepos, true, false), pom.WithOffline(tt.offline))
22212280

22222281
gotPkgs, gotDeps, err := p.Parse(t.Context(), f)
22232282
if tt.wantErr != "" {

0 commit comments

Comments
 (0)