Skip to content

feat: add support for detecting packages in JARs #4199

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+268 −4
Open

feat: add support for detecting packages in JARs #4199

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions internal/file/glob_match_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@ func TestGlobMatch(t *testing.T) {
{"a*a*a*a*b", strings.Repeat("a", 100), false},
{"*x", "xxx", true},
{"/home/place/**", "/home/place/a/thing", true},
{"/org/test/**/*.class", "/org/test/system/files/Hello.class", true},
{"**/*.class", "/org/test/system/files/Hello.class", true},
{"**/*.class", "Hello.class", false},
}

for _, test := range tests {
Expand Down
75 changes: 71 additions & 4 deletions syft/pkg/cataloger/java/archive_parser.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,12 @@ func (j *archiveParser) discoverMainPackage(ctx context.Context) (*pkg.Package,
pkgPomProject = newPomProject(ctx, j.maven, parsedPom.path, parsedPom.project)
}

var containedPackages []string

if j.cfg.DetectContainedPackages {
containedPackages = j.discoverContainedPackages()
}

return &pkg.Package{
// TODO: maybe select name should just have a pom properties in it?
Name: name,
Expand All @@ -278,10 +284,11 @@ func (j *archiveParser) discoverMainPackage(ctx context.Context) (*pkg.Package,
),
Type: j.fileInfo.pkgType(),
Metadata: pkg.JavaArchive{
VirtualPath: j.location.Path(),
Manifest: manifest,
PomProject: pkgPomProject,
ArchiveDigests: digests,
VirtualPath: j.location.Path(),
Manifest: manifest,
PomProject: pkgPomProject,
ArchiveDigests: digests,
ContainedPackages: containedPackages,
},
}, nil
}
Expand Down Expand Up @@ -868,3 +875,63 @@ func sortedIter[K cmp.Ordered, V any](values map[K]V) iter.Seq2[K, V] {
}
}
}

// Tests if the given string is a valid multi-release version as specified by
// https://docs.oracle.com/en/java/javase/11/docs/specs/jar/jar.html#multi-release-jar-files
func isValidMultiReleaseVersion(s string) bool {
if s == "" {
return false
}

if s == "9" {
return true
}

// 0 is not allowed
if s[0] < '1' || s[0] > '9' {
return false
}

// Ony digits are allowed
return strings.IndexFunc(s, func(r rune) bool {
return r < '0' || r > '9'
}) != -1
}

func (j *archiveParser) discoverContainedPackages() []string {
pkgSet := strset.New()

classes := j.fileManifest.GlobMatch(false, "**/*.class")

for _, c := range classes {
parts := strings.Split(c, "/")

if len(parts) > 3 && parts[0] == "META-INF" && parts[1] == "versions" && isValidMultiReleaseVersion(parts[2]) {
// Strip the version specific prefix, as we are interested in all packages in the JAR.
parts = parts[3:]
}

// Ignore any unnamed packages.
if len(parts) <= 1 {
continue
}

// Skip any non version specific classes in META-INF and ignore WEB-INF.
if parts[0] == "META-INF" || parts[0] == "WEB-INF" {
continue
}

pkgName := strings.Join(parts[:len(parts)-1], ".")

pkgSet.Add(pkgName)
}

if pkgSet.Size() == 0 {
return nil
}

pkgs := pkgSet.List()
slices.Sort(pkgs)

return pkgs
}
167 changes: 167 additions & 0 deletions syft/pkg/cataloger/java/archive_parser_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,89 @@ func TestParseJar(t *testing.T) {
},
},
},
ContainedPackages: []string{
"hello",
"net.bytebuddy",
"net.bytebuddy.agent.builder",
"net.bytebuddy.asm",
"net.bytebuddy.build",
"net.bytebuddy.description",
"net.bytebuddy.description.annotation",
"net.bytebuddy.description.enumeration",
"net.bytebuddy.description.field",
"net.bytebuddy.description.method",
"net.bytebuddy.description.modifier",
"net.bytebuddy.description.type",
"net.bytebuddy.dynamic",
"net.bytebuddy.dynamic.loading",
"net.bytebuddy.dynamic.scaffold",
"net.bytebuddy.dynamic.scaffold.inline",
"net.bytebuddy.dynamic.scaffold.subclass",
"net.bytebuddy.implementation",
"net.bytebuddy.implementation.attribute",
"net.bytebuddy.implementation.auxiliary",
"net.bytebuddy.implementation.bind",
"net.bytebuddy.implementation.bind.annotation",
"net.bytebuddy.implementation.bytecode",
"net.bytebuddy.implementation.bytecode.assign",
"net.bytebuddy.implementation.bytecode.assign.primitive",
"net.bytebuddy.implementation.bytecode.assign.reference",
"net.bytebuddy.implementation.bytecode.collection",
"net.bytebuddy.implementation.bytecode.constant",
"net.bytebuddy.implementation.bytecode.member",
"net.bytebuddy.jar.asm",
"net.bytebuddy.jar.asm.commons",
"net.bytebuddy.jar.asm.signature",
"net.bytebuddy.matcher",
"net.bytebuddy.pool",
"net.bytebuddy.utility",
"net.bytebuddy.utility.dispatcher",
"net.bytebuddy.utility.nullability",
"net.bytebuddy.utility.privilege",
"net.bytebuddy.utility.visitor",
"org.joda.time",
"org.joda.time.base",
"org.joda.time.chrono",
"org.joda.time.convert",
"org.joda.time.field",
"org.joda.time.format",
"org.joda.time.tz",
},
},
},
"byte-buddy": {
Name: "byte-buddy",
Version: "1.17.5",
PURL: "pkg:maven/net.bytebuddy/[email protected]",
Licenses: pkg.NewLicenseSet(),
Language: pkg.Java,
Type: pkg.JavaPkg,
Metadata: pkg.JavaArchive{
// ensure that nested packages with different names than that of the parent are appended as
// a suffix on the virtual path
VirtualPath: "test-fixtures/java-builds/packages/example-java-app-gradle-0.1.0.jar:net.bytebuddy:byte-buddy",
PomProperties: &pkg.JavaPomProperties{
Path: "META-INF/maven/net.bytebuddy/byte-buddy/pom.properties",
GroupID: "net.bytebuddy",
ArtifactID: "byte-buddy",
Version: "1.17.5",
},
PomProject: &pkg.JavaPomProject{
Path: "META-INF/maven/net.bytebuddy/byte-buddy/pom.xml",
Parent: &pkg.JavaPomParent{
GroupID: "net.bytebuddy",
ArtifactID: "byte-buddy-parent",
Version: "1.17.5",
},
GroupID: "net.bytebuddy",
ArtifactID: "byte-buddy",
Version: "1.17.5",
Name: "Byte Buddy (without dependencies)",
Description: "Byte Buddy is a Java library for creating Java classes at run time. " +
"This artifact is a build of Byte Buddy with all ASM dependencies repackaged " +
"into its own name space.",
URL: "",
},
// PomProject: &pkg.JavaPomProject{
// Path: "META-INF/maven/io.jenkins.plugins/example-jenkins-plugin/pom.xml",
// Parent: &pkg.JavaPomParent{GroupID: "org.jenkins-ci.plugins", ArtifactID: "plugin", Version: "4.46"},
Expand Down Expand Up @@ -316,6 +399,89 @@ func TestParseJar(t *testing.T) {
ArtifactID: "example-java-app-maven",
Version: "0.1.0",
},
ContainedPackages: []string{
"hello",
"net.bytebuddy",
"net.bytebuddy.agent.builder",
"net.bytebuddy.asm",
"net.bytebuddy.build",
"net.bytebuddy.description",
"net.bytebuddy.description.annotation",
"net.bytebuddy.description.enumeration",
"net.bytebuddy.description.field",
"net.bytebuddy.description.method",
"net.bytebuddy.description.modifier",
"net.bytebuddy.description.type",
"net.bytebuddy.dynamic",
"net.bytebuddy.dynamic.loading",
"net.bytebuddy.dynamic.scaffold",
"net.bytebuddy.dynamic.scaffold.inline",
"net.bytebuddy.dynamic.scaffold.subclass",
"net.bytebuddy.implementation",
"net.bytebuddy.implementation.attribute",
"net.bytebuddy.implementation.auxiliary",
"net.bytebuddy.implementation.bind",
"net.bytebuddy.implementation.bind.annotation",
"net.bytebuddy.implementation.bytecode",
"net.bytebuddy.implementation.bytecode.assign",
"net.bytebuddy.implementation.bytecode.assign.primitive",
"net.bytebuddy.implementation.bytecode.assign.reference",
"net.bytebuddy.implementation.bytecode.collection",
"net.bytebuddy.implementation.bytecode.constant",
"net.bytebuddy.implementation.bytecode.member",
"net.bytebuddy.jar.asm",
"net.bytebuddy.jar.asm.commons",
"net.bytebuddy.jar.asm.signature",
"net.bytebuddy.matcher",
"net.bytebuddy.pool",
"net.bytebuddy.utility",
"net.bytebuddy.utility.dispatcher",
"net.bytebuddy.utility.nullability",
"net.bytebuddy.utility.privilege",
"net.bytebuddy.utility.visitor",
"org.joda.time",
"org.joda.time.base",
"org.joda.time.chrono",
"org.joda.time.convert",
"org.joda.time.field",
"org.joda.time.format",
"org.joda.time.tz",
},
},
},
"byte-buddy": {
Name: "byte-buddy",
Version: "1.17.5",
PURL: "pkg:maven/net.bytebuddy/[email protected]",
Licenses: pkg.NewLicenseSet(),
Language: pkg.Java,
Type: pkg.JavaPkg,
Metadata: pkg.JavaArchive{
// ensure that nested packages with different names than that of the parent are appended as
// a suffix on the virtual path
VirtualPath: "test-fixtures/java-builds/packages/example-java-app-maven-0.1.0.jar:net.bytebuddy:byte-buddy",
PomProperties: &pkg.JavaPomProperties{
Path: "META-INF/maven/net.bytebuddy/byte-buddy/pom.properties",
GroupID: "net.bytebuddy",
ArtifactID: "byte-buddy",
Version: "1.17.5",
},
PomProject: &pkg.JavaPomProject{
Path: "META-INF/maven/net.bytebuddy/byte-buddy/pom.xml",
Parent: &pkg.JavaPomParent{
GroupID: "net.bytebuddy",
ArtifactID: "byte-buddy-parent",
Version: "1.17.5",
},
GroupID: "net.bytebuddy",
ArtifactID: "byte-buddy",
Version: "1.17.5",
Name: "Byte Buddy (without dependencies)",
Description: "Byte Buddy is a Java library for creating Java classes at run time. " +
"This artifact is a build of Byte Buddy with all ASM dependencies repackaged " +
"into its own name space.",
URL: "",
},
},
},
"joda-time": {
Expand Down Expand Up @@ -372,6 +538,7 @@ func TestParseJar(t *testing.T) {
cfg := ArchiveCatalogerConfig{
UseNetwork: false,
UseMavenLocalRepository: false,
DetectContainedPackages: true,
}
parser, cleanupFn, err := newJavaArchiveParser(
ctx,
Expand Down
10 changes: 10 additions & 0 deletions syft/pkg/cataloger/java/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ type ArchiveCatalogerConfig struct {
// ResolveTransitiveDependencies enables resolving transitive dependencies for java packages found within archives.
// app-config: java.resolve-transitive-dependencies
ResolveTransitiveDependencies bool `yaml:"resolve-transitive-dependencies" json:"resolve-transitive-dependencies" mapstructure:"resolve-transitive-dependencies"`

// DetectContainedPackages enables collecting all package names contained in a jar.
// app-config: java.detect-contained-packages
DetectContainedPackages bool `yaml:"detect-contained-packages" json:"detect-contained-packages" mapstructure:"detect-contained-packages"`
}

func DefaultArchiveCatalogerConfig() ArchiveCatalogerConfig {
Expand All @@ -45,6 +49,7 @@ func DefaultArchiveCatalogerConfig() ArchiveCatalogerConfig {
MavenBaseURL: strings.Join(mavenCfg.Repositories, ","),
MaxParentRecursiveDepth: mavenCfg.MaxParentRecursiveDepth,
ResolveTransitiveDependencies: false,
DetectContainedPackages: false,
}
}

Expand Down Expand Up @@ -81,6 +86,11 @@ func (j ArchiveCatalogerConfig) WithArchiveTraversal(search cataloging.ArchiveSe
return j
}

func (j ArchiveCatalogerConfig) WithDetectContainedPackages(detectContainedPackages bool) ArchiveCatalogerConfig {
j.DetectContainedPackages = detectContainedPackages
return j
}

func (j ArchiveCatalogerConfig) mavenConfig() maven.Config {
return maven.Config{
UseNetwork: j.UseNetwork,
Expand Down
2 changes: 2 additions & 0 deletions syft/pkg/cataloger/java/tar_wrapped_archive_parser_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,15 @@ func Test_parseTarWrappedJavaArchive(t *testing.T) {
expected: []string{
"example-java-app-maven",
"joda-time",
"byte-buddy",
},
},
{
fixture: "test-fixtures/java-builds/packages/example-java-app-maven-0.1.0.tar.gz",
expected: []string{
"example-java-app-maven",
"joda-time",
"byte-buddy",
},
},
}
Expand Down
3 changes: 3 additions & 0 deletions syft/pkg/cataloger/java/test-fixtures/java-builds/example-java-app/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ targetCompatibility = 1.8

dependencies {
implementation "joda-time:joda-time:2.2"
implementation "net.bytebuddy:byte-buddy:1.17.5"
testImplementation "junit:junit:4.12"
}
// end::dependencies[]
Expand All @@ -37,6 +38,8 @@ jar {
from {
configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) }
}

duplicatesStrategy = DuplicatesStrategy.EXCLUDE
}
// end::jar[]

Expand Down
1 change: 1 addition & 0 deletions syft/pkg/cataloger/java/test-fixtures/java-builds/example-java-app/gradle.lockfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,6 @@
# This file is expected to be part of source control.
joda-time:joda-time:2.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
junit:junit:4.12=testCompileClasspath,testRuntimeClasspath
net.bytebuddy:byte-buddy:1.17.5=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
org.hamcrest:hamcrest-core:1.3=testCompileClasspath,testRuntimeClasspath
empty=annotationProcessor,testAnnotationProcessor
7 changes: 7 additions & 0 deletions syft/pkg/cataloger/java/test-fixtures/java-builds/example-java-app/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,13 @@
</properties>

<dependencies>
<!-- tag::bytebuddy[] -->
<dependency>
<groupId>net.bytebuddy</groupId>
<artifactId>byte-buddy</artifactId>
<version>1.17.5</version>
</dependency>
<!-- end::bytebuddy[] -->
<!-- tag::joda[] -->
<dependency>
<groupId>joda-time</groupId>
Expand Down
1 change: 1 addition & 0 deletions syft/pkg/cataloger/java/zip_wrapped_archive_parser_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ func Test_parseZipWrappedJavaArchive(t *testing.T) {
expected: []string{
"example-java-app-maven",
"joda-time",
"byte-buddy",
},
},
}
Expand Down
3 changes: 3 additions & 0 deletions syft/pkg/java.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,9 @@ type JavaArchive struct {
// ArchiveDigests is cryptographic hashes of the archive file
ArchiveDigests []file.Digest `hash:"ignore" json:"digest,omitempty"`

// ContainedPackages is a list of all package names contained in the jar
ContainedPackages []string `mapstructure:"ContainedPackages" json:"containedPackages"`

// Parent is reference to parent package (for nested archives)
Parent *Package `hash:"ignore" json:"-"`
}
Expand Down
Loading