Skip to content

Add custom compressor commands to the config #635

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 13, 2025
Merged

Add custom compressor commands to the config #635

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/grype-db/cli/commands/package.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,5 +58,5 @@ func Package(app *application.Application) *cobra.Command {
}

func runPackage(cfg packageConfig) error {
return process.Package(cfg.Directory, cfg.PublishBaseURL, cfg.OverrideArchiveExtension)
return process.Package(cfg.Directory, cfg.PublishBaseURL, cfg.OverrideArchiveExtension, cfg.CompressorCommands)
}
10 changes: 4 additions & 6 deletions cmd/grype-db/cli/options/package.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,12 @@

type Package struct {
// bound options
PublishBaseURL string `yaml:"publish-base-url" json:"publish-base-url" mapstructure:"publish-base-url"`
OverrideArchiveExtension string `yaml:"override-archive-extension" json:"override-archive-extension" mapstructure:"override-archive-extension"`
PublishBaseURL string `yaml:"publish-base-url" json:"publish-base-url" mapstructure:"publish-base-url"`
OverrideArchiveExtension string `yaml:"override-archive-extension" json:"override-archive-extension" mapstructure:"override-archive-extension"`
CompressorCommands map[string]string `yaml:"compressor-commands" json:"compressor-commands" mapstructure:"compressor-commands"`

// unbound options
// (none)
compressorCommandsJSON string

Check failure on line 17 in cmd/grype-db/cli/options/package.go

View workflow job for this annotation

GitHub Actions / Static analysis 

field compressorCommandsJSON is unused (unused)
}

func DefaultPackage() Package {
Expand Down Expand Up @@ -46,8 +47,5 @@
return err
}

// set default values for non-bound struct items
// (none)

return nil
}
7 changes: 6 additions & 1 deletion internal/tarutil/populate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,12 @@ package tarutil

// PopulateWithPaths creates a compressed tar from the given paths.
func PopulateWithPaths(tarPath string, filePaths ...string) error {
w, err := NewWriter(tarPath)
return PopulateWithPathsAndCompressors(tarPath, nil, filePaths...)
}

// PopulateWithPathsAndCompressors creates a compressed tar from the given paths using custom compressor commands.
func PopulateWithPathsAndCompressors(tarPath string, compressorCommands map[string]string, filePaths ...string) error {
w, err := NewWriterWithCompressors(tarPath, compressorCommands)
if err != nil {
return err
}
Expand Down
19 changes: 17 additions & 2 deletions internal/tarutil/writer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,12 @@ type writer struct {

// NewWriter creates a new tar writer that writes to the specified archive path. Supports .tar.gz, .tar.zst, .tar.xz, and .tar file extensions.
func NewWriter(archivePath string) (Writer, error) {
w, err := newCompressor(archivePath)
return NewWriterWithCompressors(archivePath, nil)
}

// NewWriterWithCompressors creates a new tar writer with custom compressor commands. If compressorCommands is nil or empty, it uses default commands.
func NewWriterWithCompressors(archivePath string, compressorCommands map[string]string) (Writer, error) {
w, err := newCompressorWithCommands(archivePath, compressorCommands)
if err != nil {
return nil, err
}
Expand All @@ -40,12 +45,22 @@ func NewWriter(archivePath string) (Writer, error) {
}, nil
}

func newCompressor(archivePath string) (io.WriteCloser, error) {
func newCompressorWithCommands(archivePath string, compressorCommands map[string]string) (io.WriteCloser, error) {
archive, err := os.Create(archivePath)
if err != nil {
return nil, err
}

// check for custom compressor command first
for ext, cmd := range compressorCommands {
if strings.HasSuffix(archivePath, "."+ext) {
log.Debugf("using custom compressor command for %s: %s", ext, cmd)
return newShellCompressor(cmd, archive)
}
}
log.Debugf("no custom compressor command found for %s, using default", archivePath)

// fall back to default compressor commands
switch {
case strings.HasSuffix(archivePath, ".tar.gz"):
return gzip.NewWriterLevel(archive, flate.BestCompression)
Expand Down
6 changes: 3 additions & 3 deletions pkg/process/package.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@ import (
grypeDBLegacyDistribution "github.com/anchore/grype/grype/db/v5/distribution"
)

func Package(dbDir, publishBaseURL, overrideArchiveExtension string) error {
func Package(dbDir, publishBaseURL, overrideArchiveExtension string, compressorCommands map[string]string) error {
// check if metadata file exists, if so, then this
if _, err := os.Stat(filepath.Join(dbDir, grypeDBLegacyDistribution.MetadataFileName)); os.IsNotExist(err) {
// TODO: detect from disk which version of the DB is present
return v6process.CreateArchive(dbDir, overrideArchiveExtension)
return v6process.CreateArchive(dbDir, overrideArchiveExtension, compressorCommands)
}
return packageLegacyDB(dbDir, publishBaseURL, overrideArchiveExtension)
return packageLegacyDB(dbDir, publishBaseURL, overrideArchiveExtension, compressorCommands)
}
8 changes: 4 additions & 4 deletions pkg/process/package_legacy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import (
// listingFiles is a set of files that should not be included in the archive
var listingFiles = strset.New("listing.json", "latest.json", "history.json")

func packageLegacyDB(dbDir, publishBaseURL, overrideArchiveExtension string) error { //nolint:funlen
func packageLegacyDB(dbDir, publishBaseURL, overrideArchiveExtension string, compressorCommands map[string]string) error { //nolint:funlen
log.WithFields("from", dbDir, "url", publishBaseURL, "extension-override", overrideArchiveExtension).Info("packaging database")

fs := afero.NewOsFs()
Expand Down Expand Up @@ -88,7 +88,7 @@ func packageLegacyDB(dbDir, publishBaseURL, overrideArchiveExtension string) err
)
tarPath := path.Join(dbDir, tarName)

if err := populateLegacyTar(tarPath); err != nil {
if err := populateLegacyTar(tarPath, compressorCommands); err != nil {
return err
}

Expand All @@ -110,7 +110,7 @@ func packageLegacyDB(dbDir, publishBaseURL, overrideArchiveExtension string) err
return nil
}

func populateLegacyTar(tarPath string) error {
func populateLegacyTar(tarPath string, compressorCommands map[string]string) error {
originalDir, err := os.Getwd()
if err != nil {
return fmt.Errorf("unable to get CWD: %w", err)
Expand Down Expand Up @@ -142,7 +142,7 @@ func populateLegacyTar(tarPath string) error {
}
}

if err = tarutil.PopulateWithPaths(tarName, files...); err != nil {
if err = tarutil.PopulateWithPathsAndCompressors(tarName, compressorCommands, files...); err != nil {
return fmt.Errorf("unable to create db archive: %w", err)
}

Expand Down
8 changes: 4 additions & 4 deletions pkg/process/v6/archive.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import (
v6Distribution "github.com/anchore/grype/grype/db/v6/distribution"
)

func CreateArchive(dbDir, overrideArchiveExtension string) error {
func CreateArchive(dbDir, overrideArchiveExtension string, compressorCommands map[string]string) error {
extension, err := resolveExtension(overrideArchiveExtension)
if err != nil {
return err
Expand Down Expand Up @@ -71,7 +71,7 @@ func CreateArchive(dbDir, overrideArchiveExtension string) error {
files = append(files, v6.ImportMetadataFileName)
}

if err := populateTar(dbDir, tarName, files...); err != nil {
if err := populateTar(dbDir, tarName, compressorCommands, files...); err != nil {
return err
}

Expand Down Expand Up @@ -112,7 +112,7 @@ func resolveExtension(overrideArchiveExtension string) (string, error) {
return extension, nil
}

func populateTar(dbDir, tarName string, files ...string) error {
func populateTar(dbDir, tarName string, compressorCommands map[string]string, files ...string) error {
originalDir, err := os.Getwd()
if err != nil {
return fmt.Errorf("unable to get CWD: %w", err)
Expand All @@ -137,7 +137,7 @@ func populateTar(dbDir, tarName string, files ...string) error {
}
}

if err = tarutil.PopulateWithPaths(tarName, files...); err != nil {
if err = tarutil.PopulateWithPathsAndCompressors(tarName, compressorCommands, files...); err != nil {
return fmt.Errorf("unable to create db archive: %w", err)
}

Expand Down
Loading