Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 5 additions & 27 deletions .github/actions/bootstrap/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,10 @@ inputs:
description: "Python version to install"
required: true
default: "3.12"
poetry-version:
description: "Poetry version to install"
required: true
default: "1.8.5"
use-go-cache:
description: "Restore go cache"
required: true
default: "true"
use-poetry-cache:
description: "Restore poetry cache"
required: true
default: "true"
cache-key-prefix:
description: "Prefix all cache keys with this value"
required: true
Expand All @@ -52,30 +44,16 @@ runs:
with:
go-version: ${{ inputs.go-version }}

- uses: actions/setup-python@v5
if: inputs.python == 'true'
with:
python-version: ${{ inputs.python-version }}

- name: Install Poetry
if: inputs.python == 'true'
shell: bash
run: |
pipx install poetry==${{ inputs.poetry-version }}

- name: Cache Poetry cache
uses: actions/cache@v4
- name: Install uv
if: inputs.python == 'true'
uses: astral-sh/setup-uv@v5
with:
path: ~/.cache/pypoetry
key: poetry-cache-${{ runner.os }}-${{ inputs.python-version }}-${{ inputs.poetry-version }}
enable-cache: true

- name: Cache Packages
uses: actions/cache@v4
- uses: actions/setup-python@v5
if: inputs.python == 'true'
with:
path: ~/.local
key: poetry-local-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/*.yml') }}
python-version: ${{ inputs.python-version }}

- name: Bootstrap python dependencies
if: inputs.python == 'true'
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/copy-listing-from-r2.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:

- name: Upload listing file
run: |
poetry run \
uv run \
grype-db-manager \
-c ./config/grype-db-manager/publish-production.yaml \
listing upload listing.json
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/daily-db-publisher-r2.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ jobs:

- name: Generate and upload DB (schema ${{ matrix.schema-version }})
run: |
poetry run \
uv run \
grype-db-manager \
-vv \
-c ./config/grype-db-manager/publish-production-r2.yaml \
Expand Down Expand Up @@ -130,7 +130,7 @@ jobs:

- name: Publish listing file
run: |
poetry run \
uv run \
grype-db-manager \
-c ./config/grype-db-manager/publish-production-r2.yaml \
listing update
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/daily-db-publisher.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ jobs:

- name: Generate and upload DB (schema ${{ matrix.schema-version }})
run: |
poetry run \
uv run \
grype-db-manager \
-vv \
-c ./config/grype-db-manager/publish-production.yaml \
Expand Down Expand Up @@ -130,7 +130,7 @@ jobs:

# - name: Publish listing file
# run: |
# poetry run \
# uv run \
# grype-db-manager \
# -c ./config/grype-db-manager/publish-production.yaml \
# listing update
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/staging-db-publisher.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ jobs:
# b) if not using workflow_dispatch the default values are empty, which means we want these to effectively evaluate to true (so only check the negative case)
if: github.event.inputs.publish-databases != 'false'
run: |
poetry run \
uv run \
grype-db-manager \
-vv \
-c ./config/grype-db-manager/publish-staging.yaml \
Expand All @@ -105,7 +105,7 @@ jobs:
# b) if not using workflow_dispatch the default values are empty, which means we want these to effectively evaluate to true (so only check the negative case)
if: github.event.inputs.publish-listing != 'false'
run: |
poetry run \
uv run \
grype-db-manager \
-c ./config/grype-db-manager/publish-staging.yaml \
listing update
Expand Down
26 changes: 5 additions & 21 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -95,11 +95,11 @@ all: static-analysis test ## Run all checks (linting, license checks, unit, and

.PHONY: static-analysis ## Run all static analysis checks (linting and license checks)
static-analysis: check-go-mod-tidy lint check-licenses
cd manager && poetry run make static-analysis
cd manager && uv run make static-analysis

.PHONY: test
test: unit cli ## Run all tests
cd manager && poetry run make test
cd manager && uv run make test


## Bootstrapping targets #################################
Expand Down Expand Up @@ -172,19 +172,19 @@ unit: $(TEMP_DIR) ## Run Go unit tests (with coverage)
.PHONY: unit-python
unit-python: ## Run Python unit tests (with coverage)
$(call title,Running Python unit tests)
cd manager && poetry run make unit
cd manager && make unit

.PHONY: db-acceptance
db-acceptance: ## Run acceptance tests
$(call title,"Running DB acceptance tests (schema=$(schema))")
poetry run ./test/db/acceptance.sh $(schema)
uv run ./test/db/acceptance.sh $(schema)

.PHONY: cli
cli: cli-go cli-python ## Run all CLI tests

.PHONY: cli-python
cli-python: ## Run python CLI tests
cd manager && poetry run make cli
cd manager && uv run make cli

.PHONY: cli-go
cli-go: $(SNAPSHOT_DIR) ## Run go CLI tests
Expand All @@ -194,22 +194,6 @@ cli-go: $(SNAPSHOT_DIR) ## Run go CLI tests
go test -count=1 -timeout=15m -v ./test/cli


## Test-fixture-related targets #################################

.PHONY: update-test-fixtures
update-test-fixtures:
docker run \
--pull always \
--rm \
-it \
anchore/grype:latest \
-q \
-o json \
centos:8.2.2004 > publish/test-fixtures/centos-8.2.2004.json
dos2unix publish/test-fixtures/centos-8.2.2004.json
cd test/acceptance && poetry install && poetry run python grype-ingest.py capture-test-fixtures


## Data management targets #################################

.PHONY: show-providers
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ require (
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
github.com/adrg/xdg v0.5.3
github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a
github.com/anchore/grype v0.87.1-0.20250211173554-1573bd980b6c
github.com/anchore/grype v0.87.1-0.20250213225744-bb90ead7720b
github.com/anchore/syft v1.19.0
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
github.com/dave/jennifer v1.7.1
Expand Down Expand Up @@ -246,7 +246,7 @@ require (
golang.org/x/mod v0.22.0 // indirect
golang.org/x/net v0.34.0 // indirect
golang.org/x/oauth2 v0.25.0 // indirect
golang.org/x/sys v0.29.0 // indirect
golang.org/x/sys v0.30.0 // indirect
golang.org/x/term v0.28.0 // indirect
golang.org/x/time v0.10.0 // indirect
golang.org/x/tools v0.29.0 // indirect
Expand Down
11 changes: 6 additions & 5 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -698,8 +698,8 @@ github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0v
github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ=
github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4 h1:rmZG77uXgE+o2gozGEBoUMpX27lsku+xrMwlmBZJtbg=
github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
github.com/anchore/grype v0.87.1-0.20250211173554-1573bd980b6c h1:S/wwuiO3TiBgy9StBFrmTi4HVyeVbbViBBLn9TQJNxE=
github.com/anchore/grype v0.87.1-0.20250211173554-1573bd980b6c/go.mod h1:COAoNjT1UktC8ZRvkZfPtW4L8pXpiS8krCqobKvjLkw=
github.com/anchore/grype v0.87.1-0.20250213225744-bb90ead7720b h1:2xB2h3XQnuOmQDU/TaoGhfdZJs0xdWsAUlMY2Bi1bO8=
github.com/anchore/grype v0.87.1-0.20250213225744-bb90ead7720b/go.mod h1:bCSgpmhQMP9jaUWV1DfwfyiAodKgi8tN/mcDinHXXmY=
github.com/anchore/packageurl-go v0.1.1-0.20250117185454-edf36a908b10 h1:zBedM9ZGYbs/61QC4ZOKxtChx5njXKHgHqDeHuUxrTw=
github.com/anchore/packageurl-go v0.1.1-0.20250117185454-edf36a908b10/go.mod h1:KoYIv7tdP5+CC9VGkeZV4/vGCKsY55VvoG+5dadg4YI=
github.com/anchore/stereoscope v0.0.13 h1:9Ivkh7k+vOeG3JHrt44jOg/8UdZrCvMsSjLQ7trHBig=
Expand Down Expand Up @@ -768,8 +768,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/charmbracelet/bubbles v0.20.0 h1:jSZu6qD8cRQ6k9OMfR1WlM+ruM8fkPWkHvQWD9LIutE=
github.com/charmbracelet/bubbles v0.20.0/go.mod h1:39slydyswPy+uVOHZ5x/GjwVAFkCsV8IIVy+4MhzwwU=
github.com/charmbracelet/bubbletea v1.3.0 h1:fPMyirm0u3Fou+flch7hlJN9krlnVURrkUVDwqXjoAc=
github.com/charmbracelet/bubbletea v1.3.0/go.mod h1:eTaHfqbIwvBhFQM/nlT1NsGc4kp8jhF8LfUK67XiTDM=
github.com/charmbracelet/bubbletea v1.3.3 h1:WpU6fCY0J2vDWM3zfS3vIDi/ULq3SYphZhkAGGvmEUY=
github.com/charmbracelet/bubbletea v1.3.3/go.mod h1:dtcUCyCGEX3g9tosuYiut3MXgY/Jsv9nKVdibKKRRXo=
github.com/charmbracelet/lipgloss v1.0.0 h1:O7VkGDvqEdGi93X+DeqsQ7PKHDgtQfF8j8/O2qFMQNg=
github.com/charmbracelet/lipgloss v1.0.0/go.mod h1:U5fy9Z+C38obMs+T+tJqst9VGzlOYGj4ri9reL3qUlo=
github.com/charmbracelet/x/ansi v0.8.0 h1:9GTq3xq9caJW8ZrBTe0LIe2fvfLR/bYXKTx2llXn7xE=
Expand Down Expand Up @@ -1886,8 +1886,9 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
Expand Down
35 changes: 17 additions & 18 deletions manager/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -10,55 +10,54 @@ static-analysis: lint ## Run all static analyses
.PHONY: test
test: unit ## Run all tests

virtual-env-check:
@ if [ "${VIRTUAL_ENV}" = "" ]; then \
echo "$(ERROR)Not in a virtual environment. Try running with 'poetry run' or enter a 'poetry shell' session.$(RESET)"; \
exit 1; \
fi


## Bootstrapping targets #################################

.PHONY: bootstrap
bootstrap: ## Install all dependencies
$(call title,Bootstrapping dependencies)
poetry install
uv sync --all-extras --dev


## Static analysis targets #################################

.PHONY: lint
lint: virtual-env-check ## Show linting issues (ruff)
ruff check .
lint: ## Show linting issues (ruff)
uv run ruff format --check
uv run ruff check .

.PHONY: lint-fix
lint-fix: virtual-env-check ## Fix linting issues (ruff)
ruff check . --fix
lint-fix: format ## Fix linting issues (ruff)
uv run ruff check . --fix

.PHONY: format
format: ## Format (ruff)
uv run ruff format


## Testing targets #################################

.PHONY: unit
unit: virtual-env-check ## Run unit tests
pytest --cov=grype_db_manager --cov-report=html -vv tests/unit/
unit: ## Run unit tests
uv run pytest --cov=grype_db_manager --cov-report=html -vv tests/unit/

.PHONY: cli
cli: virtual-env-check ## Run cli tests
cd tests/cli && make
cli: ## Run cli tests
cd tests/cli && uv run make


## DB Testing targets #################################

.PHONY: db-acceptance
db-acceptance: virtual-env-check ## Run DB acceptance tests
db-acceptance: ## Run DB acceptance tests
@ echo "Building and testing DB schema=$(schema_version)"
if [ -z "$(schema_version)" ]; then \
echo "schema_version is not set"; \
exit 1; \
fi

export DB_ID=$(shell grype-db-manager db build --schema-version $(schema_version))
grype-db-manager db validaate $(DB_ID)
export DB_ID=$(shell uv run grype-db-manager db build --schema-version $(schema_version))
uv run grype-db-manager db validaate $(DB_ID)


## Halp! #################################
Expand Down
1 change: 0 additions & 1 deletion manager/src/grype_db_manager/cli/db.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,6 @@ def _validate_db(
# resolve tool versions and install them
yardstick.store.config.set_values(store_root=cfg.data.yardstick_root)


validations_enabled = db.schema.validations_enabled(db_info.schema_version)
if not validations_enabled and not force:
click.echo(f"{Format.BOLD}{Format.OKGREEN}Validation disabled, skipping{Format.RESET}")
Expand Down
3 changes: 1 addition & 2 deletions manager/src/grype_db_manager/cli/listing.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,7 @@ def create_listing(cfg: config.Application, ignore_missing_listing: bool) -> str

if missing_basenames:
logging.warning(
f"missing {len(missing_basenames)} databases in S3 which were in the existing"
" listing file (removing entries in the next listing file)",
f"missing {len(missing_basenames)} databases in S3 which were in the existing listing file (removing entries in the next listing file)",
)
for basename in missing_basenames:
logging.warning(f" - {basename}")
Expand Down
2 changes: 2 additions & 0 deletions manager/src/grype_db_manager/db/format.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,11 @@ def get_section_index(value: int, min_value: int, max_value: int, sections: int,
def rgb_ansi(r: int, g: int, b: int) -> str:
return f"\033[38;2;{r};{g};{b}m"


def reset_ansi() -> str:
return "\033[0m"


def format_value_red_green_spectrum(
value: int,
min_value: int = 0,
Expand Down
2 changes: 1 addition & 1 deletion manager/src/grype_db_manager/db/listing.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def basename(self) -> str:

def age_in_days(self, now: datetime.datetime | None = None) -> int:
if not now:
now = datetime.datetime.now(tz=datetime.timezone.utc)
now = datetime.datetime.now(tz=datetime.UTC)
return (now - iso8601.parse_date(self.built)).days


Expand Down
6 changes: 1 addition & 5 deletions manager/src/grype_db_manager/db/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,11 +54,7 @@ def register_mapping(file: str) -> None:

@lru_cache
def _mapping() -> dict[str, Any]:
content = (
files("grype_db_manager.data").joinpath("schema-info.json").read_text()
if _mapping_file_content is None
else _mapping_file_content
)
content = files("grype_db_manager.data").joinpath("schema-info.json").read_text() if _mapping_file_content is None else _mapping_file_content
return json.loads(content)


Expand Down
4 changes: 1 addition & 3 deletions manager/src/grype_db_manager/db/validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,9 +102,7 @@ def _is_result_set_consistent(result_set_object: artifact.ResultSet, request_ima
if s.config and "grype" in s.config.tool:
grype_requests_by_image[s.config.full_image].append(s.request)

missing_grype_requests = [
image for image in request_images if image not in grype_requests_by_image or len(grype_requests_by_image[image]) != 2
]
missing_grype_requests = [image for image in request_images if image not in grype_requests_by_image or len(grype_requests_by_image[image]) != 2]

if missing_grype_requests:
logging.warning(f"result-set has missing grype requests: {missing_grype_requests}")
Expand Down
2 changes: 1 addition & 1 deletion manager/src/grype_db_manager/distribution.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ def age_from_basename(basename: str) -> int | None:


def _now() -> datetime.datetime:
return datetime.datetime.now(tz=datetime.timezone.utc)
return datetime.datetime.now(tz=datetime.UTC)


def hash_file(path: str) -> str:
Expand Down
Loading
Loading