anchore · kzantow · Aug 16, 2024 · Aug 16, 2024 · Aug 16, 2024 · Aug 16, 2024
diff --git a/.github/workflows/daily-db-publisher-r2.yaml b/.github/workflows/daily-db-publisher-r2.yaml
@@ -0,0 +1,147 @@
+name: 'Daily DB Publisher R2'
+on:
+  # allow for kicking off DB builds manually
+  workflow_dispatch:
+    inputs:
+      publish-databases:
+        description: "build new databases and upload to S3"
+        type: boolean
+        required: true
+        default: true
+      publish-listing:
+        description: "use S3 state to update and publish listing file"
+        type: boolean
+        required: true
+        default: true
+
+  # run 4 AM (UTC) daily
+  # schedule:
+  #  - cron:  '0 4 * * *'
+
+env:
+  CGO_ENABLED: "0"
+  SLACK_NOTIFICATIONS: true
+  FORCE_COLOR: true
+
+jobs:
+  discover-schema-versions:
+    # note about workflow dispatch inputs and booleans:
+    # a) booleans come across as string types :(
+    # b) if not using workflow_dispatch the default values are empty, which means we want these to effectively evaluate to true (so only check the negative case)
+    if: ${{ github.event.inputs.publish-databases != 'false' }}
+    name: "Pull vulnerability data"
+    runs-on: ubuntu-20.04
+    outputs:
+      schema-versions: ${{ steps.read-schema-versions.outputs.schema-versions }}
+      pull-date: ${{ steps.timestamp.outputs.date }}
+    # set the permissions granted to the github token to read the pull cache from ghcr.io
+    permissions:
+      contents: read
+      packages: read
+    steps:
+
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
+
+      - name: Read supported schema versions
+        id: read-schema-versions
+        run: |
+          content=`cat manager/src/grype_db_manager/data/schema-info.json | jq -c '[.available[] | select(.supported == true) | .schema]'`
+          echo "schema-versions=$content" >> $GITHUB_OUTPUT
+
+  generate-and-publish-dbs:
+    # note about workflow dispatch inputs and booleans:
+    # a) booleans come across as string types :(
+    # b) if not using workflow_dispatch the default values are empty, which means we want these to effectively evaluate to true (so only check the negative case)
+    if: ${{ github.event.inputs.publish-databases != 'false' }}
+    name: "Generate and publish DBs"
+    needs: discover-schema-versions
+    runs-on: ubuntu-22.04-4core-16gb
+    strategy:
+      matrix:
+        schema-version: ${{fromJson(needs.discover-schema-versions.outputs.schema-versions)}}
+    # set the permissions granted to the github token to read the pull cache from ghcr.io
+    permissions:
+      contents: read
+      packages: read
+    steps:
+
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
+        with:
+          submodules: true
+
+      - name: Bootstrap environment
+        uses: ./.github/actions/bootstrap
+
+      - name: Login to ghcr.io
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | oras login ghcr.io --username ${{ github.actor }} --password-stdin
+
+      - name: Pull vulnerability data
+        run: make download-all-provider-cache
+
+      - name: Generate and upload DB (schema ${{ matrix.schema-version }})
+        run: |
+          poetry run \
+            grype-db-manager \
+              -c ./config/grype-db-manager/publish-production-r2.yaml \
+              db build-and-upload \
+                --schema-version ${{ matrix.schema-version }} \
+                -vvv
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.TOOLBOX_CLOUDFLARE_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLBOX_CLOUDFLARE_SECRET_ACCESS_KEY }}
+          GRYPE_DB_MANAGER_DISTRIBUTION_S3_ENDPOINT_URL: ${{ secrets.TOOLBOX_CLOUDFLARE_R2_ENDPOINT }}
+
+      - uses: 8398a7/action-slack@28ba43ae48961b90635b50953d216767a6bea486 #v3.16.2
+        with:
+          status: ${{ job.status }}
+          fields: workflow,eventName,job
+          text: Publishing the Grype DB has failed (schema ${{ matrix.schema-version }})
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }}
+        if: ${{ failure() && env.SLACK_NOTIFICATIONS == 'true' }}
+
+  publish-listing-file:
+    # fun! https://github.com/actions/runner/issues/491#issuecomment-850884422
+    # essentially even if the workflow dispatch job is skipping steps, we still want to run this step.
+    # however, if not running from a workflow dispatch then we want the job ordering to be honored.
+    # also...
+    # note about workflow dispatch inputs and booleans:
+    # a) booleans come across as string types :(
+    # b) if not using workflow_dispatch the default values are empty, which means we want these to effectively evaluate to true (so only check the negative case)
+    if: |
+      always() &&
+      (needs.generate-and-publish-dbs.result == 'success' || needs.generate-and-publish-dbs.result == 'skipped') &&
+      github.event.inputs.publish-listing != 'false'
+
+    name: "Publish listing file"
+    needs: generate-and-publish-dbs
+    runs-on: ubuntu-22.04-4core-16gb
+    steps:
+
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
+        with:
+          submodules: true
+
+      - name: Bootstrap environment
+        uses: ./.github/actions/bootstrap
+
+      - name: Publish listing file
+        run: |
+          poetry run \
+            grype-db-manager \
+              -c ./config/grype-db-manager/publish-production-r2.yaml \
+                listing update
+        env:
+            AWS_ACCESS_KEY_ID: ${{ secrets.TOOLBOX_CLOUDFLARE_ACCESS_KEY_ID }}
+            AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLBOX_CLOUDFLARE_SECRET_ACCESS_KEY }}
+            GRYPE_DB_MANAGER_DISTRIBUTION_S3_ENDPOINT_URL: ${{ secrets.TOOLBOX_CLOUDFLARE_R2_ENDPOINT }}
+
+      - uses: 8398a7/action-slack@28ba43ae48961b90635b50953d216767a6bea486 #v3.16.2
+        with:
+          status: ${{ job.status }}
+          fields: workflow,eventName,job
+          text: Publishing the Grype DB listing file has failed
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }}
+        if: ${{ failure() && env.SLACK_NOTIFICATIONS == 'true' }}
diff --git a/config/grype-db-manager/include.d/distribution-production-r2.yaml b/config/grype-db-manager/include.d/distribution-production-r2.yaml
@@ -0,0 +1,7 @@
+# distribution:
+
+listing-file-name: listing.json
+s3-path: databases
+s3-bucket: oss-prod-anchore
+aws-region: auto
+download-url-prefix: https://grype.anchore.io
diff --git a/config/grype-db-manager/include.d/grype-db-local-build-r2.yaml b/config/grype-db-manager/include.d/grype-db-local-build-r2.yaml
@@ -0,0 +1,9 @@
+# grype-db:
+
+# use the current repo at the current commit as the source of truth for the grype-db build source.
+# note: assume this will be invoked from the root of the repo
+version: file://.
+
+# grype-db application configuration to use.
+# note: assume this will be invoked from the root of the repo
+config: config/grype-db/publish-nightly-r2.yaml
diff --git a/config/grype-db-manager/publish-production-r2.yaml b/config/grype-db-manager/publish-production-r2.yaml
@@ -0,0 +1,9 @@
+# this configuration is intended to be used for nightly builds of the database in production
+
+data: !include config/grype-db-manager/include.d/data.yaml
+
+grype-db: !include config/grype-db-manager/include.d/grype-db-local-build-r2.yaml
+
+distribution: !include config/grype-db-manager/include.d/distribution-production-r2.yaml
+
+validate: !include config/grype-db-manager/include.d/validate.yaml
diff --git a/config/grype-db/publish-nightly-r2.yaml b/config/grype-db/publish-nightly-r2.yaml
@@ -0,0 +1,26 @@
+# this is a grype-db application configuration file intended for use with the daily db publisher workflow
+
+provider:
+  root: data/vunnel
+
+  # No manual configs are provided since 'provider.vunnel.generateConfigs' is set to true
+  # this means that well run vunnel to get the list of supported providers. All supported providers
+  # will be included in the database build. This prevents the need from manually updating this file
+  # for every new provider that is added.
+  #
+  # Any providers that should be excluded from processing should be added to the 'provider.vunnel.excludeProviders' list.
+  configs: []
+
+  vunnel:
+    executor: docker
+    docker-tag: latest
+    generate-configs: true
+    env:
+      GITHUB_TOKEN: $GITHUB_TOKEN
+      NVD_API_KEY: $NVD_API_KEY
+
+pull:
+  parallelism: 4
+
+package:
+  publish-base-url: https://grype.anchore.io/databases