Add container out of memory metrics

saschagrunert · Paul Lou · commit 5b79f0e18fbb · 2022-03-04T00:01:12.000-08:00
This patch adds a new metric
`container_runtime_crio_containers_oom_total` and
`container_runtime_crio_containers_oom`, which collects out of memory
(oom) metrics global and by container name. This also includes
sandboxes, since we reuse the CRI-O internal name for every container.

Signed-off-by: Sascha Grunert &lt;sgrunert@redhat.com&gt;
diff --git a/internal/oci/runtime_oci.go b/internal/oci/runtime_oci.go
@@ -18,6 +18,7 @@ import (
 	"github.com/containers/storage/pkg/pools"
 	"github.com/cri-o/cri-o/internal/log"
 	"github.com/cri-o/cri-o/pkg/config"
+	"github.com/cri-o/cri-o/server/metrics"
 	"github.com/cri-o/cri-o/utils"
 	"github.com/fsnotify/fsnotify"
 	json "github.com/json-iterator/go"
@@ -892,6 +893,17 @@ func (r *runtimeOCI) UpdateContainerStatus(c *Container) error {
 	oomFilePath := filepath.Join(c.bundlePath, "oom")
 	if _, err = os.Stat(oomFilePath); err == nil {
 		c.state.OOMKilled = true
+
+		// Collect total metric
+		metrics.CRIOContainersOOMTotal.Inc()
+
+		// Collect metric by container name
+		counter, err := metrics.CRIOContainersOOM.GetMetricWithLabelValues(c.Name())
+		if err != nil {
+			logrus.Warnf("Unable to write OOM metric by container: %v", err)
+		} else {
+			counter.Inc()
+		}
 	}
 
 	return nil
diff --git a/internal/oci/runtime_vm.go b/internal/oci/runtime_vm.go
@@ -17,6 +17,7 @@ import (
 	"github.com/containerd/containerd/runtime/v2/task"
 	"github.com/containerd/ttrpc"
 	"github.com/containers/libpod/pkg/cgroups"
+	"github.com/cri-o/cri-o/server/metrics"
 	"github.com/cri-o/cri-o/utils"
 	"github.com/cri-o/cri-o/utils/errdefs"
 	"github.com/cri-o/cri-o/utils/fifo"
@@ -664,6 +665,17 @@ func (r *runtimeVM) updateContainerStatus(c *Container) error {
 		oomFilePath := filepath.Join(c.bundlePath, "oom")
 		if _, err = os.Stat(oomFilePath); err == nil {
 			c.state.OOMKilled = true
+
+			// Collect total metric
+			metrics.CRIOContainersOOMTotal.Inc()
+
+			// Collect metric by container name
+			counter, err := metrics.CRIOContainersOOM.GetMetricWithLabelValues(c.Name())
+			if err != nil {
+				logrus.Warnf("Unable to write OOM metric by container: %v", err)
+			} else {
+				counter.Inc()
+			}
 		}
 	}
 	return nil
@@ -729,12 +741,12 @@ func (r *runtimeVM) ContainerStats(c *Container, _ string) (*ContainerStats, err
 		return nil, errors.Wrap(err, "failed to extract container metrics")
 	}
 
-	metrics, ok := stats.(*cgroups.Metrics)
+	m, ok := stats.(*cgroups.Metrics)
 	if !ok {
 		return nil, errors.Errorf("Unknown stats type %T", stats)
 	}
 
-	return metricsToCtrStats(c, metrics), nil
+	return metricsToCtrStats(c, m), nil
 }
 
 // SignalContainer sends a signal to a container process.
diff --git a/server/metrics/metrics.go b/server/metrics/metrics.go
@@ -35,6 +35,12 @@ const (
 	// CRIOImageLayerReuseKey is the key for the CRI-O image layer reuse metrics.
 	CRIOImageLayerReuseKey = "crio_image_layer_reuse"
 
+	// CRIOContainersOOMTotalKey is the key for the total CRI-O container out of memory metrics.
+	CRIOContainersOOMTotalKey = "crio_containers_oom_total"
+
+	// CRIOContainersOOMKey is the key for the CRI-O container out of memory metrics per container name.
+	CRIOContainersOOMKey = "crio_containers_oom"
+
 	subsystem = "container_runtime"
 )
 
@@ -131,6 +137,25 @@ var (
 		},
 		[]string{"name"},
 	)
+
+	// CRIOContainersOOMTotal collects container out of memory (oom) metrics for every container and sandboxes.
+	CRIOContainersOOMTotal = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Subsystem: subsystem,
+			Name:      CRIOContainersOOMTotalKey,
+			Help:      "Amount of containers killed because they ran out of memory (OOM)",
+		},
+	)
+
+	// CRIOContainersOOM collects container out of memory (oom) metrics per container and sandbox name.
+	CRIOContainersOOM = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Subsystem: subsystem,
+			Name:      CRIOContainersOOMKey,
+			Help:      "Amount of containers killed because they ran out of memory (OOM) by their name",
+		},
+		[]string{"name"},
+	)
 )
 
 var registerMetrics sync.Once
@@ -147,6 +172,8 @@ func Register() {
 		prometheus.MustRegister(CRIOImagePullsFailures)
 		prometheus.MustRegister(CRIOImagePullsSuccesses)
 		prometheus.MustRegister(CRIOImageLayerReuse)
+		prometheus.MustRegister(CRIOContainersOOMTotal)
+		prometheus.MustRegister(CRIOContainersOOM)
 	})
 }
 
diff --git a/test/ctr.bats b/test/ctr.bats
@@ -1494,72 +1494,6 @@ function wait_until_exit() {
 	[ "$status" -eq 0 ]
 }
 
-@test "ctr expose metrics with default port" {
-	# start crio with default port 9090
-	port="9090"
-	CONTAINER_ENABLE_METRICS=true start_crio
-	# ensure metrics port is listening
-	listened=$(check_metrics_port $port)
-	if [[ "$listened" -ne 0 ]]; then
-		skip "$CONTAINER_METRICS_PORT is not listening"
-	fi
-
-	run crictl runp "$TESTDATA"/sandbox_config.json
-	echo "$output"
-	[ "$status" -eq 0 ]
-	pod_id="$output"
-	run crictl create "$pod_id" "$TESTDATA"/container_redis.json "$TESTDATA"/sandbox_config.json
-	echo "$output"
-	[ "$status" -eq 0 ]
-	ctr_id="$output"
-	run crictl start "$ctr_id"
-	[ "$status" -eq 0 ]
-
-	# get metrics
-	run curl http://localhost:$port/metrics -k
-	[ "$status" -eq 0 ]
-
-	run crictl stopp "$pod_id"
-	echo "$output"
-	[ "$status" -eq 0 ]
-	run crictl rmp "$pod_id"
-	echo "$output"
-	[ "$status" -eq 0 ]
-}
-
-@test "ctr expose metrics with custom port" {
-	# start crio with custom port
-	port="4321"
-	CONTAINER_ENABLE_METRICS=true CONTAINER_METRICS_PORT=$port start_crio
-	# ensure metrics port is listening
-	listened=$(check_metrics_port $port)
-	if [[ "$listened" -ne 0 ]]; then
-		skip "$CONTAINER_METRICS_PORT is not listening"
-	fi
-
-	run crictl runp "$TESTDATA"/sandbox_config.json
-	echo "$output"
-	[ "$status" -eq 0 ]
-	pod_id="$output"
-	run crictl create "$pod_id" "$TESTDATA"/container_redis.json "$TESTDATA"/sandbox_config.json
-	echo "$output"
-	[ "$status" -eq 0 ]
-	ctr_id="$output"
-	run crictl start "$ctr_id"
-	[ "$status" -eq 0 ]
-
-	# get metrics
-	run curl http://localhost:$port/metrics -k
-	[ "$status" -eq 0 ]
-
-	run crictl stopp "$pod_id"
-	echo "$output"
-	[ "$status" -eq 0 ]
-	run crictl rmp "$pod_id"
-	echo "$output"
-	[ "$status" -eq 0 ]
-}
-
 
 @test "privileged ctr -- check for rw mounts" {
 	start_crio
diff --git a/test/helpers.bash b/test/helpers.bash
@@ -369,6 +369,11 @@ function check_journald() {
     echo "0"
 }
 
+# get a random available port
+function free_port() {
+    python -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()'
+}
+
 # Check whether metrics port is listening
 function check_metrics_port() {
     if ! netstat -lanp | grep "$1" >/dev/null; then
diff --git a/test/metrics.bats b/test/metrics.bats
@@ -0,0 +1,64 @@
+#!/usr/bin/env bats
+# vim: set syntax=sh:
+
+load helpers
+
+function setup() {
+	setup_test
+}
+
+function teardown() {
+	cleanup_test
+}
+
+@test "metrics with default port" {
+	# start crio with default port 9090
+	PORT="9090"
+	CONTAINER_ENABLE_METRICS=true start_crio
+	if ! port_listens "$PORT"; then
+		skip "Metrics port $PORT not listening"
+	fi
+
+	# get metrics
+	curl -sf "http://localhost:$PORT/metrics"
+}
+
+@test "metrics with random port" {
+	# start crio with custom port
+	PORT=$(free_port)
+	CONTAINER_ENABLE_METRICS=true CONTAINER_METRICS_PORT=$PORT start_crio
+
+	crictl run "$TESTDATA"/container_redis.json "$TESTDATA"/sandbox_config.json
+
+	# get metrics
+	curl -sf "http://localhost:$PORT/metrics" | grep crio_operations
+}
+
+@test "metrics container oom" {
+	PORT=$(free_port)
+	CONTAINER_ENABLE_METRICS=true CONTAINER_METRICS_PORT=$PORT start_crio
+
+	jq '.image.image = "quay.io/crio/oom"
+        | .linux.resources.memory_limit_in_bytes = 25165824
+        | .command = ["/oom"]' \
+		"$TESTDATA/container_config.json" > "$TESTDIR/config.json"
+	CTR_ID=$(crictl run "$TESTDIR/config.json" "$TESTDATA/sandbox_config.json")
+
+	# Wait for container to OOM
+	CNT=0
+	while [ $CNT -le 100 ]; do
+		CNT=$((CNT + 1))
+		OUTPUT=$(crictl inspect --output yaml "$CTR_ID")
+		if [[ "$OUTPUT" == *"OOMKilled"* ]]; then
+			break
+		fi
+		sleep 10
+	done
+	[[ "$OUTPUT" == *"OOMKilled"* ]]
+
+	METRIC=$(curl -sf "http://localhost:$PORT/metrics" | grep '^container_runtime_crio_containers_oom_total')
+	[[ "$METRIC" == 'container_runtime_crio_containers_oom_total 1' ]]
+
+	METRIC=$(curl -sf "http://localhost:$PORT/metrics" | grep 'crio_containers_oom{')
+	[[ "$METRIC" == 'container_runtime_crio_containers_oom{name="k8s_container1_podsandbox1_redhat.test.crio_redhat-test-crio_1"} 1' ]]
+}
diff --git a/tutorials/metrics.md b/tutorials/metrics.md
@@ -38,6 +38,8 @@ Beside the [default golang based metrics][2], CRI-O provides the following addit
 | `crio_image_pulls_by_name_skipped`     | `name`                                                                                                                                  | Counter | Bytes skipped by CRI-O image pulls by name.                              |
 | `crio_image_pulls_successes`           | `name`                                                                                                                                  | Counter | Successful image pulls by image name                                     |
 | `crio_image_pulls_failures`            | `name`, `error`                                                                                                                         | Counter | Failed image pulls by image name and their error category.               |
+| `crio_containers_oom_total`                  |                                                                                                                                         | Counter | Total number of containers killed because they ran out of memory (OOM)                               |
+| `crio_containers_oom`                        | `name`                                                                                                                                  | Counter | Containers killed because they ran out of memory (OOM) by their name                                 |
 
 - Available CRI-O RPC's from the [gRPC API][3]: `Attach`, `ContainerStats`, `ContainerStatus`,
   `CreateContainer`, `Exec`, `ExecSync`, `ImageFsInfo`, `ImageStatus`,
