This commit introduces a new `housekeeping` value for the `irq-load-balancing.crio.io` annotation.

When `housekeeping` is set:
* The housekeeping CPU set is injected into the container's environment variables as `OPENSHIFT_HOUSEKEEPING_CPUS`
* IRQ SMP affinity bits are not disabled on the housekeeping CPUs when adding a new container
* The housekeeping CPUs are chosen as the first CPU within each container plus its thread siblings

#!/bin/bash

set -eux

cleanup() {
    set +e
    set +u
    echo "Running cleanup..."
    kubectl delete pod qos-demo
    kubectl wait --for=delete pod/qos-demo --timeout=180s
    if [ "$crio_pid" != "" ]; then
    	kill $crio_pid
    fi
    sleep 10
    killall crio || true
    sleep 5
    systemctl restart crio
    wait_cluster_ready

    set +x
    echo "======================"
    echo "For logs, see $tmp_log"
    echo "======================"
}
trap cleanup EXIT

# Adjust this to your cluster, in my case, I need 7 pods with READY 1/1 to be up for the cluster
# to be o.k.
wait_cluster_ready() {
    for i in {0..9}; do
         echo "Waiting for cluster ready, iteration $i"
         sleep 15
	 out=$(timeout 10 kubectl get pods -A | grep "1/1" | wc -l | tr -d '\n')
         if [ "${out}" == "7" ]; then
             return
         else
             echo "Cluster not ready: $out"
	 fi
    done
    exit 1
}

generate_pod_yaml() {
    i=$1
    f="$(mktemp)"
    envsubst < ${DIR}/${pods_yaml[$i]} > $f
    echo $f
}

get_events() {
    i=$1
    sleep 2
    for c in {0..9}; do
        if ! kubectl get events | grep -q "${expected_event[$i]}"; then
            echo "Couldn't find expected event, expected: ${expected_event[$i]}"
        else
            kubectl get events | grep "${expected_event[$i]}"
            return
        fi
        sleep 30
    done
    echo "Couldn't find events after $((i + 1)) tries"
    exit 1
}


DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )

declare -A annotations
annotations[0]='annotations:
   irq-load-balancing.crio.io: "housekeeping"'

annotations[1]='annotations:
    irq-load-balancing.crio.io: "disable"'

annotations[2]='annotations:
    irq-load-balancing.crio.io: "true"'

annotations[3]='annotations:
    irq-load-balancing.crio.io: "garbage"'

annotations[4]='annotations:
    irq-load-balancing.crio.io: ""'

declare -A expected_smp_affinity
expected_smp_affinity[0]="3d1f"
expected_smp_affinity[1]="3c0f"
expected_smp_affinity[2]="3c0f"
expected_smp_affinity[3]="3fff"
expected_smp_affinity[4]="3fff"

declare -A expected_event
expected_event[0]=""
expected_event[1]=""
expected_event[2]=""
expected_event[3]=""
expected_event[4]=""

declare -A pods_yaml
pods_yaml[0]="pods.yaml.j2"
pods_yaml[1]="pods.yaml.j2"
pods_yaml[2]="pods.yaml.j2"
pods_yaml[3]="pods.yaml.j2"
pods_yaml[4]="pods.yaml.j2"

expected_reset_affinity="3fff"
affinity_file="/proc/irq/default_smp_affinity"
CRIO="/home/akaris/development/cri-o/bin/crio"
tmp_log="$(mktemp)"

systemctl stop crio
sleep 5
killall crio || true
sleep 5
# Start from clean sheet for SMP affinity ...
echo $expected_reset_affinity > $affinity_file
# Start custom crio ...
${CRIO} 2>&1 | tee "${tmp_log}" | grep smp &
crio_pid=$!
echo "CRIO PID: ${crio_pid}"

wait_cluster_ready

mask=$(cat ${affinity_file} | tr -d '\n')
echo "Starting SMP affinity mask: $mask"
set +x
echo ""
echo ""
echo ""
set -x
for j in {0..1}; do
for i in "${!annotations[@]}"; do
        kubectl delete events --all --timeout=10s
	set +x
	echo "======================="
	echo "Running smoke test ${j}|${i}"
	echo "======================="
	set -x
	export annotation="${annotations[$i]}"
        echo -e "$annotation"
	timeout 15 kubectl apply -f $(generate_pod_yaml $i)
	if [ "${expected_event[$i]}" == "" ]; then
	    kubectl wait --for=condition=Ready pod/qos-demo --timeout=120s
	else
	    kubectl wait --for=condition=Ready pod/qos-demo --timeout=120s || true
	fi
	mask=$(cat ${affinity_file} | tr -d '\n')
	expected_mask="${expected_smp_affinity[$i]}"
	echo "Got mask: $mask, expected mask: $expected_mask"
	if [ "${expected_event[$i]}" == "" ]; then
	    set +x
	    echo "=== HOUSEKEEPING ENV VARS IN CONTAINERS: ==="
	    for cname in qos-demo-ctr qos-demo-ctr-2 qos-demo-ctr-3; do
	       echo "Container $cname"
	       kubectl exec -c $cname qos-demo -- env | grep -i house || true
            done
	    set -x
	    if [ "${mask}" != "${expected_mask}" ]; then
	        exit 1
	    fi
	else
	    get_events $i
	fi
	kubectl delete pod qos-demo
	kubectl wait --for=delete pod/qos-demo --timeout=180s
	mask=$(cat ${affinity_file} | tr -d '\n')
	echo "After reset --- Got mask: $mask, expected mask: $expected_reset_affinity"
	if [ "${mask}" != "${expected_reset_affinity}" ]; then
	    echo "WARN - NOT RESET CORRECTLY BUT MAY BE DUE TO RACE"
	    echo $expected_reset_affinity > $affinity_file
	    # exit 1
	fi
	set +x
	echo ""
	echo ""
	echo ""
	echo ""
	set -x
done
done

apiVersion: v1
kind: Pod
metadata:
  name: qos-demo
  $annotation
spec:
  hostNetwork: true
  runtimeClassName: performance-performance
  containers:
  - name: qos-demo-ctr
    image: quay.io/akaris/nice-test
    command:
    - "/bin/sleep"
    - "infinity"
    resources:
      limits:
        memory: "1Gi"
        cpu: "4"
      requests:
        memory: "1Gi"
        cpu: "4"
  - name: qos-demo-ctr-2
    image: quay.io/akaris/nice-test
    command:
    - "/bin/sleep"
    - "infinity"
    resources:
      limits:
        memory: "1Gi"
        cpu: "2"
      requests:
        memory: "1Gi"
        cpu: "2"
  - name: qos-demo-ctr-3
    image: quay.io/akaris/nice-test
    command:
    - "/bin/sleep"
    - "infinity"
    resources:
      limits:
        memory: "1Gi"
        cpu: "1200m"
      requests:
        memory: "1Gi"
        cpu: "1200m"

[root@centos9 pod]# grep -i full /var/lib/kubelet/config.yaml -C4
syncFrequency: 0s
volumeStatsAggPeriod: 0s
cpuManagerPolicy: static
cpuManagerPolicyOptions:
  full-pcpus-only: "true"
cpuManagerReconcilePeriod: 5s
reservedSystemCPUs: 0-3

# cat /etc/crio/crio.conf.d/99-runtimes.conf 
[crio.runtime]
infra_ctr_cpuset = "0-3"




# The CRI-O will check the allowed_annotations under the runtime handler and apply high-performance hooks when one of
# high-performance annotations presents under it.
# We should provide the runtime_path because we need to inform that we want to re-use runc binary and we
# do not have high-performance binary under the $PATH that will point to it.
[crio.runtime.runtimes.high-performance]
inherit_default_runtime = true
allowed_annotations = ["cpu-load-balancing.crio.io", "cpu-quota.crio.io", "irq-load-balancing.crio.io", "cpu-c-states.crio.io", "cpu-freq-governor.crio.io"]

expected_smp_affinity[0]="3f3f"