Skip to content

Commit f1d3a46

Browse files
Merge pull request cri-o#5571 from klihub/devel/cdi-support
server: add support for CDI device injection.
2 parents 71e9257 + 1281651 commit f1d3a46

File tree

121 files changed

+3157
-286
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

121 files changed

+3157
-286
lines changed

completions/bash/crio

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ h
2020
--big-files-temporary-dir
2121
--bind-mount-prefix
2222
--blockio-config-file
23+
--cdi-spec-dirs
2324
--cgroup-manager
2425
--clean-shutdown-file
2526
--cni-config-dir

completions/fish/crio.fish

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ complete -c crio -n '__fish_crio_no_subcommand' -f -l apparmor-profile -r -d 'Na
1616
complete -c crio -n '__fish_crio_no_subcommand' -f -l big-files-temporary-dir -r -d 'Path to the temporary directory to use for storing big files, used to store image blobs and data streams related to containers image management.'
1717
complete -c crio -n '__fish_crio_no_subcommand' -f -l bind-mount-prefix -r -d 'A prefix to use for the source of the bind mounts. This option would be useful if you were running CRI-O in a container. And had `/` mounted on `/host` in your container. Then if you ran CRI-O with the `--bind-mount-prefix=/host` option, CRI-O would add /host to any bind mounts it is handed over CRI. If Kubernetes asked to have `/var/lib/foobar` bind mounted into the container, then CRI-O would bind mount `/host/var/lib/foobar`. Since CRI-O itself is running in a container with `/` or the host mounted on `/host`, the container would end up with `/var/lib/foobar` from the host mounted in the container rather then `/var/lib/foobar` from the CRI-O container. (default: "")'
1818
complete -c crio -n '__fish_crio_no_subcommand' -f -l blockio-config-file -r -d 'Path to the blockio class configuration file for configuring the cgroup blockio controller.'
19+
complete -c crio -n '__fish_crio_no_subcommand' -f -l cdi-spec-dirs -r -d 'Directories to scan for CDI Spec files'
1920
complete -c crio -n '__fish_crio_no_subcommand' -f -l cgroup-manager -r -d 'cgroup manager (cgroupfs or systemd)'
2021
complete -c crio -n '__fish_crio_no_subcommand' -l clean-shutdown-file -r -d 'Location for CRI-O to lay down the clean shutdown file. It indicates whether we\'ve had time to sync changes to disk before shutting down. If not found, crio wipe will clear the storage directory'
2122
complete -c crio -n '__fish_crio_no_subcommand' -l cni-config-dir -r -d 'CNI configuration files directory'

completions/zsh/_crio

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ it later with **--config**. Global options will modify the output.'
2727
'--big-files-temporary-dir'
2828
'--bind-mount-prefix'
2929
'--blockio-config-file'
30+
'--cdi-spec-dirs'
3031
'--cgroup-manager'
3132
'--clean-shutdown-file'
3233
'--cni-config-dir'

docs/crio.8.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ crio
1818
[--big-files-temporary-dir]=[value]
1919
[--bind-mount-prefix]=[value]
2020
[--blockio-config-file]=[value]
21+
[--cdi-spec-dirs]=[value]
2122
[--cgroup-manager]=[value]
2223
[--clean-shutdown-file]=[value]
2324
[--cni-config-dir]=[value]
@@ -148,6 +149,8 @@ crio [GLOBAL OPTIONS] command [COMMAND OPTIONS] [ARGUMENTS...]
148149

149150
**--blockio-config-file**="": Path to the blockio class configuration file for configuring the cgroup blockio controller.
150151

152+
**--cdi-spec-dirs**="": Directories to scan for CDI Spec files (default: [/etc/cdi /var/run/cdi])
153+
151154
**--cgroup-manager**="": cgroup manager (cgroupfs or systemd) (default: systemd)
152155

153156
**--clean-shutdown-file**="": Location for CRI-O to lay down the clean shutdown file. It indicates whether we've had time to sync changes to disk before shutting down. If not found, crio wipe will clear the storage directory (default: /var/lib/crio/clean.shutdown)

docs/crio.conf.5.md

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,17 @@ the container runtime configuration.
141141
**blockio_config_file**=""
142142
Path to the blockio class configuration file for configuring the cgroup blockio controller.
143143

144+
**cdi_spec_dirs**=[]
145+
Directories to scan for Container Device Interface Specifications to enable CDI device injection. For more details about CDI and the syntax of CDI Spec files please refer to https://github.com/container-orchestrated-devices/container-device-interface.
146+
147+
Directories later in the list have precedence over earlier ones. The default directory list is:
148+
```
149+
cdi_spec_dirs = [
150+
"/etc/cdi",
151+
"/var/run/cdi",
152+
]
153+
```
154+
144155
**irqbalance_config_file**="/etc/sysconfig/irqbalance"
145156
Used to change irqbalance service config file which is used by CRI-O.
146157
For CentOS/SUSE, this file is located at /etc/sysconfig/irqbalance. For Ubuntu, this file is located at /etc/default/irqbalance.

go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ require (
66
github.com/BurntSushi/toml v0.4.1
77
github.com/Microsoft/go-winio v0.5.1
88
github.com/blang/semver v3.5.1+incompatible
9+
github.com/container-orchestrated-devices/container-device-interface v0.0.0-20220224133719-e5457123010b
910
github.com/containerd/cgroups v1.0.2
1011
github.com/containerd/containerd v1.5.8
1112
github.com/containerd/cri-containerd v1.19.0
@@ -62,7 +63,7 @@ require (
6263
go.opentelemetry.io/otel/sdk v1.2.0
6364
golang.org/x/net v0.0.0-20211005001312-d4b1ae081e3b
6465
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
65-
golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef
66+
golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c
6667
google.golang.org/grpc v1.43.0
6768
k8s.io/api v0.22.2
6869
k8s.io/apimachinery v0.22.2
@@ -94,7 +95,6 @@ require (
9495
github.com/cheggaaa/pb/v3 v3.0.5 // indirect
9596
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
9697
github.com/cilium/ebpf v0.6.2 // indirect
97-
github.com/container-orchestrated-devices/container-device-interface v0.0.0-20210325223243-f99e8b6c10b9 // indirect
9898
github.com/containerd/console v1.0.2 // indirect
9999
github.com/containerd/fifo v1.0.0 // indirect
100100
github.com/containerd/go-runc v1.0.0 // indirect

go.sum

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -239,8 +239,9 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH
239239
github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
240240
github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
241241
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
242-
github.com/container-orchestrated-devices/container-device-interface v0.0.0-20210325223243-f99e8b6c10b9 h1:Kn0s9/APRtr5dk/83aXj97WX0+PYnJK9BO8g0Xclm0I=
243242
github.com/container-orchestrated-devices/container-device-interface v0.0.0-20210325223243-f99e8b6c10b9/go.mod h1:eQt66kIaJpUhCrjCtBFQGQxGLbAUl0OuuwjTH16ON4s=
243+
github.com/container-orchestrated-devices/container-device-interface v0.0.0-20220224133719-e5457123010b h1:kfww7SrumR0HoZTwEXRDy2CzwhhmigCWK3mvvhYDaUQ=
244+
github.com/container-orchestrated-devices/container-device-interface v0.0.0-20220224133719-e5457123010b/go.mod h1:E1zcucIkq9P3eyNmY+68dBQsTcsXJh9cgRo2IVNScKQ=
244245
github.com/container-storage-interface/spec v1.5.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s=
245246
github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE=
246247
github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU=
@@ -1074,6 +1075,7 @@ github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.m
10741075
github.com/opencontainers/runtime-spec v1.0.3-0.20210709190330-896175883324 h1:EPCbQw9GYqjcp8Y5Oz/RzEO3hlxA1MiQJ3DiYgzzUqg=
10751076
github.com/opencontainers/runtime-spec v1.0.3-0.20210709190330-896175883324/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
10761077
github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
1078+
github.com/opencontainers/runtime-tools v0.0.0-20190417131837-cd1349b7c47e/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
10771079
github.com/opencontainers/runtime-tools v0.9.0/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
10781080
github.com/opencontainers/runtime-tools v0.9.1-0.20210326182921-59cdde06764b h1:BDT83NWjBPS3Esj/BS1Cf9F4NYzYmiitJZ0TlXx5M4w=
10791081
github.com/opencontainers/runtime-tools v0.9.1-0.20210326182921-59cdde06764b/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
@@ -1700,8 +1702,9 @@ golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBc
17001702
golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
17011703
golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
17021704
golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1703-
golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef h1:fPxZ3Umkct3LZ8gK9nbk+DWDJ9fstZa2grBn+lWVKPs=
17041705
golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1706+
golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c h1:DHcbWVXeY+0Y8HHKR+rbLwnoh2F4tNCY7rTiHJ30RmA=
1707+
golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
17051708
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
17061709
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
17071710
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

internal/criocli/criocli.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,9 @@ func mergeConfig(config *libconfig.Config, ctx *cli.Context) error {
261261
if ctx.IsSet("allowed-devices") {
262262
config.AllowedDevices = StringSliceTrySplit(ctx, "allowed-devices")
263263
}
264+
if ctx.IsSet("cdi-spec-dirs") {
265+
config.CDISpecDirs = StringSliceTrySplit(ctx, "cdi-spec-dirs")
266+
}
264267
if ctx.IsSet("device-ownership-from-security-context") {
265268
config.DeviceOwnershipFromSecurityContext = ctx.Bool("device-ownership-from-security-context")
266269
}
@@ -842,6 +845,12 @@ func getCrioFlags(defConf *libconfig.Config) []cli.Flag {
842845
Value: cli.NewStringSlice(defConf.AdditionalDevices...),
843846
EnvVars: []string{"CONTAINER_ADDITIONAL_DEVICES"},
844847
},
848+
&cli.StringSliceFlag{
849+
Name: "cdi-spec-dirs",
850+
Usage: "Directories to scan for CDI Spec files",
851+
Value: cli.NewStringSlice(defConf.CDISpecDirs...),
852+
EnvVars: []string{"CONTAINER_CDI_SPEC_DIRS"},
853+
},
845854
&cli.BoolFlag{
846855
Name: "device-ownership-from-security-context",
847856
Usage: "Set devices' uid/gid ownership from runAsUser/runAsGroup",

internal/factory/container/device.go

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
11
package container
22

33
import (
4+
"context"
45
"os"
56
"path/filepath"
67
"strings"
78

9+
"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
810
devicecfg "github.com/cri-o/cri-o/internal/config/device"
11+
12+
"github.com/cri-o/cri-o/internal/log"
913
"github.com/cri-o/cri-o/utils"
1014
securejoin "github.com/cyphar/filepath-securejoin"
1115
"github.com/opencontainers/runc/libcontainer/devices"
@@ -39,8 +43,13 @@ func (c *container) SpecAddDevices(configuredDevices, annotationDevices []device
3943
return err
4044
}
4145

42-
// Finally, add container config devices
43-
return c.specAddContainerConfigDevices(enableDeviceOwnershipFromSecurityContext)
46+
// Then, add container config devices
47+
if err := c.specAddContainerConfigDevices(enableDeviceOwnershipFromSecurityContext); err != nil {
48+
return err
49+
}
50+
51+
// Finally, inject CDI devices
52+
return c.specInjectCDIDevices()
4453
}
4554

4655
func (c *container) specAddHostDevicesIfPrivileged(privilegedWithoutHostDevices bool) error {
@@ -165,6 +174,38 @@ func (c *container) specAddContainerConfigDevices(enableDeviceOwnershipFromSecur
165174
return nil
166175
}
167176

177+
func (c *container) specInjectCDIDevices() error {
178+
// TODO: Once CRI is extended with native CDI support this will need to be updated...
179+
_, names, err := cdi.ParseAnnotations(c.Config().GetAnnotations())
180+
if err != nil {
181+
return errors.Wrap(err, "failed to parse CDI device annotations")
182+
}
183+
if names == nil {
184+
return nil
185+
}
186+
187+
registry := cdi.GetRegistry()
188+
if err := registry.Refresh(); err != nil {
189+
// We don't consider registry refresh failure a fatal error.
190+
// For instance, a dynamically generated invalid CDI Spec file for
191+
// any particular vendor shouldn't prevent injection of devices of
192+
// different vendors. CDI itself knows better and it will fail the
193+
// injection if necessary.
194+
195+
log.Warnf(context.TODO(), "CDI registry has errors: %v", err)
196+
}
197+
198+
if _, err := registry.InjectDevices(c.Spec().Config, names...); err != nil {
199+
return errors.Wrap(err, "CDI device injection failed")
200+
}
201+
202+
// One crucial thing to keep in mind is that CDI device injection
203+
// might add OCI Spec environment variables, hooks, and mounts as
204+
// well. Therefore it is important that none of the corresponding
205+
// OCI Spec fields are reset up in the call stack once we return.
206+
return nil
207+
}
208+
168209
// getDeviceUserGroupID() is used to find the right uid/gid
169210
// value for the device node created in the container namespace.
170211
// The runtime executes mknod() and chmod()s the created

internal/factory/container/device_test.go

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,16 @@
11
package container_test
22

33
import (
4+
"fmt"
5+
"io/ioutil"
6+
"path/filepath"
7+
48
. "github.com/onsi/ginkgo"
59
. "github.com/onsi/gomega"
10+
11+
rspec "github.com/opencontainers/runtime-spec/specs-go"
12+
13+
"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
614
"github.com/opencontainers/runc/libcontainer/devices"
715
types "k8s.io/cri-api/pkg/apis/runtime/v1"
816
)
@@ -180,4 +188,152 @@ var _ = t.Describe("Container", func() {
180188
})
181189
}
182190
})
191+
192+
t.Describe("SpecAdd(CDI)Devices", func() {
193+
writeCDISpecFiles := func(content []string) error {
194+
if len(content) == 0 {
195+
return nil
196+
}
197+
198+
dir := t.MustTempDir("cdi")
199+
for idx, data := range content {
200+
file := filepath.Join(dir, fmt.Sprintf("spec-%d.yaml", idx))
201+
err := ioutil.WriteFile(file, []byte(data), 0o644)
202+
if err != nil {
203+
return err
204+
}
205+
}
206+
207+
return cdi.GetRegistry(cdi.WithSpecDirs(dir)).Refresh()
208+
}
209+
210+
type testdata struct {
211+
testDescription string
212+
cdiSpecFiles []string
213+
annotations map[string]string
214+
expectError bool
215+
expectDevices []rspec.LinuxDevice
216+
expectEnv []string
217+
}
218+
219+
tests := []testdata{
220+
{
221+
testDescription: "Expect no CDI error for nil annotations",
222+
},
223+
{
224+
testDescription: "Expect no CDI error for empty annotations",
225+
annotations: map[string]string{},
226+
},
227+
{
228+
testDescription: "Expect CDI error for invalid CDI device reference in annotations",
229+
annotations: map[string]string{
230+
cdi.AnnotationPrefix + "devices": "foobar",
231+
},
232+
expectError: true,
233+
},
234+
{
235+
testDescription: "Expect CDI error for unresolvable devices",
236+
annotations: map[string]string{
237+
cdi.AnnotationPrefix + "vendor1_devices": "vendor1.com/device=no-such-dev",
238+
},
239+
expectError: true,
240+
},
241+
{
242+
testDescription: "Expect properly injected resolvable CDI devices",
243+
cdiSpecFiles: []string{
244+
`
245+
cdiVersion: "0.2.0"
246+
kind: "vendor1.com/device"
247+
devices:
248+
- name: foo
249+
containerEdits:
250+
deviceNodes:
251+
- path: /dev/loop8
252+
type: b
253+
major: 7
254+
minor: 8
255+
env:
256+
- FOO=injected
257+
containerEdits:
258+
env:
259+
- "VENDOR1=present"
260+
`,
261+
`
262+
cdiVersion: "0.2.0"
263+
kind: "vendor2.com/device"
264+
devices:
265+
- name: bar
266+
containerEdits:
267+
deviceNodes:
268+
- path: /dev/loop9
269+
type: b
270+
major: 7
271+
minor: 9
272+
env:
273+
- BAR=injected
274+
containerEdits:
275+
env:
276+
- "VENDOR2=present"
277+
`,
278+
},
279+
annotations: map[string]string{
280+
cdi.AnnotationPrefix + "vendor1_devices": "vendor1.com/device=foo",
281+
cdi.AnnotationPrefix + "vendor2_devices": "vendor2.com/device=bar",
282+
},
283+
expectDevices: []rspec.LinuxDevice{
284+
{
285+
Path: "/dev/loop8",
286+
Type: "b",
287+
Major: 7,
288+
Minor: 8,
289+
},
290+
{
291+
Path: "/dev/loop9",
292+
Type: "b",
293+
Major: 7,
294+
Minor: 9,
295+
},
296+
},
297+
expectEnv: []string{
298+
"FOO=injected",
299+
"VENDOR1=present",
300+
"BAR=injected",
301+
"VENDOR2=present",
302+
},
303+
},
304+
}
305+
306+
for _, test := range tests {
307+
test := test
308+
It(test.testDescription, func() {
309+
// Given
310+
config := &types.ContainerConfig{
311+
Metadata: &types.ContainerMetadata{Name: "name"},
312+
Annotations: test.annotations,
313+
Linux: &types.LinuxContainerConfig{
314+
SecurityContext: &types.LinuxContainerSecurityContext{},
315+
},
316+
Devices: []*types.Device{},
317+
}
318+
sboxConfig := &types.PodSandboxConfig{
319+
Linux: &types.LinuxPodSandboxConfig{
320+
SecurityContext: &types.LinuxSandboxSecurityContext{},
321+
},
322+
}
323+
Expect(sut.SetConfig(config, sboxConfig)).To(BeNil())
324+
Expect(sut.SetPrivileged()).To(BeNil())
325+
Expect(writeCDISpecFiles(test.cdiSpecFiles)).To(BeNil())
326+
327+
// When
328+
err := sut.SpecAddDevices(nil, nil, false, false)
329+
330+
// Then
331+
Expect(err != nil).To(Equal(test.expectError))
332+
if err == nil {
333+
Expect(sut.Spec().Config.Process.Env).Should(ContainElements(test.expectEnv))
334+
Expect(sut.Spec().Config.Linux.Devices).Should(ContainElements(test.expectDevices))
335+
}
336+
})
337+
}
338+
})
183339
})

0 commit comments

Comments
 (0)