Skip to content

Commit a0508c2

Browse files
openshift-merge-robotopenshift-merge-robot
authored
Merge pull request cri-o#6379 from adrianreber/2022-11-15-infra-less
Support checkpointing infra less containers
2 parents 0cd8020 + fe24583 commit a0508c2

File tree

13 files changed

+184
-175
lines changed

13 files changed

+184
-175
lines changed

‎internal/lib/restore.go‎

Lines changed: 54 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88

99
metadata "github.com/checkpoint-restore/checkpointctl/lib"
1010
"github.com/checkpoint-restore/go-criu/v6/stats"
11+
"github.com/containers/podman/v4/pkg/annotations"
1112
"github.com/containers/podman/v4/pkg/checkpoint/crutils"
1213
"github.com/containers/storage/pkg/archive"
1314
"github.com/cri-o/cri-o/internal/log"
@@ -31,8 +32,10 @@ func (c *ContainerServer) ContainerRestore(ctx context.Context, opts *ContainerC
3132
}
3233

3334
// Get config.json
34-
configFile := filepath.Join(ctr.Dir(), "config.json")
35-
ctrSpec, err := generate.NewFromFile(configFile)
35+
// This file is generated twice by earlier code. Once in BundlePath() and
36+
// once in Dir(). This code takes the version from Dir(), modifies it and
37+
// overwrites both versions (Dir() and BundlePath())
38+
ctrSpec, err := generate.NewFromFile(filepath.Join(ctr.Dir(), "config.json"))
3639
if err != nil {
3740
return "", err
3841
}
@@ -54,15 +57,6 @@ func (c *ContainerServer) ContainerRestore(ctx context.Context, opts *ContainerC
5457
if err != nil {
5558
return "", err
5659
}
57-
ic := sb.InfraContainer()
58-
if ic == nil {
59-
return "", fmt.Errorf("infra container of sandbox %v not found", sb.Name())
60-
}
61-
infraConfigFile := filepath.Join(ic.BundlePath(), "config.json")
62-
specgen, err := generate.NewFromFile(infraConfigFile)
63-
if err != nil {
64-
return "", err
65-
}
6660

6761
if ctr.RestoreArchive() != "" {
6862
if ctr.RestoreIsOCIImage() {
@@ -209,7 +203,55 @@ func (c *ContainerServer) ContainerRestore(ctx context.Context, opts *ContainerC
209203
}
210204
}
211205

212-
if err := c.runtime.RestoreContainer(ctx, ctr, specgen.Config, ic.State().Pid, sb.CgroupParent()); err != nil {
206+
// We need to adapt the to be restored container to the sandbox created for this container.
207+
208+
// The container will be restored in another sandbox. Adapt to
209+
// namespaces of the new sandbox
210+
for i, n := range ctrSpec.Config.Linux.Namespaces {
211+
if n.Path == "" {
212+
// The namespace in the original container did not point to
213+
// an existing interface. Leave it as it is.
214+
// CRIU will restore the namespace
215+
continue
216+
}
217+
for _, np := range sb.NamespacePaths() {
218+
if string(np.Type()) == string(n.Type) {
219+
ctrSpec.Config.Linux.Namespaces[i].Path = np.Path()
220+
break
221+
}
222+
}
223+
}
224+
225+
// Update Sandbox Name
226+
ctrSpec.AddAnnotation(annotations.SandboxName, sb.Name())
227+
// Update Sandbox ID
228+
ctrSpec.AddAnnotation(annotations.SandboxID, opts.Pod)
229+
230+
mData := fmt.Sprintf(
231+
"k8s_%s_%s_%s_%s0",
232+
ctr.Name(),
233+
sb.KubeName(),
234+
sb.Namespace(),
235+
sb.Metadata().Uid,
236+
)
237+
ctrSpec.AddAnnotation(annotations.Name, mData)
238+
239+
ctr.SetSandbox(opts.Pod)
240+
241+
saveOptions := generate.ExportOptions{}
242+
if err := ctrSpec.SaveToFile(filepath.Join(ctr.Dir(), "config.json"), saveOptions); err != nil {
243+
return "", err
244+
}
245+
if err := ctrSpec.SaveToFile(filepath.Join(ctr.BundlePath(), "config.json"), saveOptions); err != nil {
246+
return "", err
247+
}
248+
249+
if err := c.runtime.RestoreContainer(
250+
ctx,
251+
ctr,
252+
sb.CgroupParent(),
253+
sb.MountLabel(),
254+
); err != nil {
213255
return "", fmt.Errorf("failed to restore container %s: %w", ctr.ID(), err)
214256
}
215257
if err := c.ContainerStateToDisk(ctx, ctr); err != nil {

‎internal/lib/restore_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ var _ = t.Describe("ContainerRestore", func() {
8787
// Then
8888
Expect(err).NotTo(BeNil())
8989
Expect(res).To(Equal(""))
90-
Expect(err.Error()).To(Equal(`infra container of sandbox not found`))
90+
Expect(err.Error()).To(Equal(`failed to restore container containerID: a complete checkpoint for this container cannot be found, cannot restore: stat checkpoint/inventory.img: no such file or directory`))
9191
})
9292
})
9393
t.Describe("ContainerRestore", func() {

‎internal/oci/oci.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ type RuntimeImpl interface {
7676
int32, io.ReadWriteCloser) error
7777
ReopenContainerLog(context.Context, *Container) error
7878
CheckpointContainer(context.Context, *Container, *rspec.Spec, bool) error
79-
RestoreContainer(context.Context, *Container, *rspec.Spec, int, string) error
79+
RestoreContainer(context.Context, *Container, string, string) error
8080
}
8181

8282
// New creates a new Runtime with options provided
@@ -428,11 +428,11 @@ func (r *Runtime) CheckpointContainer(ctx context.Context, c *Container, specgen
428428
}
429429

430430
// RestoreContainer restores a container.
431-
func (r *Runtime) RestoreContainer(ctx context.Context, c *Container, sbSpec *rspec.Spec, infraPid int, cgroupParent string) error {
431+
func (r *Runtime) RestoreContainer(ctx context.Context, c *Container, cgroupParent, mountLabel string) error {
432432
impl, err := r.RuntimeImpl(c)
433433
if err != nil {
434434
return err
435435
}
436436

437-
return impl.RestoreContainer(ctx, c, sbSpec, infraPid, cgroupParent)
437+
return impl.RestoreContainer(ctx, c, cgroupParent, mountLabel)
438438
}

‎internal/oci/oci_test.go‎

Lines changed: 19 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,7 @@ var _ = t.Describe("Oci", func() {
182182
Skip("CRIU is missing or too old.")
183183
}
184184
// Given
185-
beforeEach(sandboxID)
185+
beforeEach()
186186
defer os.RemoveAll("dump.log")
187187
config.Runtimes["runc"] = &libconfig.RuntimeHandler{
188188
RuntimePath: "/bin/true",
@@ -209,7 +209,7 @@ var _ = t.Describe("Oci", func() {
209209
}
210210
// Given
211211
defer os.RemoveAll("dump.log")
212-
beforeEach(sandboxID)
212+
beforeEach()
213213
config.Runtimes["runc"] = &libconfig.RuntimeHandler{
214214
RuntimePath: "/bin/false",
215215
}
@@ -235,57 +235,44 @@ var _ = t.Describe("Oci", func() {
235235
Skip("CRIU is missing or too old.")
236236
}
237237
// Given
238-
beforeEach(sandboxID)
238+
beforeEach()
239239
config.Runtimes["runc"] = &libconfig.RuntimeHandler{
240240
RuntimePath: "/bin/true",
241+
MonitorPath: "/bin/true",
241242
}
242243

243-
specgen := &specs.Spec{
244-
Version: "1.0.0",
245-
}
246244
err := os.Mkdir("checkpoint", 0o700)
247245
Expect(err).To(BeNil())
248246
defer os.RemoveAll("checkpoint")
249247
inventory, err := os.OpenFile("checkpoint/inventory.img", os.O_RDONLY|os.O_CREATE, 0o644)
250248
Expect(err).To(BeNil())
251249
inventory.Close()
252250

253-
// When
254-
err = sut.RestoreContainer(context.Background(), myContainer, specgen, 42, "no-parent-cgroup-exists")
255-
256-
// Then
257-
Expect(err).NotTo(BeNil())
258-
Expect(err.Error()).To(Equal("failed to detect destination sandbox of to be restored container containerID"))
259-
})
260-
It("RestoreContainer should fail with destination sandbox detection", func() {
261-
if !criu.CheckForCriu(criu.PodCriuVersion) {
262-
Skip("CRIU is missing or too old.")
263-
}
264-
// Given
265-
beforeEach("")
266251
specgen := &specs.Spec{
267-
Version: "1.0.0",
252+
Version: "1.0.0",
253+
Annotations: map[string]string{"io.kubernetes.cri-o.SandboxID": "sandboxID"},
254+
Linux: &specs.Linux{
255+
MountLabel: ".",
256+
},
257+
Process: &specs.Process{
258+
SelinuxLabel: "",
259+
},
268260
}
269-
err := os.Mkdir("checkpoint", 0o700)
270-
Expect(err).To(BeNil())
271-
defer os.RemoveAll("checkpoint")
272-
inventory, err := os.OpenFile("checkpoint/inventory.img", os.O_RDONLY|os.O_CREATE, 0o644)
273-
Expect(err).To(BeNil())
274-
inventory.Close()
261+
myContainer.SetSpec(specgen)
275262

276263
// When
277-
err = sut.RestoreContainer(context.Background(), myContainer, specgen, 42, "no-parent-cgroup-exists")
264+
err = sut.RestoreContainer(context.Background(), myContainer, "no-parent-cgroup-exists", "label")
278265

279266
// Then
280267
Expect(err).NotTo(BeNil())
281-
Expect(err.Error()).To(Equal("failed to detect sandbox of to be restored container containerID"))
268+
Expect(err.Error()).To(ContainSubstring("failed"))
282269
})
283270
It("RestoreContainer should fail", func() {
284271
if !criu.CheckForCriu(criu.PodCriuVersion) {
285272
Skip("CRIU is missing or too old.")
286273
}
287274
// Given
288-
beforeEach(sandboxID)
275+
beforeEach()
289276
config.Runtimes["runc"] = &libconfig.RuntimeHandler{
290277
RuntimePath: "/bin/true",
291278
MonitorPath: "/bin/true",
@@ -325,7 +312,7 @@ var _ = t.Describe("Oci", func() {
325312
config.Conmon = "/bin/true"
326313

327314
// When
328-
err = sut.RestoreContainer(context.Background(), myContainer, specgen, 42, "no-parent-cgroup-exists")
315+
err = sut.RestoreContainer(context.Background(), myContainer, "no-parent-cgroup-exists", "label")
329316
defer os.RemoveAll("restore.log")
330317

331318
// Then
@@ -337,16 +324,9 @@ var _ = t.Describe("Oci", func() {
337324
Skip("CRIU is missing or too old.")
338325
}
339326
// Given
340-
beforeEach(sandboxID)
341-
specgen := &specs.Spec{
342-
Version: "1.0.0",
343-
Annotations: map[string]string{"io.kubernetes.cri-o.SandboxID": "sandboxID"},
344-
Linux: &specs.Linux{
345-
MountLabel: ".",
346-
},
347-
}
327+
beforeEach()
348328
// When
349-
err := sut.RestoreContainer(context.Background(), myContainer, specgen, 42, "no-parent-cgroup-exists")
329+
err := sut.RestoreContainer(context.Background(), myContainer, "no-parent-cgroup-exists", "label")
350330

351331
// Then
352332
Expect(err).NotTo(BeNil())

‎internal/oci/runtime_oci.go‎

Lines changed: 2 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ import (
1717
metadata "github.com/checkpoint-restore/checkpointctl/lib"
1818
"github.com/containernetworking/plugins/pkg/ns"
1919
conmonconfig "github.com/containers/conmon/runner/config"
20-
"github.com/containers/podman/v4/pkg/annotations"
2120
"github.com/containers/podman/v4/pkg/checkpoint/crutils"
2221
"github.com/containers/podman/v4/pkg/criu"
2322
"github.com/containers/storage/pkg/pools"
@@ -30,7 +29,6 @@ import (
3029
"github.com/fsnotify/fsnotify"
3130
json "github.com/json-iterator/go"
3231
rspec "github.com/opencontainers/runtime-spec/specs-go"
33-
"github.com/opencontainers/runtime-tools/generate"
3432
"github.com/sirupsen/logrus"
3533
"golang.org/x/net/context"
3634
"golang.org/x/sys/unix"
@@ -1528,7 +1526,7 @@ func (r *runtimeOCI) CheckpointContainer(ctx context.Context, c *Container, spec
15281526
}
15291527

15301528
// RestoreContainer restores a container.
1531-
func (r *runtimeOCI) RestoreContainer(ctx context.Context, c *Container, sbSpec *rspec.Spec, infraPid int, cgroupParent string) error {
1529+
func (r *runtimeOCI) RestoreContainer(ctx context.Context, c *Container, cgroupParent, mountLabel string) error {
15321530
if err := r.checkpointRestoreSupported(); err != nil {
15331531
return err
15341532
}
@@ -1555,78 +1553,6 @@ func (r *runtimeOCI) RestoreContainer(ctx context.Context, c *Container, sbSpec
15551553
return fmt.Errorf("error removing container %s winsz file: %w", c.ID(), err)
15561554
}
15571555

1558-
// Figure out if this container will be restored in another sandbox
1559-
oldSbID := c.Sandbox()
1560-
if oldSbID == "" {
1561-
return fmt.Errorf("failed to detect sandbox of to be restored container %s", c.ID())
1562-
}
1563-
newSbID := sbSpec.Annotations[annotations.SandboxID]
1564-
if newSbID == "" {
1565-
return fmt.Errorf("failed to detect destination sandbox of to be restored container %s", c.ID())
1566-
}
1567-
1568-
// Get config.json to adapt for restore (mostly annotations for restore in another sandbox)
1569-
configFile := filepath.Join(c.BundlePath(), "config.json")
1570-
specgen, err := generate.NewFromFile(configFile)
1571-
if err != nil {
1572-
return err
1573-
}
1574-
1575-
if oldSbID != newSbID {
1576-
// The container will be restored in another (not the original) sandbox
1577-
// Adapt to namespaces of the new sandbox
1578-
for i, n := range specgen.Config.Linux.Namespaces {
1579-
if n.Path == "" {
1580-
// The namespace in the original container did not point to
1581-
// an existing interface. Leave it as it is.
1582-
continue
1583-
}
1584-
for _, on := range sbSpec.Linux.Namespaces {
1585-
if on.Type == n.Type {
1586-
var nsPath string
1587-
if n.Type == rspec.NetworkNamespace {
1588-
// Type for network namespaces is 'network'.
1589-
// The kernel link is 'net'.
1590-
nsPath = fmt.Sprintf("/proc/%d/ns/%s", infraPid, "net")
1591-
} else {
1592-
nsPath = fmt.Sprintf("/proc/%d/ns/%s", infraPid, n.Type)
1593-
}
1594-
specgen.Config.Linux.Namespaces[i].Path = nsPath
1595-
break
1596-
}
1597-
}
1598-
}
1599-
1600-
// Update Sandbox Name
1601-
specgen.AddAnnotation(annotations.SandboxName, sbSpec.Annotations[annotations.Name])
1602-
// Update Sandbox ID
1603-
specgen.AddAnnotation(annotations.SandboxID, newSbID)
1604-
1605-
// Update Name
1606-
ctrMetadata := types.ContainerMetadata{}
1607-
err = json.Unmarshal([]byte(sbSpec.Annotations[annotations.Metadata]), &ctrMetadata)
1608-
if err != nil {
1609-
return err
1610-
}
1611-
ctrName := ctrMetadata.Name
1612-
1613-
podMetadata := types.PodSandboxMetadata{}
1614-
err = json.Unmarshal([]byte(specgen.Config.Annotations[annotations.Metadata]), &podMetadata)
1615-
if err != nil {
1616-
return err
1617-
}
1618-
uid := podMetadata.Uid
1619-
mData := fmt.Sprintf("k8s_%s_%s_%s_%s0", ctrName, sbSpec.Annotations[annotations.KubeName], sbSpec.Annotations[annotations.Namespace], uid)
1620-
specgen.AddAnnotation(annotations.Name, mData)
1621-
1622-
c.SetSandbox(newSbID)
1623-
1624-
saveOptions := generate.ExportOptions{}
1625-
if err := specgen.SaveToFile(configFile, saveOptions); err != nil {
1626-
return err
1627-
}
1628-
}
1629-
16301556
c.state.InitPid = 0
16311557
c.state.InitStartTime = ""
16321558

@@ -1639,7 +1565,7 @@ func (r *runtimeOCI) RestoreContainer(ctx context.Context, c *Container, sbSpec
16391565
if err := crutils.CRCreateFileWithLabel(
16401566
c.BundlePath(),
16411567
metadata.RestoreLogFile,
1642-
specgen.Config.Linux.MountLabel,
1568+
mountLabel,
16431569
); err != nil {
16441570
return err
16451571
}

‎internal/oci/runtime_pod.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -159,11 +159,10 @@ func (r *runtimePod) CheckpointContainer(
159159
func (r *runtimePod) RestoreContainer(
160160
ctx context.Context,
161161
c *Container,
162-
sbSpec *rspec.Spec,
163-
infraPid int,
164162
cgroupParent string,
163+
mountLabel string,
165164
) error {
166-
return r.oci.RestoreContainer(ctx, c, sbSpec, infraPid, cgroupParent)
165+
return r.oci.RestoreContainer(ctx, c, cgroupParent, mountLabel)
167166
}
168167

169168
func (r *runtimePod) ExecContainer(ctx context.Context, c *Container, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resizeChan <-chan remotecommand.TerminalSize) error {

‎internal/oci/runtime_vm.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1051,7 +1051,7 @@ func (r *runtimeVM) CheckpointContainer(ctx context.Context, c *Container, specg
10511051
}
10521052

10531053
// RestoreContainer not implemented for runtimeVM
1054-
func (r *runtimeVM) RestoreContainer(ctx context.Context, c *Container, sbSpec *rspec.Spec, infraPid int, cgroupParent string) error {
1054+
func (r *runtimeVM) RestoreContainer(ctx context.Context, c *Container, cgroupParent, mountLabel string) error {
10551055
log.Debugf(ctx, "RuntimeVM.RestoreContainer() start")
10561056
defer log.Debugf(ctx, "RuntimeVM.RestoreContainer() end")
10571057

‎internal/oci/suite_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,12 @@ const (
3333
containerID = "containerID"
3434
)
3535

36-
func beforeEach(sbID string) {
36+
func beforeEach() {
3737
var err error
3838
myContainer, err = oci.NewContainer(containerID, "", "", "",
3939
make(map[string]string), make(map[string]string),
4040
make(map[string]string), "", "", "",
41-
&types.ContainerMetadata{}, sbID, false,
41+
&types.ContainerMetadata{}, sandboxID, false,
4242
false, false, "", "", time.Now(), "")
4343
Expect(err).To(BeNil())
4444
}

0 commit comments

Comments
 (0)