fix deadlock when the container is in uninterruptible sleep

bitoku · bitoku · commit 103e03f8abcc · 2025-07-08T12:05:38.000Z
Signed-off-by: Ayato Tokubi <atokubi@redhat.com> (cherry picked from commit 1e751b4) Signed-off-by: Ayato Tokubi <atokubi@redhat.com> # Conflicts: # internal/oci/container.go # internal/oci/runtime_oci.go
diff --git a/internal/oci/container.go b/internal/oci/container.go
@@ -69,6 +69,7 @@ type Container struct {
 	stopLock              sync.Mutex
 	stopTimeoutChan       chan int64
 	stopWatchers          []chan struct{}
+	stopKillLoopBegun     bool
 	pidns                 nsmgr.Namespace
 	restore               bool
 	restoreArchivePath    string
@@ -155,22 +156,23 @@ func NewContainer(id, name, bundlePath, logPath string, labels, crioAnnotations,
 			ImageRef: externalImageRef,
 			ImageId:  imageIDString,
 		},
-		name:            name,
-		bundlePath:      bundlePath,
-		logPath:         logPath,
-		terminal:        terminal,
-		stdin:           stdin,
-		stdinOnce:       stdinOnce,
-		runtimeHandler:  runtimeHandler,
-		crioAnnotations: crioAnnotations,
-		imageName:       imageName,
-		imageID:         imageID,
-		dir:             dir,
-		state:           state,
-		stopSignal:      stopSignal,
-		stopTimeoutChan: make(chan int64, 10),
-		stopWatchers:    []chan struct{}{},
-		execPIDs:        map[int]bool{},
+		name:              name,
+		bundlePath:        bundlePath,
+		logPath:           logPath,
+		terminal:          terminal,
+		stdin:             stdin,
+		stdinOnce:         stdinOnce,
+		runtimeHandler:    runtimeHandler,
+		crioAnnotations:   crioAnnotations,
+		imageName:         imageName,
+		imageID:           imageID,
+		dir:               dir,
+		state:             state,
+		stopSignal:        stopSignal,
+		stopTimeoutChan:   make(chan int64, 10),
+		stopWatchers:      []chan struct{}{},
+		stopKillLoopBegun: false,
+		execPIDs:          map[int]bool{},
 	}
 	return c, nil
 }
@@ -611,14 +613,30 @@ func (c *Container) SetAsStopping() (setToStopping bool) {
 	return false
 }
 
+// SetStopKillLoopBegun sets the stopKillLoopBegun flag to true.
+func (c *Container) SetStopKillLoopBegun() {
+	c.stopLock.Lock()
+	defer c.stopLock.Unlock()
+	c.stopKillLoopBegun = true
+}
+
 func (c *Container) WaitOnStopTimeout(ctx context.Context, timeout int64) {
 	c.stopLock.Lock()
 	if !c.stopping {
 		c.stopLock.Unlock()
 		return
 	}
 
-	c.stopTimeoutChan <- timeout
+	// Don't use the stopTimeoutChan when the container is in kill loop
+	// because values in the channel are no longer consumed.
+	if !c.stopKillLoopBegun {
+		// Use select and default not to block when the stopTimeoutChan is full.
+		// The channel is very unlikely to be full, but it could happen in theory.
+		select {
+		case c.stopTimeoutChan <- timeout:
+		default:
+		}
+	}
 
 	watcher := make(chan struct{}, 1)
 	c.stopWatchers = append(c.stopWatchers, watcher)
diff --git a/internal/oci/runtime_oci.go b/internal/oci/runtime_oci.go
@@ -949,7 +949,9 @@ func (r *runtimeOCI) StopLoopForContainer(c *Container, bm kwait.BackoffManager)
 			}
 
 		case <-time.After(time.Until(targetTime)):
-			log.Warnf(ctx, "Stopping container %s with stop signal timed out. Killing...", c.ID())
+			log.Warnf(ctx, "Stopping container %s with stop signal(%s) timed out. Killing...", c.ID(), c.GetStopSignal())
+			c.SetStopKillLoopBegun()
+
 			goto killContainer
 
 		case <-done:
@@ -971,9 +973,11 @@ killContainer:
 		}
 
 		if err := c.Living(); err != nil {
+			log.Debugf(ctx, "Container is no longer alive")
 			stop()
 			return
 		}
+		log.Debugf(ctx, "Killing failed for some reasons, retrying...")
 		// Reschedule the timer so that the periodic reminder can continue.
 		blockedTimer.Reset(stopProcessBlockedInterval)
 	}, bm, true, ctx.Done())

Original file line number	Diff line number	Diff line change
`@@ -949,7 +949,9 @@ func (r runtimeOCI) StopLoopForContainer(c Container, bm kwait.BackoffManager)`
`949`	`949`	`}`
`950`	`950`
`951`	`951`	`case <-time.After(time.Until(targetTime)):`
`952`		`- log.Warnf(ctx, "Stopping container %s with stop signal timed out. Killing...", c.ID())`
	`952`	`+ log.Warnf(ctx, "Stopping container %s with stop signal(%s) timed out. Killing...", c.ID(), c.GetStopSignal())`
	`953`	`+ c.SetStopKillLoopBegun()`
	`954`	`+`
`953`	`955`	`goto killContainer`
`954`	`956`
`955`	`957`	`case <-done:`
`@@ -971,9 +973,11 @@ killContainer:`
`971`	`973`	`}`
`972`	`974`
`973`	`975`	`if err := c.Living(); err != nil {`
	`976`	`+ log.Debugf(ctx, "Container is no longer alive")`
`974`	`977`	`stop()`
`975`	`978`	`return`
`976`	`979`	`}`
	`980`	`+ log.Debugf(ctx, "Killing failed for some reasons, retrying...")`
`977`	`981`	`// Reschedule the timer so that the periodic reminder can continue.`
`978`	`982`	`blockedTimer.Reset(stopProcessBlockedInterval)`
`979`	`983`	`}, bm, true, ctx.Done())`