Merge pull request #9331 from bitoku/release-1.29

openshift-merge-bot[bot] · web-flow · commit 6bda4d5c874a · 2025-07-21T11:42:29.000Z
OCPBUGS-58509: [release-1.29] fix deadlock when the container is in uninterruptible sleep
diff --git a/internal/oci/container.go b/internal/oci/container.go
@@ -68,6 +68,7 @@ type Container struct {
 	stopLock              sync.Mutex
 	stopTimeoutChan       chan int64
 	stopWatchers          []chan struct{}
+	stopKillLoopBegun     bool
 	pidns                 nsmgr.Namespace
 	restore               bool
 	restoreArchivePath    string
@@ -145,22 +146,23 @@ func NewContainer(id, name, bundlePath, logPath string, labels, crioAnnotations,
 			},
 			ImageRef: externalImageRef,
 		},
-		name:            name,
-		bundlePath:      bundlePath,
-		logPath:         logPath,
-		terminal:        terminal,
-		stdin:           stdin,
-		stdinOnce:       stdinOnce,
-		runtimeHandler:  runtimeHandler,
-		crioAnnotations: crioAnnotations,
-		imageName:       imageName,
-		imageID:         imageID,
-		dir:             dir,
-		state:           state,
-		stopSignal:      stopSignal,
-		stopTimeoutChan: make(chan int64, 10),
-		stopWatchers:    []chan struct{}{},
-		execPIDs:        map[int]bool{},
+		name:              name,
+		bundlePath:        bundlePath,
+		logPath:           logPath,
+		terminal:          terminal,
+		stdin:             stdin,
+		stdinOnce:         stdinOnce,
+		runtimeHandler:    runtimeHandler,
+		crioAnnotations:   crioAnnotations,
+		imageName:         imageName,
+		imageID:           imageID,
+		dir:               dir,
+		state:             state,
+		stopSignal:        stopSignal,
+		stopTimeoutChan:   make(chan int64, 10),
+		stopWatchers:      []chan struct{}{},
+		stopKillLoopBegun: false,
+		execPIDs:          map[int]bool{},
 	}
 	return c, nil
 }
@@ -600,14 +602,30 @@ func (c *Container) SetAsStopping() (setToStopping bool) {
 	return false
 }
 
+// SetStopKillLoopBegun sets the stopKillLoopBegun flag to true.
+func (c *Container) SetStopKillLoopBegun() {
+	c.stopLock.Lock()
+	defer c.stopLock.Unlock()
+	c.stopKillLoopBegun = true
+}
+
 func (c *Container) WaitOnStopTimeout(ctx context.Context, timeout int64) {
 	c.stopLock.Lock()
 	if !c.stopping {
 		c.stopLock.Unlock()
 		return
 	}
 
-	c.stopTimeoutChan <- timeout
+	// Don't use the stopTimeoutChan when the container is in kill loop
+	// because values in the channel are no longer consumed.
+	if !c.stopKillLoopBegun {
+		// Use select and default not to block when the stopTimeoutChan is full.
+		// The channel is very unlikely to be full, but it could happen in theory.
+		select {
+		case c.stopTimeoutChan <- timeout:
+		default:
+		}
+	}
 
 	watcher := make(chan struct{}, 1)
 	c.stopWatchers = append(c.stopWatchers, watcher)
diff --git a/internal/oci/runtime_oci.go b/internal/oci/runtime_oci.go
@@ -948,6 +948,7 @@ func (r *runtimeOCI) StopLoopForContainer(c *Container, bm kwait.BackoffManager)
 
 		case <-time.After(time.Until(targetTime)):
 			log.Warnf(ctx, "Stopping container %s with stop signal timed out. Killing...", c.ID())
+			c.SetStopKillLoopBegun()
 			goto killContainer
 
 		case <-done:
@@ -968,9 +969,11 @@ killContainer:
 		}
 
 		if err := c.Living(); err != nil {
+			log.Debugf(ctx, "Container is no longer alive")
 			stop()
 			return
 		}
+		log.Debugf(ctx, "Killing failed for some reasons, retrying...")
 		// Reschedule the timer so that the periodic reminder can continue.
 		blockedTimer.Reset(stopProcessBlockedInterval)
 	}, bm, true, ctx.Done())
