From 3fd2fd59668c9d3916929d74f58f8084ef37b5ff Mon Sep 17 00:00:00 2001
From: Mikhail Klementev <blame@dumpstack.io>
Date: Wed, 5 Apr 2023 16:24:45 +0000
Subject: [PATCH] Implements retry for failed kernel installations

---
 kernel.go | 38 +++++++++++++++++++++++---------------
 1 file changed, 23 insertions(+), 15 deletions(-)

diff --git a/kernel.go b/kernel.go
index bfc03e5..efdead8 100644
--- a/kernel.go
+++ b/kernel.go
@@ -25,11 +25,12 @@ import (
 )
 
 type KernelCmd struct {
-	NoDownload bool `help:"do not download qemu image while kernel generation"`
-	UseHost    bool `help:"also use host kernels"`
-	Force      bool `help:"force reinstall kernel"`
-	NoHeaders  bool `help:"do not install kernel headers"`
-	Shuffle    bool `help:"randomize kernels installation order"`
+	NoDownload bool  `help:"do not download qemu image while kernel generation"`
+	UseHost    bool  `help:"also use host kernels"`
+	Force      bool  `help:"force reinstall kernel"`
+	NoHeaders  bool  `help:"do not install kernel headers"`
+	Shuffle    bool  `help:"randomize kernels installation order"`
+	Retries    int64 `help:"amount of tries for each kernel" default="10"`
 
 	List        KernelListCmd        `cmd:"" help:"list kernels"`
 	Autogen     KernelAutogenCmd     `cmd:"" help:"generate kernels based on the current config"`
@@ -58,7 +59,7 @@ func (cmd *KernelListCmd) Run(g *Globals) (err error) {
 }
 
 type KernelAutogenCmd struct {
-	Max int64 `help:"download random kernels from set defined by regex in release_mask, but no more than X for each of release_mask" default:"100"`
+	Max int64 `help:"download kernels from set defined by regex in release_mask, but no more than X for each of release_mask" default:"100500"`
 }
 
 func (cmd KernelAutogenCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
@@ -76,7 +77,7 @@ func (cmd KernelAutogenCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
 		err = generateKernels(sk,
 			g.Config.Docker.Registry,
 			g.Config.Docker.Commands,
-			cmd.Max,
+			cmd.Max, kernelCmd.Retries,
 			!kernelCmd.NoDownload,
 			kernelCmd.Force,
 			!kernelCmd.NoHeaders,
@@ -109,7 +110,7 @@ func (cmd *KernelGenallCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
 	err = generateKernels(km,
 		g.Config.Docker.Registry,
 		g.Config.Docker.Commands,
-		math.MaxUint32,
+		math.MaxUint32, kernelCmd.Retries,
 		!kernelCmd.NoDownload,
 		kernelCmd.Force,
 		!kernelCmd.NoHeaders,
@@ -142,7 +143,7 @@ func (cmd *KernelInstallCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
 	err = generateKernels(km,
 		g.Config.Docker.Registry,
 		g.Config.Docker.Commands,
-		math.MaxUint32,
+		math.MaxUint32, kernelCmd.Retries,
 		!kernelCmd.NoDownload,
 		kernelCmd.Force,
 		!kernelCmd.NoHeaders,
@@ -718,7 +719,7 @@ func shuffleStrings(a []string) []string {
 }
 
 func generateKernels(km config.KernelMask, registry string,
-	commands []config.DockerCommand, max int64,
+	commands []config.DockerCommand, max, retries int64,
 	download, force, headers, shuffle bool) (err error) {
 
 	log.Info().Msgf("Generating for kernel mask %v", km)
@@ -759,11 +760,18 @@ func generateKernels(km config.KernelMask, registry string,
 
 		log.Info().Msgf("%d/%d %s", i+1, len(pkgs), pkg)
 
-		err = installKernel(km, pkg, force, headers)
-		if err == nil {
-			max--
-		} else {
-			log.Fatal().Err(err).Msg("install kernel")
+		var attempt int64
+		for {
+			attempt++
+
+			err = installKernel(km, pkg, force, headers)
+			if err == nil {
+				max--
+			} else if attempt >= retries {
+				log.Fatal().Err(err).Msg("install kernel")
+			} else {
+				log.Warn().Err(err).Msg("install kernel")
+			}
 		}
 	}