feat: retry on internal error

cmd/pew.go

@@ -87,6 +87,7 @@ type PewCmd struct {
 
 	Threshold             float64 `help:"reliablity threshold for exit code" default:"1.00"`
 	IncludeInternalErrors bool    `help:"count internal errors as part of the success rate"`
+	InternalErrorsRetries int     `help:"amount of retries on internal errors" default:"3"`
 
 	OutputOnSuccess bool `help:"show output on success"`
 	RealtimeOutput  bool `help:"show realtime output"`
@@ -466,12 +467,33 @@ func (cmd PewCmd) testArtifact(swg *sizedwaitgroup.SizedWaitGroup,
 		Str("kernel", ki.KernelRelease).
 		Logger()
 
+	retriesLeft := cmd.InternalErrorsRetries
+	var stop bool
+	for !stop {
 		ka.Process(slog, ki, cmd.OutputOnSuccess, cmd.RealtimeOutput,
 			cmd.Endless, cmd.Binary, cmd.EndlessStress, cmd.EndlessTimeout,
-		func(q *qemu.System, ka artifact.Artifact, ki distro.KernelInfo, result *artifact.Result) {
+			func(q *qemu.System, ka artifact.Artifact, ki distro.KernelInfo, res *artifact.Result) {
-			dumpResult(q, ka, ki, result, cmd.Dist, cmd.Tag, cmd.Binary, cmd.DB)
+				if res.InternalError == nil {
+					cmd.dumpResult(q, ka, ki, res)
+					stop = true
+					return
+				}
+
+				q.Log.Warn().Err(res.InternalError).
+					Str("panic", fmt.Sprintf("%v", q.KernelPanic)).
+					Str("timeout", fmt.Sprintf("%v", q.KilledByTimeout)).
+					Int("retries_left", retriesLeft).
+					Msg("internal")
+
+				if retriesLeft == 0 {
+					state.InternalErrors += 1
+					stop = true
+					return
+				}
+				retriesLeft -= 1
 			},
 		)
+	}
 }
 
 func shuffleKernels(a []distro.KernelInfo) []distro.KernelInfo {
@@ -567,27 +589,14 @@ func genOkFail(name string, ok bool) (aurv aurora.Value) {
 	return
 }
 
-func dumpResult(q *qemu.System, ka artifact.Artifact, ki distro.KernelInfo,
+func (cmd PewCmd) dumpResult(q *qemu.System, ka artifact.Artifact, ki distro.KernelInfo, res *artifact.Result) {
-	res *artifact.Result, dist, tag, binary string, db *sql.DB) {
-
-	// TODO refactor
-
-	if res.InternalError != nil {
-		q.Log.Warn().Err(res.InternalError).
-			Str("panic", fmt.Sprintf("%v", q.KernelPanic)).
-			Str("timeout", fmt.Sprintf("%v", q.KilledByTimeout)).
-			Msg("internal")
-		res.InternalErrorString = res.InternalError.Error()
-		state.InternalErrors += 1
-	} else {
-		colored := ""
-
 	state.Overall += 1
 
 	if res.Test.Ok {
 		state.Success += 1
 	}
 
+	colored := ""
 	switch ka.Type {
 	case artifact.KernelExploit:
 		colored = aurora.Sprintf("%s %s",
@@ -615,20 +624,27 @@ func dumpResult(q *qemu.System, ka artifact.Artifact, ki distro.KernelInfo,
 	} else {
 		q.Log.Info().Msgf("%v", colored)
 	}
-	}
 
-	err := addToLog(db, q, ka, ki, res, tag)
+	err := addToLog(cmd.DB, q, ka, ki, res, cmd.Tag)
 	if err != nil {
-		q.Log.Warn().Err(err).Msgf("[db] addToLog (%v)", ka)
+		q.Log.Error().Err(err).Msgf("[db] addToLog (%v)", ka)
 	}
 
-	if binary == "" && dist != pathDevNull {
+	if cmd.Binary != "" {
-		err = os.MkdirAll(dist, os.ModePerm)
+		return
+	}
+
+	if cmd.Dist == pathDevNull { // why?
+		return
+	}
+
+	err = os.MkdirAll(cmd.Dist, os.ModePerm)
 	if err != nil {
-			log.Warn().Err(err).Msgf("os.MkdirAll (%v)", ka)
+		log.Error().Err(err).Msgf("os.MkdirAll (%v)", ka)
+		return
 	}
 
-		path := fmt.Sprintf("%s/%s-%s-%s", dist, ki.Distro.ID,
+	path := fmt.Sprintf("%s/%s-%s-%s", cmd.Dist, ki.Distro.ID,
 		ki.Distro.Release, ki.KernelRelease)
 	if ka.Type != artifact.KernelExploit {
 		path += ".ko"
@@ -636,7 +652,7 @@ func dumpResult(q *qemu.System, ka artifact.Artifact, ki distro.KernelInfo,
 
 	err = artifact.CopyFile(res.BuildArtifact, path)
 	if err != nil {
-			log.Warn().Err(err).Msgf("copy file (%v)", ka)
+		log.Error().Err(err).Msgf("copy file (%v)", ka)
-		}
+		return
 	}
 }