2023-03-22 17:45:56 +00:00
|
|
|
|
// Copyright 2023 Mikhail Klementev. All rights reserved.
|
2018-11-17 19:37:04 +00:00
|
|
|
|
// Use of this source code is governed by a AGPLv3 license
|
|
|
|
|
// (or later) that can be found in the LICENSE file.
|
|
|
|
|
|
2024-02-17 22:38:43 +00:00
|
|
|
|
package cmd
|
2018-11-17 19:37:04 +00:00
|
|
|
|
|
|
|
|
|
import (
|
2019-08-13 21:54:59 +00:00
|
|
|
|
"database/sql"
|
2018-11-17 19:37:04 +00:00
|
|
|
|
"errors"
|
|
|
|
|
"fmt"
|
2019-08-14 17:36:36 +00:00
|
|
|
|
"io"
|
2018-11-17 19:37:04 +00:00
|
|
|
|
"math/rand"
|
|
|
|
|
"os"
|
|
|
|
|
"os/exec"
|
|
|
|
|
"strings"
|
|
|
|
|
"time"
|
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
"github.com/davecgh/go-spew/spew"
|
2024-02-26 08:55:27 +00:00
|
|
|
|
"github.com/google/uuid"
|
2018-11-17 19:37:04 +00:00
|
|
|
|
"github.com/remeh/sizedwaitgroup"
|
2023-03-22 18:32:40 +00:00
|
|
|
|
"github.com/rs/zerolog"
|
2023-03-18 21:30:07 +00:00
|
|
|
|
"github.com/rs/zerolog/log"
|
2020-05-17 15:40:34 +00:00
|
|
|
|
"gopkg.in/logrusorgru/aurora.v2"
|
2018-11-17 19:37:04 +00:00
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
"code.dumpstack.io/tools/out-of-tree/api"
|
|
|
|
|
"code.dumpstack.io/tools/out-of-tree/artifact"
|
|
|
|
|
"code.dumpstack.io/tools/out-of-tree/client"
|
2019-02-02 21:24:29 +00:00
|
|
|
|
"code.dumpstack.io/tools/out-of-tree/config"
|
2023-05-18 16:07:24 +00:00
|
|
|
|
"code.dumpstack.io/tools/out-of-tree/distro"
|
2019-02-02 21:24:29 +00:00
|
|
|
|
"code.dumpstack.io/tools/out-of-tree/qemu"
|
2018-11-17 19:37:04 +00:00
|
|
|
|
)
|
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
const pathDevNull = "/dev/null"
|
|
|
|
|
|
2024-02-17 22:38:43 +00:00
|
|
|
|
type LevelWriter struct {
|
|
|
|
|
io.Writer
|
|
|
|
|
Level zerolog.Level
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (lw *LevelWriter) WriteLevel(l zerolog.Level, p []byte) (n int, err error) {
|
|
|
|
|
if l >= lw.Level {
|
|
|
|
|
return lw.Writer.Write(p)
|
|
|
|
|
}
|
|
|
|
|
return len(p), nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var ConsoleWriter, FileWriter LevelWriter
|
|
|
|
|
|
|
|
|
|
var LogLevel zerolog.Level
|
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
type runstate struct {
|
|
|
|
|
Overall, Success float64
|
|
|
|
|
InternalErrors int
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var (
|
|
|
|
|
state runstate
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
func successRate(state runstate) float64 {
|
|
|
|
|
return state.Success / state.Overall
|
|
|
|
|
}
|
|
|
|
|
|
2023-01-31 07:13:33 +00:00
|
|
|
|
type PewCmd struct {
|
|
|
|
|
Max int64 `help:"test no more than X kernels" default:"100500"`
|
|
|
|
|
Runs int64 `help:"runs per each kernel" default:"1"`
|
2023-05-08 22:21:28 +00:00
|
|
|
|
RootFS string `help:"override rootfs image" type:"existingfile"`
|
2023-01-31 07:13:33 +00:00
|
|
|
|
Guess bool `help:"try all defined kernels"`
|
2023-04-05 11:29:31 +00:00
|
|
|
|
Shuffle bool `help:"randomize kernels test order"`
|
2023-01-31 07:13:33 +00:00
|
|
|
|
Binary string `help:"use binary, do not build"`
|
|
|
|
|
Test string `help:"override path for test"`
|
|
|
|
|
Dist string `help:"build result path" default:"/dev/null"`
|
2023-01-31 09:05:43 +00:00
|
|
|
|
Threads int `help:"threads" default:"1"`
|
2023-01-31 07:13:33 +00:00
|
|
|
|
Tag string `help:"log tagging"`
|
|
|
|
|
Timeout time.Duration `help:"timeout after tool will not spawn new tests"`
|
2023-01-31 09:05:43 +00:00
|
|
|
|
|
2024-10-05 23:26:39 +00:00
|
|
|
|
KernelRegex string `help:"set kernel regex"`
|
|
|
|
|
DistroID string `help:"set distribution"`
|
|
|
|
|
DistroRelease string `help:"set distribution release"`
|
|
|
|
|
|
2023-02-01 07:37:08 +00:00
|
|
|
|
ArtifactConfig string `help:"path to artifact config" type:"path"`
|
|
|
|
|
|
2023-05-08 19:56:10 +00:00
|
|
|
|
QemuTimeout time.Duration `help:"timeout for qemu"`
|
|
|
|
|
QemuAfterStartTimeout time.Duration `help:"timeout after starting of the qemu vm before tests"`
|
|
|
|
|
DockerTimeout time.Duration `help:"timeout for docker"`
|
2023-01-31 09:34:12 +00:00
|
|
|
|
|
2023-05-31 18:21:30 +00:00
|
|
|
|
Threshold float64 `help:"reliablity threshold for exit code" default:"1.00"`
|
|
|
|
|
IncludeInternalErrors bool `help:"count internal errors as part of the success rate"`
|
2023-04-05 11:29:31 +00:00
|
|
|
|
|
2023-05-08 21:19:06 +00:00
|
|
|
|
Endless bool `help:"endless tests"`
|
|
|
|
|
EndlessTimeout time.Duration `help:"timeout between tests" default:"1m"`
|
2023-05-08 22:01:10 +00:00
|
|
|
|
EndlessStress string `help:"endless stress script" type:"existingfile"`
|
2023-05-08 21:19:06 +00:00
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
DB *sql.DB `kong:"-" json:"-"`
|
|
|
|
|
Kcfg config.KernelConfig `kong:"-" json:"-"`
|
|
|
|
|
TimeoutDeadline time.Time `kong:"-" json:"-"`
|
|
|
|
|
|
2024-02-25 18:02:03 +00:00
|
|
|
|
Watch bool `help:"watch job status"`
|
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
repoName string
|
|
|
|
|
commit string
|
|
|
|
|
|
|
|
|
|
useRemote bool
|
|
|
|
|
remoteAddr string
|
2024-02-26 08:55:27 +00:00
|
|
|
|
|
|
|
|
|
// UUID of the job set
|
|
|
|
|
groupUUID string
|
2023-01-31 07:13:33 +00:00
|
|
|
|
}
|
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
func (cmd *PewCmd) getRepoName(worktree string, ka artifact.Artifact) {
|
|
|
|
|
raw, err := exec.Command("git", "--work-tree="+worktree,
|
|
|
|
|
"rev-list", "--max-parents=0", "HEAD").CombinedOutput()
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Error().Err(err).Msg(string(raw))
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
cmd.repoName = fmt.Sprintf("%s-%s", ka.Name, string(raw[:7]))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (cmd *PewCmd) syncRepo(worktree string, ka artifact.Artifact) (err error) {
|
|
|
|
|
c := client.Client{RemoteAddr: cmd.remoteAddr}
|
|
|
|
|
|
|
|
|
|
cmd.getRepoName(worktree, ka)
|
|
|
|
|
|
|
|
|
|
raw, err := exec.Command("git", "--work-tree="+worktree,
|
|
|
|
|
"rev-parse", "HEAD").CombinedOutput()
|
|
|
|
|
if err != nil {
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
cmd.commit = strings.TrimSuffix(string(raw), "\n")
|
|
|
|
|
|
|
|
|
|
_, err = c.GetRepo(cmd.repoName)
|
|
|
|
|
if err != nil && err != client.ErrRepoNotFound {
|
|
|
|
|
log.Error().Err(err).Msg("GetRepo API error")
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if err == client.ErrRepoNotFound {
|
|
|
|
|
log.Warn().Msg("repo not found")
|
|
|
|
|
log.Info().Msg("add repo")
|
|
|
|
|
log.Warn().Msgf("%v", spew.Sdump(ka))
|
|
|
|
|
err = c.AddRepo(api.Repo{Name: cmd.repoName})
|
|
|
|
|
if err != nil {
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
err = c.PushRepo(api.Repo{Name: cmd.repoName, Path: worktree})
|
2023-01-31 07:13:33 +00:00
|
|
|
|
if err != nil {
|
2024-02-20 13:25:31 +00:00
|
|
|
|
log.Error().Err(err).Msg("push repo error")
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (cmd *PewCmd) Run(g *Globals) (err error) {
|
2024-02-26 08:55:27 +00:00
|
|
|
|
cmd.groupUUID = uuid.New().String()
|
|
|
|
|
log.Info().Str("group", cmd.groupUUID).Msg("")
|
2024-02-20 13:25:31 +00:00
|
|
|
|
cmd.useRemote = g.Remote
|
|
|
|
|
cmd.remoteAddr = g.RemoteAddr
|
|
|
|
|
|
|
|
|
|
if cmd.useRemote {
|
|
|
|
|
c := client.Client{RemoteAddr: cmd.remoteAddr}
|
|
|
|
|
cmd.Kcfg.Kernels, err = c.Kernels()
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Fatal().Err(err).Msg("read kernels config")
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
cmd.Kcfg, err = config.ReadKernelConfig(
|
|
|
|
|
g.Config.Kernels)
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Fatal().Err(err).Msg("read kernels config")
|
|
|
|
|
}
|
2023-01-31 07:13:33 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if cmd.Timeout != 0 {
|
2023-03-18 22:34:30 +00:00
|
|
|
|
log.Info().Msgf("Set global timeout to %s", cmd.Timeout)
|
2024-02-20 13:25:31 +00:00
|
|
|
|
cmd.TimeoutDeadline = time.Now().Add(cmd.Timeout)
|
2023-01-31 07:13:33 +00:00
|
|
|
|
}
|
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
cmd.DB, err = openDatabase(g.Config.Database)
|
2023-01-31 07:13:33 +00:00
|
|
|
|
if err != nil {
|
2023-03-18 22:34:30 +00:00
|
|
|
|
log.Fatal().Err(err).
|
|
|
|
|
Msgf("Cannot open database %s", g.Config.Database)
|
2023-01-31 07:13:33 +00:00
|
|
|
|
}
|
2024-02-20 13:25:31 +00:00
|
|
|
|
defer cmd.DB.Close()
|
2023-01-31 07:13:33 +00:00
|
|
|
|
|
2023-02-01 07:37:08 +00:00
|
|
|
|
var configPath string
|
|
|
|
|
if cmd.ArtifactConfig == "" {
|
|
|
|
|
configPath = g.WorkDir + "/.out-of-tree.toml"
|
|
|
|
|
} else {
|
|
|
|
|
configPath = cmd.ArtifactConfig
|
|
|
|
|
}
|
2024-02-20 13:25:31 +00:00
|
|
|
|
|
|
|
|
|
ka, err := artifact.Artifact{}.Read(configPath)
|
2023-01-31 09:05:43 +00:00
|
|
|
|
if err != nil {
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
if cmd.useRemote {
|
|
|
|
|
err = cmd.syncRepo(g.WorkDir, ka)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2023-05-24 15:46:58 +00:00
|
|
|
|
if len(ka.Targets) == 0 || cmd.Guess {
|
|
|
|
|
log.Debug().Msg("will use all available targets")
|
2023-05-21 14:40:24 +00:00
|
|
|
|
|
|
|
|
|
for _, dist := range distro.List() {
|
2024-02-20 13:25:31 +00:00
|
|
|
|
ka.Targets = append(ka.Targets, artifact.Target{
|
2023-05-21 14:40:24 +00:00
|
|
|
|
Distro: dist,
|
2024-02-20 13:25:31 +00:00
|
|
|
|
Kernel: artifact.Kernel{
|
2023-05-21 14:40:24 +00:00
|
|
|
|
Regex: ".*",
|
|
|
|
|
},
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2023-01-31 09:05:43 +00:00
|
|
|
|
if ka.SourcePath == "" {
|
|
|
|
|
ka.SourcePath = g.WorkDir
|
|
|
|
|
}
|
|
|
|
|
|
2024-10-05 23:26:39 +00:00
|
|
|
|
if cmd.KernelRegex != "" {
|
2024-02-20 13:25:31 +00:00
|
|
|
|
var km artifact.Target
|
2024-10-05 23:26:39 +00:00
|
|
|
|
km.Kernel.Regex = cmd.KernelRegex
|
|
|
|
|
|
|
|
|
|
if cmd.DistroID == "" {
|
|
|
|
|
err = errors.New("--distro-id is required")
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var dt distro.ID
|
|
|
|
|
dt, err = distro.NewID(cmd.DistroID)
|
2023-01-31 09:05:43 +00:00
|
|
|
|
if err != nil {
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
2024-10-05 23:26:39 +00:00
|
|
|
|
km.Distro.ID = dt
|
|
|
|
|
|
|
|
|
|
if cmd.DistroRelease != "" {
|
|
|
|
|
km.Distro.Release = cmd.DistroRelease
|
|
|
|
|
}
|
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
ka.Targets = []artifact.Target{km}
|
2024-10-05 23:26:39 +00:00
|
|
|
|
} else if cmd.DistroID != "" {
|
|
|
|
|
var km artifact.Target
|
|
|
|
|
|
|
|
|
|
var dt distro.ID
|
|
|
|
|
dt, err = distro.NewID(cmd.DistroID)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
km.Distro.ID = dt
|
|
|
|
|
|
|
|
|
|
if cmd.DistroRelease != "" {
|
|
|
|
|
km.Distro.Release = cmd.DistroRelease
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ka.Targets = []artifact.Target{km}
|
|
|
|
|
} else if cmd.DistroRelease != "" {
|
|
|
|
|
err = errors.New("--distro-release has no use on its own")
|
|
|
|
|
return
|
2023-01-31 09:05:43 +00:00
|
|
|
|
}
|
|
|
|
|
|
2024-10-05 20:28:48 +00:00
|
|
|
|
if ka.Qemu.Timeout.Duration == 0 {
|
|
|
|
|
ka.Qemu.Timeout.Duration = g.Config.Qemu.Timeout.Duration
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ka.Docker.Timeout.Duration == 0 {
|
|
|
|
|
ka.Docker.Timeout.Duration = g.Config.Docker.Timeout.Duration
|
|
|
|
|
}
|
2024-02-20 13:25:31 +00:00
|
|
|
|
|
2023-01-31 09:05:43 +00:00
|
|
|
|
if cmd.QemuTimeout != 0 {
|
2024-02-20 23:00:52 +00:00
|
|
|
|
ka.Qemu.Timeout.Duration = cmd.QemuTimeout
|
2023-01-31 09:05:43 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if cmd.DockerTimeout != 0 {
|
2024-02-20 23:00:52 +00:00
|
|
|
|
ka.Docker.Timeout.Duration = cmd.DockerTimeout
|
2023-01-31 09:05:43 +00:00
|
|
|
|
}
|
|
|
|
|
|
2024-10-05 20:36:17 +00:00
|
|
|
|
log.Info().Msgf("Qemu timeout: %s", ka.Qemu.Timeout.Duration)
|
|
|
|
|
log.Info().Msgf("Docker timeout: %s", ka.Docker.Timeout.Duration)
|
|
|
|
|
|
2023-03-16 09:41:49 +00:00
|
|
|
|
if cmd.Tag == "" {
|
|
|
|
|
cmd.Tag = fmt.Sprintf("%d", time.Now().Unix())
|
|
|
|
|
}
|
2024-02-20 13:25:31 +00:00
|
|
|
|
if !cmd.useRemote {
|
|
|
|
|
log.Info().Str("tag", cmd.Tag).Msg("")
|
|
|
|
|
}
|
2023-03-16 09:41:49 +00:00
|
|
|
|
|
2023-04-05 11:29:31 +00:00
|
|
|
|
err = cmd.performCI(ka)
|
2023-01-31 09:05:43 +00:00
|
|
|
|
if err != nil {
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
if cmd.useRemote {
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
2023-05-31 16:14:08 +00:00
|
|
|
|
if state.InternalErrors > 0 {
|
2023-05-31 18:21:30 +00:00
|
|
|
|
s := "not counted towards success rate"
|
|
|
|
|
if cmd.IncludeInternalErrors {
|
|
|
|
|
s = "included in success rate"
|
|
|
|
|
}
|
2023-05-31 16:14:08 +00:00
|
|
|
|
log.Warn().Msgf("%d internal errors "+
|
2023-05-31 18:21:30 +00:00
|
|
|
|
"(%s)", state.InternalErrors, s)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if cmd.IncludeInternalErrors {
|
|
|
|
|
state.Overall += float64(state.InternalErrors)
|
2023-05-31 16:14:08 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
msg := fmt.Sprintf("Success rate: %.02f (%d/%d), Threshold: %.02f",
|
|
|
|
|
successRate(state),
|
|
|
|
|
int(state.Success), int(state.Overall),
|
|
|
|
|
cmd.Threshold)
|
|
|
|
|
|
2023-01-31 09:56:49 +00:00
|
|
|
|
if successRate(state) < cmd.Threshold {
|
2023-05-31 16:14:08 +00:00
|
|
|
|
log.Error().Msg(msg)
|
2023-01-31 09:34:12 +00:00
|
|
|
|
err = errors.New("reliability threshold not met")
|
2023-05-31 16:14:08 +00:00
|
|
|
|
} else {
|
|
|
|
|
log.Info().Msg(msg)
|
2023-01-31 09:34:12 +00:00
|
|
|
|
}
|
2023-05-31 16:14:08 +00:00
|
|
|
|
|
2023-01-31 09:05:43 +00:00
|
|
|
|
return
|
2023-01-31 07:13:33 +00:00
|
|
|
|
}
|
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
func (cmd PewCmd) watchJob(swg *sizedwaitgroup.SizedWaitGroup,
|
|
|
|
|
slog zerolog.Logger, uuid string) {
|
2018-12-07 03:14:10 +00:00
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
defer swg.Done() // FIXME
|
2019-08-14 17:36:36 +00:00
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
c := client.Client{RemoteAddr: cmd.remoteAddr}
|
2023-03-19 13:14:14 +00:00
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
var err error
|
|
|
|
|
var st api.Status
|
2023-02-16 10:21:44 +00:00
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
for {
|
|
|
|
|
st, err = c.JobStatus(uuid)
|
2019-08-17 10:00:01 +00:00
|
|
|
|
if err != nil {
|
2024-02-20 13:25:31 +00:00
|
|
|
|
slog.Error().Err(err).Msg("")
|
|
|
|
|
continue
|
2019-08-17 10:00:01 +00:00
|
|
|
|
}
|
2024-02-20 13:25:31 +00:00
|
|
|
|
if st == api.StatusSuccess || st == api.StatusFailure {
|
|
|
|
|
break
|
2019-08-17 10:00:01 +00:00
|
|
|
|
}
|
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
time.Sleep(time.Second)
|
2019-08-17 10:00:01 +00:00
|
|
|
|
}
|
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
switch st {
|
|
|
|
|
case api.StatusSuccess:
|
|
|
|
|
slog.Info().Msg("success")
|
|
|
|
|
case api.StatusFailure:
|
|
|
|
|
slog.Warn().Msg("failure")
|
2023-08-24 00:52:21 +00:00
|
|
|
|
}
|
2019-08-17 10:00:01 +00:00
|
|
|
|
}
|
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
func (cmd PewCmd) remote(swg *sizedwaitgroup.SizedWaitGroup,
|
|
|
|
|
ka artifact.Artifact, ki distro.KernelInfo) {
|
2019-08-17 10:00:01 +00:00
|
|
|
|
|
2024-02-25 18:02:03 +00:00
|
|
|
|
defer swg.Done()
|
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
slog := log.With().
|
|
|
|
|
Str("distro_type", ki.Distro.ID.String()).
|
|
|
|
|
Str("distro_release", ki.Distro.Release).
|
|
|
|
|
Str("kernel", ki.KernelRelease).
|
|
|
|
|
Logger()
|
2019-08-17 10:00:01 +00:00
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
job := api.Job{}
|
2024-02-26 08:55:27 +00:00
|
|
|
|
job.Group = cmd.groupUUID
|
2024-02-20 13:25:31 +00:00
|
|
|
|
job.RepoName = cmd.repoName
|
|
|
|
|
job.Commit = cmd.commit
|
2023-05-07 18:14:59 +00:00
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
job.Artifact = ka
|
|
|
|
|
job.Target = ki
|
2023-02-15 11:48:25 +00:00
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
c := client.Client{RemoteAddr: cmd.remoteAddr}
|
|
|
|
|
uuid, err := c.AddJob(job)
|
|
|
|
|
slog = slog.With().Str("uuid", uuid).Logger()
|
2023-03-22 17:45:56 +00:00
|
|
|
|
if err != nil {
|
2024-02-20 13:25:31 +00:00
|
|
|
|
slog.Error().Err(err).Msg("cannot add job")
|
2023-03-22 17:45:56 +00:00
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
slog.Info().Msg("add")
|
2023-03-22 17:45:56 +00:00
|
|
|
|
|
2024-02-25 18:02:03 +00:00
|
|
|
|
if cmd.Watch {
|
|
|
|
|
// FIXME dummy (almost)
|
|
|
|
|
go cmd.watchJob(swg, slog, uuid)
|
|
|
|
|
}
|
2023-02-15 11:48:25 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-04-05 11:29:31 +00:00
|
|
|
|
func (cmd PewCmd) testArtifact(swg *sizedwaitgroup.SizedWaitGroup,
|
2024-02-20 13:25:31 +00:00
|
|
|
|
ka artifact.Artifact, ki distro.KernelInfo) {
|
2018-11-17 19:37:04 +00:00
|
|
|
|
|
2023-03-22 17:45:56 +00:00
|
|
|
|
defer swg.Done()
|
|
|
|
|
|
2023-05-02 11:10:57 +00:00
|
|
|
|
logdir := "logs/" + cmd.Tag
|
|
|
|
|
err := os.MkdirAll(logdir, os.ModePerm)
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Error().Err(err).Msgf("mkdir %s", logdir)
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
logfile := fmt.Sprintf("logs/%s/%s-%s-%s.log",
|
|
|
|
|
cmd.Tag,
|
2023-05-18 16:07:24 +00:00
|
|
|
|
ki.Distro.ID.String(),
|
|
|
|
|
ki.Distro.Release,
|
2023-05-02 11:10:57 +00:00
|
|
|
|
ki.KernelRelease,
|
|
|
|
|
)
|
|
|
|
|
f, err := os.Create(logfile)
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Error().Err(err).Msgf("create %s", logfile)
|
|
|
|
|
return
|
|
|
|
|
}
|
2023-05-07 16:03:28 +00:00
|
|
|
|
defer f.Close()
|
2023-05-02 11:10:57 +00:00
|
|
|
|
|
|
|
|
|
slog := zerolog.New(zerolog.MultiLevelWriter(
|
2024-02-17 22:38:43 +00:00
|
|
|
|
&ConsoleWriter,
|
|
|
|
|
&FileWriter,
|
2023-05-02 12:38:06 +00:00
|
|
|
|
&zerolog.ConsoleWriter{
|
|
|
|
|
Out: f,
|
|
|
|
|
FieldsExclude: []string{
|
|
|
|
|
"distro_release",
|
|
|
|
|
"distro_type",
|
|
|
|
|
"kernel",
|
|
|
|
|
},
|
2023-05-02 17:40:21 +00:00
|
|
|
|
NoColor: true,
|
2023-05-02 12:38:06 +00:00
|
|
|
|
},
|
2023-05-02 11:10:57 +00:00
|
|
|
|
))
|
|
|
|
|
|
2024-02-17 22:38:43 +00:00
|
|
|
|
switch LogLevel {
|
2023-05-02 11:10:57 +00:00
|
|
|
|
case zerolog.TraceLevel, zerolog.DebugLevel:
|
|
|
|
|
slog = slog.With().Caller().Logger()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
slog = slog.With().Timestamp().
|
2023-05-18 16:07:24 +00:00
|
|
|
|
Str("distro_type", ki.Distro.ID.String()).
|
|
|
|
|
Str("distro_release", ki.Distro.Release).
|
2023-03-19 12:36:19 +00:00
|
|
|
|
Str("kernel", ki.KernelRelease).
|
|
|
|
|
Logger()
|
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
ka.Process(slog, ki,
|
|
|
|
|
cmd.Endless, cmd.Binary, cmd.EndlessStress, cmd.EndlessTimeout,
|
|
|
|
|
func(q *qemu.System, ka artifact.Artifact, ki distro.KernelInfo, result *artifact.Result) {
|
|
|
|
|
dumpResult(q, ka, ki, result, cmd.Dist, cmd.Tag, cmd.Binary, cmd.DB)
|
|
|
|
|
},
|
|
|
|
|
)
|
2018-11-17 19:37:04 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-05-23 21:33:50 +00:00
|
|
|
|
func shuffleKernels(a []distro.KernelInfo) []distro.KernelInfo {
|
2019-08-12 23:21:38 +00:00
|
|
|
|
// Fisher–Yates shuffle
|
|
|
|
|
for i := len(a) - 1; i > 0; i-- {
|
|
|
|
|
j := rand.Intn(i + 1)
|
|
|
|
|
a[i], a[j] = a[j], a[i]
|
|
|
|
|
}
|
|
|
|
|
return a
|
|
|
|
|
}
|
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
func (cmd PewCmd) process(swg *sizedwaitgroup.SizedWaitGroup,
|
|
|
|
|
ka artifact.Artifact, kernel distro.KernelInfo) {
|
|
|
|
|
|
|
|
|
|
if cmd.useRemote {
|
|
|
|
|
go cmd.remote(swg, ka, kernel)
|
|
|
|
|
} else {
|
|
|
|
|
go cmd.testArtifact(swg, ka, kernel)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (cmd PewCmd) performCI(ka artifact.Artifact) (err error) {
|
2018-11-17 19:37:04 +00:00
|
|
|
|
found := false
|
2023-04-05 11:29:31 +00:00
|
|
|
|
max := cmd.Max
|
2018-11-17 19:37:04 +00:00
|
|
|
|
|
2023-05-30 20:55:23 +00:00
|
|
|
|
threadCounter := 0
|
2023-05-30 20:52:41 +00:00
|
|
|
|
|
2023-04-05 11:29:31 +00:00
|
|
|
|
swg := sizedwaitgroup.New(cmd.Threads)
|
|
|
|
|
if cmd.Shuffle {
|
2024-02-20 13:25:31 +00:00
|
|
|
|
cmd.Kcfg.Kernels = shuffleKernels(cmd.Kcfg.Kernels)
|
2023-04-05 11:29:31 +00:00
|
|
|
|
}
|
2024-02-20 13:25:31 +00:00
|
|
|
|
for _, kernel := range cmd.Kcfg.Kernels {
|
2019-08-12 23:21:38 +00:00
|
|
|
|
if max <= 0 {
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
|
2018-11-17 19:37:04 +00:00
|
|
|
|
var supported bool
|
|
|
|
|
supported, err = ka.Supported(kernel)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
2023-06-02 09:52:07 +00:00
|
|
|
|
if kernel.Blocklisted {
|
|
|
|
|
log.Debug().Str("kernel", kernel.KernelVersion).
|
|
|
|
|
Msgf("skip (blocklisted)")
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
if cmd.RootFS != "" {
|
|
|
|
|
kernel.RootFS = cmd.RootFS
|
|
|
|
|
}
|
|
|
|
|
|
2018-11-17 19:37:04 +00:00
|
|
|
|
if supported {
|
|
|
|
|
found = true
|
2019-08-17 09:35:36 +00:00
|
|
|
|
max--
|
2023-04-05 11:29:31 +00:00
|
|
|
|
for i := int64(0); i < cmd.Runs; i++ {
|
2024-02-20 13:25:31 +00:00
|
|
|
|
if !cmd.TimeoutDeadline.IsZero() &&
|
|
|
|
|
time.Now().After(cmd.TimeoutDeadline) {
|
2023-04-05 11:29:31 +00:00
|
|
|
|
|
2019-08-30 00:34:14 +00:00
|
|
|
|
break
|
|
|
|
|
}
|
2019-08-16 18:50:34 +00:00
|
|
|
|
swg.Add()
|
2023-05-30 20:55:23 +00:00
|
|
|
|
if threadCounter < cmd.Threads {
|
2023-05-30 21:04:06 +00:00
|
|
|
|
time.Sleep(time.Second)
|
2023-05-30 20:55:23 +00:00
|
|
|
|
threadCounter++
|
2023-05-30 20:52:41 +00:00
|
|
|
|
}
|
2024-02-20 13:25:31 +00:00
|
|
|
|
|
|
|
|
|
go cmd.process(&swg, ka, kernel)
|
2019-08-16 18:50:34 +00:00
|
|
|
|
}
|
2018-11-17 19:37:04 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
swg.Wait()
|
|
|
|
|
|
|
|
|
|
if !found {
|
2024-02-20 11:37:19 +00:00
|
|
|
|
err = errors.New("no supported kernels found")
|
2018-11-17 19:37:04 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
func kernelMask(kernel string) (km artifact.Target, err error) {
|
2018-11-25 14:43:58 +00:00
|
|
|
|
parts := strings.Split(kernel, ":")
|
|
|
|
|
if len(parts) != 2 {
|
2024-02-20 11:37:19 +00:00
|
|
|
|
err = errors.New("kernel is not 'distroType:regex'")
|
2018-11-25 14:43:58 +00:00
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
2023-05-18 16:07:24 +00:00
|
|
|
|
dt, err := distro.NewID(parts[0])
|
2018-11-25 14:43:58 +00:00
|
|
|
|
if err != nil {
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
2024-02-20 13:25:31 +00:00
|
|
|
|
km = artifact.Target{
|
2023-05-18 18:48:09 +00:00
|
|
|
|
Distro: distro.Distro{ID: dt},
|
2024-02-20 13:25:31 +00:00
|
|
|
|
Kernel: artifact.Kernel{Regex: parts[1]},
|
|
|
|
|
}
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func genOkFail(name string, ok bool) (aurv aurora.Value) {
|
|
|
|
|
s := " " + name
|
|
|
|
|
if name == "" {
|
|
|
|
|
s = ""
|
|
|
|
|
}
|
|
|
|
|
if ok {
|
|
|
|
|
s += " SUCCESS "
|
|
|
|
|
aurv = aurora.BgGreen(aurora.Black(s))
|
|
|
|
|
} else {
|
|
|
|
|
s += " FAILURE "
|
|
|
|
|
aurv = aurora.BgRed(aurora.White(aurora.Bold(s)))
|
2023-05-18 18:48:09 +00:00
|
|
|
|
}
|
2018-11-25 14:43:58 +00:00
|
|
|
|
return
|
|
|
|
|
}
|
2024-02-20 13:25:31 +00:00
|
|
|
|
|
|
|
|
|
func dumpResult(q *qemu.System, ka artifact.Artifact, ki distro.KernelInfo,
|
|
|
|
|
res *artifact.Result, dist, tag, binary string, db *sql.DB) {
|
|
|
|
|
|
|
|
|
|
// TODO refactor
|
|
|
|
|
|
|
|
|
|
if res.InternalError != nil {
|
|
|
|
|
q.Log.Warn().Err(res.InternalError).
|
|
|
|
|
Str("panic", fmt.Sprintf("%v", q.KernelPanic)).
|
|
|
|
|
Str("timeout", fmt.Sprintf("%v", q.KilledByTimeout)).
|
|
|
|
|
Msg("internal")
|
|
|
|
|
res.InternalErrorString = res.InternalError.Error()
|
|
|
|
|
state.InternalErrors += 1
|
|
|
|
|
} else {
|
|
|
|
|
colored := ""
|
|
|
|
|
|
|
|
|
|
state.Overall += 1
|
|
|
|
|
|
|
|
|
|
if res.Test.Ok {
|
|
|
|
|
state.Success += 1
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
switch ka.Type {
|
|
|
|
|
case artifact.KernelExploit:
|
|
|
|
|
colored = aurora.Sprintf("%s %s",
|
|
|
|
|
genOkFail("BUILD", res.Build.Ok),
|
|
|
|
|
genOkFail("LPE", res.Test.Ok))
|
|
|
|
|
case artifact.KernelModule:
|
|
|
|
|
colored = aurora.Sprintf("%s %s %s",
|
|
|
|
|
genOkFail("BUILD", res.Build.Ok),
|
|
|
|
|
genOkFail("INSMOD", res.Run.Ok),
|
|
|
|
|
genOkFail("TEST", res.Test.Ok))
|
|
|
|
|
case artifact.Script:
|
|
|
|
|
colored = aurora.Sprintf("%s",
|
|
|
|
|
genOkFail("", res.Test.Ok))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
additional := ""
|
|
|
|
|
if q.KernelPanic {
|
|
|
|
|
additional = "(panic)"
|
|
|
|
|
} else if q.KilledByTimeout {
|
|
|
|
|
additional = "(timeout)"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if additional != "" {
|
|
|
|
|
q.Log.Info().Msgf("%v %v", colored, additional)
|
|
|
|
|
} else {
|
|
|
|
|
q.Log.Info().Msgf("%v", colored)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
err := addToLog(db, q, ka, ki, res, tag)
|
|
|
|
|
if err != nil {
|
|
|
|
|
q.Log.Warn().Err(err).Msgf("[db] addToLog (%v)", ka)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if binary == "" && dist != pathDevNull {
|
|
|
|
|
err = os.MkdirAll(dist, os.ModePerm)
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Warn().Err(err).Msgf("os.MkdirAll (%v)", ka)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
path := fmt.Sprintf("%s/%s-%s-%s", dist, ki.Distro.ID,
|
|
|
|
|
ki.Distro.Release, ki.KernelRelease)
|
|
|
|
|
if ka.Type != artifact.KernelExploit {
|
|
|
|
|
path += ".ko"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
err = artifact.CopyFile(res.BuildArtifact, path)
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Warn().Err(err).Msgf("copy file (%v)", ka)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|