1
0
out-of-tree/cmd/pew.go

602 lines
13 KiB
Go
Raw Normal View History

// Copyright 2023 Mikhail Klementev. All rights reserved.
2018-11-17 19:37:04 +00:00
// Use of this source code is governed by a AGPLv3 license
// (or later) that can be found in the LICENSE file.
2024-02-17 22:38:43 +00:00
package cmd
2018-11-17 19:37:04 +00:00
import (
2019-08-13 21:54:59 +00:00
"database/sql"
2018-11-17 19:37:04 +00:00
"errors"
"fmt"
"io"
2018-11-17 19:37:04 +00:00
"math/rand"
"os"
"os/exec"
"strings"
"time"
2024-02-20 13:25:31 +00:00
"github.com/davecgh/go-spew/spew"
2024-02-26 08:55:27 +00:00
"github.com/google/uuid"
2018-11-17 19:37:04 +00:00
"github.com/remeh/sizedwaitgroup"
2023-03-22 18:32:40 +00:00
"github.com/rs/zerolog"
2023-03-18 21:30:07 +00:00
"github.com/rs/zerolog/log"
"gopkg.in/logrusorgru/aurora.v2"
2018-11-17 19:37:04 +00:00
2024-02-20 13:25:31 +00:00
"code.dumpstack.io/tools/out-of-tree/api"
"code.dumpstack.io/tools/out-of-tree/artifact"
"code.dumpstack.io/tools/out-of-tree/client"
2019-02-02 21:24:29 +00:00
"code.dumpstack.io/tools/out-of-tree/config"
"code.dumpstack.io/tools/out-of-tree/distro"
2019-02-02 21:24:29 +00:00
"code.dumpstack.io/tools/out-of-tree/qemu"
2018-11-17 19:37:04 +00:00
)
2024-02-20 13:25:31 +00:00
const pathDevNull = "/dev/null"
2024-02-17 22:38:43 +00:00
type LevelWriter struct {
io.Writer
Level zerolog.Level
}
func (lw *LevelWriter) WriteLevel(l zerolog.Level, p []byte) (n int, err error) {
if l >= lw.Level {
return lw.Writer.Write(p)
}
return len(p), nil
}
var ConsoleWriter, FileWriter LevelWriter
var LogLevel zerolog.Level
2024-02-20 13:25:31 +00:00
type runstate struct {
Overall, Success float64
InternalErrors int
}
var (
state runstate
)
func successRate(state runstate) float64 {
return state.Success / state.Overall
}
2023-01-31 07:13:33 +00:00
type PewCmd struct {
Max int64 `help:"test no more than X kernels" default:"100500"`
Runs int64 `help:"runs per each kernel" default:"1"`
Kernel string `help:"override kernel regex"`
2023-05-08 22:21:28 +00:00
RootFS string `help:"override rootfs image" type:"existingfile"`
2023-01-31 07:13:33 +00:00
Guess bool `help:"try all defined kernels"`
Shuffle bool `help:"randomize kernels test order"`
2023-01-31 07:13:33 +00:00
Binary string `help:"use binary, do not build"`
Test string `help:"override path for test"`
Dist string `help:"build result path" default:"/dev/null"`
2023-01-31 09:05:43 +00:00
Threads int `help:"threads" default:"1"`
2023-01-31 07:13:33 +00:00
Tag string `help:"log tagging"`
Timeout time.Duration `help:"timeout after tool will not spawn new tests"`
2023-01-31 09:05:43 +00:00
2023-02-01 07:37:08 +00:00
ArtifactConfig string `help:"path to artifact config" type:"path"`
QemuTimeout time.Duration `help:"timeout for qemu"`
QemuAfterStartTimeout time.Duration `help:"timeout after starting of the qemu vm before tests"`
DockerTimeout time.Duration `help:"timeout for docker"`
2023-01-31 09:34:12 +00:00
Threshold float64 `help:"reliablity threshold for exit code" default:"1.00"`
IncludeInternalErrors bool `help:"count internal errors as part of the success rate"`
2023-05-08 21:19:06 +00:00
Endless bool `help:"endless tests"`
EndlessTimeout time.Duration `help:"timeout between tests" default:"1m"`
EndlessStress string `help:"endless stress script" type:"existingfile"`
2023-05-08 21:19:06 +00:00
2024-02-20 13:25:31 +00:00
DB *sql.DB `kong:"-" json:"-"`
Kcfg config.KernelConfig `kong:"-" json:"-"`
TimeoutDeadline time.Time `kong:"-" json:"-"`
2024-02-25 18:02:03 +00:00
Watch bool `help:"watch job status"`
2024-02-20 13:25:31 +00:00
repoName string
commit string
useRemote bool
remoteAddr string
2024-02-26 08:55:27 +00:00
// UUID of the job set
groupUUID string
2023-01-31 07:13:33 +00:00
}
2024-02-20 13:25:31 +00:00
func (cmd *PewCmd) getRepoName(worktree string, ka artifact.Artifact) {
raw, err := exec.Command("git", "--work-tree="+worktree,
"rev-list", "--max-parents=0", "HEAD").CombinedOutput()
if err != nil {
log.Error().Err(err).Msg(string(raw))
return
}
cmd.repoName = fmt.Sprintf("%s-%s", ka.Name, string(raw[:7]))
}
func (cmd *PewCmd) syncRepo(worktree string, ka artifact.Artifact) (err error) {
c := client.Client{RemoteAddr: cmd.remoteAddr}
cmd.getRepoName(worktree, ka)
raw, err := exec.Command("git", "--work-tree="+worktree,
"rev-parse", "HEAD").CombinedOutput()
if err != nil {
return
}
cmd.commit = strings.TrimSuffix(string(raw), "\n")
_, err = c.GetRepo(cmd.repoName)
if err != nil && err != client.ErrRepoNotFound {
log.Error().Err(err).Msg("GetRepo API error")
return
}
if err == client.ErrRepoNotFound {
log.Warn().Msg("repo not found")
log.Info().Msg("add repo")
log.Warn().Msgf("%v", spew.Sdump(ka))
err = c.AddRepo(api.Repo{Name: cmd.repoName})
if err != nil {
return
}
}
err = c.PushRepo(api.Repo{Name: cmd.repoName, Path: worktree})
2023-01-31 07:13:33 +00:00
if err != nil {
2024-02-20 13:25:31 +00:00
log.Error().Err(err).Msg("push repo error")
return
}
return
}
func (cmd *PewCmd) Run(g *Globals) (err error) {
2024-02-26 08:55:27 +00:00
cmd.groupUUID = uuid.New().String()
log.Info().Str("group", cmd.groupUUID).Msg("")
2024-02-20 13:25:31 +00:00
cmd.useRemote = g.Remote
cmd.remoteAddr = g.RemoteAddr
if cmd.useRemote {
c := client.Client{RemoteAddr: cmd.remoteAddr}
cmd.Kcfg.Kernels, err = c.Kernels()
if err != nil {
log.Fatal().Err(err).Msg("read kernels config")
}
} else {
cmd.Kcfg, err = config.ReadKernelConfig(
g.Config.Kernels)
if err != nil {
log.Fatal().Err(err).Msg("read kernels config")
}
2023-01-31 07:13:33 +00:00
}
if cmd.Timeout != 0 {
2023-03-18 22:34:30 +00:00
log.Info().Msgf("Set global timeout to %s", cmd.Timeout)
2024-02-20 13:25:31 +00:00
cmd.TimeoutDeadline = time.Now().Add(cmd.Timeout)
2023-01-31 07:13:33 +00:00
}
2024-02-20 13:25:31 +00:00
cmd.DB, err = openDatabase(g.Config.Database)
2023-01-31 07:13:33 +00:00
if err != nil {
2023-03-18 22:34:30 +00:00
log.Fatal().Err(err).
Msgf("Cannot open database %s", g.Config.Database)
2023-01-31 07:13:33 +00:00
}
2024-02-20 13:25:31 +00:00
defer cmd.DB.Close()
2023-01-31 07:13:33 +00:00
2023-02-01 07:37:08 +00:00
var configPath string
if cmd.ArtifactConfig == "" {
configPath = g.WorkDir + "/.out-of-tree.toml"
} else {
configPath = cmd.ArtifactConfig
}
2024-02-20 13:25:31 +00:00
ka, err := artifact.Artifact{}.Read(configPath)
2023-01-31 09:05:43 +00:00
if err != nil {
return
}
2024-02-20 13:25:31 +00:00
if cmd.useRemote {
err = cmd.syncRepo(g.WorkDir, ka)
if err != nil {
return
}
}
if len(ka.Targets) == 0 || cmd.Guess {
log.Debug().Msg("will use all available targets")
for _, dist := range distro.List() {
2024-02-20 13:25:31 +00:00
ka.Targets = append(ka.Targets, artifact.Target{
Distro: dist,
2024-02-20 13:25:31 +00:00
Kernel: artifact.Kernel{
Regex: ".*",
},
})
}
}
2023-01-31 09:05:43 +00:00
if ka.SourcePath == "" {
ka.SourcePath = g.WorkDir
}
if cmd.Kernel != "" {
2024-02-20 13:25:31 +00:00
var km artifact.Target
2023-01-31 09:05:43 +00:00
km, err = kernelMask(cmd.Kernel)
if err != nil {
return
}
2024-02-20 13:25:31 +00:00
ka.Targets = []artifact.Target{km}
2023-01-31 09:05:43 +00:00
}
2024-02-20 13:25:31 +00:00
// TODO there was a lib for merge structures
ka.Qemu.Timeout.Duration = g.Config.Qemu.Timeout.Duration
ka.Docker.Timeout.Duration = g.Config.Docker.Timeout.Duration
2023-01-31 09:05:43 +00:00
if cmd.QemuTimeout != 0 {
2023-03-18 22:34:30 +00:00
log.Info().Msgf("Set qemu timeout to %s", cmd.QemuTimeout)
2024-02-20 13:25:31 +00:00
g.Config.Qemu.Timeout.Duration = cmd.QemuTimeout
2024-02-20 23:00:52 +00:00
ka.Qemu.Timeout.Duration = cmd.QemuTimeout
2023-01-31 09:05:43 +00:00
}
if cmd.DockerTimeout != 0 {
2023-03-18 22:34:30 +00:00
log.Info().Msgf("Set docker timeout to %s", cmd.DockerTimeout)
2024-02-20 13:25:31 +00:00
g.Config.Docker.Timeout.Duration = cmd.DockerTimeout
2024-02-20 23:00:52 +00:00
ka.Docker.Timeout.Duration = cmd.DockerTimeout
2023-01-31 09:05:43 +00:00
}
2023-03-16 09:41:49 +00:00
if cmd.Tag == "" {
cmd.Tag = fmt.Sprintf("%d", time.Now().Unix())
}
2024-02-20 13:25:31 +00:00
if !cmd.useRemote {
log.Info().Str("tag", cmd.Tag).Msg("")
}
2023-03-16 09:41:49 +00:00
err = cmd.performCI(ka)
2023-01-31 09:05:43 +00:00
if err != nil {
return
}
2024-02-20 13:25:31 +00:00
if cmd.useRemote {
return
}
2023-05-31 16:14:08 +00:00
if state.InternalErrors > 0 {
s := "not counted towards success rate"
if cmd.IncludeInternalErrors {
s = "included in success rate"
}
2023-05-31 16:14:08 +00:00
log.Warn().Msgf("%d internal errors "+
"(%s)", state.InternalErrors, s)
}
if cmd.IncludeInternalErrors {
state.Overall += float64(state.InternalErrors)
2023-05-31 16:14:08 +00:00
}
msg := fmt.Sprintf("Success rate: %.02f (%d/%d), Threshold: %.02f",
successRate(state),
int(state.Success), int(state.Overall),
cmd.Threshold)
2023-01-31 09:56:49 +00:00
if successRate(state) < cmd.Threshold {
2023-05-31 16:14:08 +00:00
log.Error().Msg(msg)
2023-01-31 09:34:12 +00:00
err = errors.New("reliability threshold not met")
2023-05-31 16:14:08 +00:00
} else {
log.Info().Msg(msg)
2023-01-31 09:34:12 +00:00
}
2023-05-31 16:14:08 +00:00
2023-01-31 09:05:43 +00:00
return
2023-01-31 07:13:33 +00:00
}
2024-02-20 13:25:31 +00:00
func (cmd PewCmd) watchJob(swg *sizedwaitgroup.SizedWaitGroup,
slog zerolog.Logger, uuid string) {
2024-02-20 13:25:31 +00:00
defer swg.Done() // FIXME
2024-02-20 13:25:31 +00:00
c := client.Client{RemoteAddr: cmd.remoteAddr}
2023-03-19 13:14:14 +00:00
2024-02-20 13:25:31 +00:00
var err error
var st api.Status
2023-02-16 10:21:44 +00:00
2024-02-20 13:25:31 +00:00
for {
st, err = c.JobStatus(uuid)
2019-08-17 10:00:01 +00:00
if err != nil {
2024-02-20 13:25:31 +00:00
slog.Error().Err(err).Msg("")
continue
2019-08-17 10:00:01 +00:00
}
2024-02-20 13:25:31 +00:00
if st == api.StatusSuccess || st == api.StatusFailure {
break
2019-08-17 10:00:01 +00:00
}
2024-02-20 13:25:31 +00:00
time.Sleep(time.Second)
2019-08-17 10:00:01 +00:00
}
2024-02-20 13:25:31 +00:00
switch st {
case api.StatusSuccess:
slog.Info().Msg("success")
case api.StatusFailure:
slog.Warn().Msg("failure")
2023-08-24 00:52:21 +00:00
}
2019-08-17 10:00:01 +00:00
}
2024-02-20 13:25:31 +00:00
func (cmd PewCmd) remote(swg *sizedwaitgroup.SizedWaitGroup,
ka artifact.Artifact, ki distro.KernelInfo) {
2019-08-17 10:00:01 +00:00
2024-02-25 18:02:03 +00:00
defer swg.Done()
2024-02-20 13:25:31 +00:00
slog := log.With().
Str("distro_type", ki.Distro.ID.String()).
Str("distro_release", ki.Distro.Release).
Str("kernel", ki.KernelRelease).
Logger()
2019-08-17 10:00:01 +00:00
2024-02-20 13:25:31 +00:00
job := api.Job{}
2024-02-26 08:55:27 +00:00
job.Group = cmd.groupUUID
2024-02-20 13:25:31 +00:00
job.RepoName = cmd.repoName
job.Commit = cmd.commit
2023-05-07 18:14:59 +00:00
2024-02-20 13:25:31 +00:00
job.Artifact = ka
job.Target = ki
2023-02-15 11:48:25 +00:00
2024-02-20 13:25:31 +00:00
c := client.Client{RemoteAddr: cmd.remoteAddr}
uuid, err := c.AddJob(job)
slog = slog.With().Str("uuid", uuid).Logger()
if err != nil {
2024-02-20 13:25:31 +00:00
slog.Error().Err(err).Msg("cannot add job")
return
}
2024-02-20 13:25:31 +00:00
slog.Info().Msg("add")
2024-02-25 18:02:03 +00:00
if cmd.Watch {
// FIXME dummy (almost)
go cmd.watchJob(swg, slog, uuid)
}
2023-02-15 11:48:25 +00:00
}
func (cmd PewCmd) testArtifact(swg *sizedwaitgroup.SizedWaitGroup,
2024-02-20 13:25:31 +00:00
ka artifact.Artifact, ki distro.KernelInfo) {
2018-11-17 19:37:04 +00:00
defer swg.Done()
logdir := "logs/" + cmd.Tag
err := os.MkdirAll(logdir, os.ModePerm)
if err != nil {
log.Error().Err(err).Msgf("mkdir %s", logdir)
return
}
logfile := fmt.Sprintf("logs/%s/%s-%s-%s.log",
cmd.Tag,
ki.Distro.ID.String(),
ki.Distro.Release,
ki.KernelRelease,
)
f, err := os.Create(logfile)
if err != nil {
log.Error().Err(err).Msgf("create %s", logfile)
return
}
defer f.Close()
slog := zerolog.New(zerolog.MultiLevelWriter(
2024-02-17 22:38:43 +00:00
&ConsoleWriter,
&FileWriter,
&zerolog.ConsoleWriter{
Out: f,
FieldsExclude: []string{
"distro_release",
"distro_type",
"kernel",
},
2023-05-02 17:40:21 +00:00
NoColor: true,
},
))
2024-02-17 22:38:43 +00:00
switch LogLevel {
case zerolog.TraceLevel, zerolog.DebugLevel:
slog = slog.With().Caller().Logger()
}
slog = slog.With().Timestamp().
Str("distro_type", ki.Distro.ID.String()).
Str("distro_release", ki.Distro.Release).
2023-03-19 12:36:19 +00:00
Str("kernel", ki.KernelRelease).
Logger()
2024-02-20 13:25:31 +00:00
ka.Process(slog, ki,
cmd.Endless, cmd.Binary, cmd.EndlessStress, cmd.EndlessTimeout,
func(q *qemu.System, ka artifact.Artifact, ki distro.KernelInfo, result *artifact.Result) {
dumpResult(q, ka, ki, result, cmd.Dist, cmd.Tag, cmd.Binary, cmd.DB)
},
)
2018-11-17 19:37:04 +00:00
}
func shuffleKernels(a []distro.KernelInfo) []distro.KernelInfo {
2019-08-12 23:21:38 +00:00
// FisherYates shuffle
for i := len(a) - 1; i > 0; i-- {
j := rand.Intn(i + 1)
a[i], a[j] = a[j], a[i]
}
return a
}
2024-02-20 13:25:31 +00:00
func (cmd PewCmd) process(swg *sizedwaitgroup.SizedWaitGroup,
ka artifact.Artifact, kernel distro.KernelInfo) {
if cmd.useRemote {
go cmd.remote(swg, ka, kernel)
} else {
go cmd.testArtifact(swg, ka, kernel)
}
}
func (cmd PewCmd) performCI(ka artifact.Artifact) (err error) {
2018-11-17 19:37:04 +00:00
found := false
max := cmd.Max
2018-11-17 19:37:04 +00:00
2023-05-30 20:55:23 +00:00
threadCounter := 0
2023-05-30 20:52:41 +00:00
swg := sizedwaitgroup.New(cmd.Threads)
if cmd.Shuffle {
2024-02-20 13:25:31 +00:00
cmd.Kcfg.Kernels = shuffleKernels(cmd.Kcfg.Kernels)
}
2024-02-20 13:25:31 +00:00
for _, kernel := range cmd.Kcfg.Kernels {
2019-08-12 23:21:38 +00:00
if max <= 0 {
break
}
2018-11-17 19:37:04 +00:00
var supported bool
supported, err = ka.Supported(kernel)
if err != nil {
return
}
if kernel.Blocklisted {
log.Debug().Str("kernel", kernel.KernelVersion).
Msgf("skip (blocklisted)")
continue
}
2024-02-20 13:25:31 +00:00
if cmd.RootFS != "" {
kernel.RootFS = cmd.RootFS
}
2018-11-17 19:37:04 +00:00
if supported {
found = true
2019-08-17 09:35:36 +00:00
max--
for i := int64(0); i < cmd.Runs; i++ {
2024-02-20 13:25:31 +00:00
if !cmd.TimeoutDeadline.IsZero() &&
time.Now().After(cmd.TimeoutDeadline) {
2019-08-30 00:34:14 +00:00
break
}
swg.Add()
2023-05-30 20:55:23 +00:00
if threadCounter < cmd.Threads {
time.Sleep(time.Second)
2023-05-30 20:55:23 +00:00
threadCounter++
2023-05-30 20:52:41 +00:00
}
2024-02-20 13:25:31 +00:00
go cmd.process(&swg, ka, kernel)
}
2018-11-17 19:37:04 +00:00
}
}
swg.Wait()
if !found {
2024-02-20 11:37:19 +00:00
err = errors.New("no supported kernels found")
2018-11-17 19:37:04 +00:00
}
return
}
2024-02-20 13:25:31 +00:00
func kernelMask(kernel string) (km artifact.Target, err error) {
parts := strings.Split(kernel, ":")
if len(parts) != 2 {
2024-02-20 11:37:19 +00:00
err = errors.New("kernel is not 'distroType:regex'")
return
}
dt, err := distro.NewID(parts[0])
if err != nil {
return
}
2024-02-20 13:25:31 +00:00
km = artifact.Target{
Distro: distro.Distro{ID: dt},
2024-02-20 13:25:31 +00:00
Kernel: artifact.Kernel{Regex: parts[1]},
}
return
}
func genOkFail(name string, ok bool) (aurv aurora.Value) {
s := " " + name
if name == "" {
s = ""
}
if ok {
s += " SUCCESS "
aurv = aurora.BgGreen(aurora.Black(s))
} else {
s += " FAILURE "
aurv = aurora.BgRed(aurora.White(aurora.Bold(s)))
}
return
}
2024-02-20 13:25:31 +00:00
func dumpResult(q *qemu.System, ka artifact.Artifact, ki distro.KernelInfo,
res *artifact.Result, dist, tag, binary string, db *sql.DB) {
// TODO refactor
if res.InternalError != nil {
q.Log.Warn().Err(res.InternalError).
Str("panic", fmt.Sprintf("%v", q.KernelPanic)).
Str("timeout", fmt.Sprintf("%v", q.KilledByTimeout)).
Msg("internal")
res.InternalErrorString = res.InternalError.Error()
state.InternalErrors += 1
} else {
colored := ""
state.Overall += 1
if res.Test.Ok {
state.Success += 1
}
switch ka.Type {
case artifact.KernelExploit:
colored = aurora.Sprintf("%s %s",
genOkFail("BUILD", res.Build.Ok),
genOkFail("LPE", res.Test.Ok))
case artifact.KernelModule:
colored = aurora.Sprintf("%s %s %s",
genOkFail("BUILD", res.Build.Ok),
genOkFail("INSMOD", res.Run.Ok),
genOkFail("TEST", res.Test.Ok))
case artifact.Script:
colored = aurora.Sprintf("%s",
genOkFail("", res.Test.Ok))
}
additional := ""
if q.KernelPanic {
additional = "(panic)"
} else if q.KilledByTimeout {
additional = "(timeout)"
}
if additional != "" {
q.Log.Info().Msgf("%v %v", colored, additional)
} else {
q.Log.Info().Msgf("%v", colored)
}
}
err := addToLog(db, q, ka, ki, res, tag)
if err != nil {
q.Log.Warn().Err(err).Msgf("[db] addToLog (%v)", ka)
}
if binary == "" && dist != pathDevNull {
err = os.MkdirAll(dist, os.ModePerm)
if err != nil {
log.Warn().Err(err).Msgf("os.MkdirAll (%v)", ka)
}
path := fmt.Sprintf("%s/%s-%s-%s", dist, ki.Distro.ID,
ki.Distro.Release, ki.KernelRelease)
if ka.Type != artifact.KernelExploit {
path += ".ko"
}
err = artifact.CopyFile(res.BuildArtifact, path)
if err != nil {
log.Warn().Err(err).Msgf("copy file (%v)", ka)
}
}
}