1
0
out-of-tree/pew.go

864 lines
19 KiB
Go
Raw Normal View History

// Copyright 2023 Mikhail Klementev. All rights reserved.
2018-11-17 19:37:04 +00:00
// Use of this source code is governed by a AGPLv3 license
// (or later) that can be found in the LICENSE file.
package main
import (
2023-04-06 18:14:24 +00:00
"bufio"
2019-08-13 21:54:59 +00:00
"database/sql"
2018-11-17 19:37:04 +00:00
"errors"
"fmt"
"io"
2018-11-17 19:37:04 +00:00
"io/ioutil"
"math/rand"
"os"
"os/exec"
"strings"
"time"
"github.com/otiai10/copy"
"github.com/remeh/sizedwaitgroup"
2023-03-22 18:32:40 +00:00
"github.com/rs/zerolog"
2023-03-18 21:30:07 +00:00
"github.com/rs/zerolog/log"
"gopkg.in/logrusorgru/aurora.v2"
2018-11-17 19:37:04 +00:00
2019-02-02 21:24:29 +00:00
"code.dumpstack.io/tools/out-of-tree/config"
"code.dumpstack.io/tools/out-of-tree/container"
"code.dumpstack.io/tools/out-of-tree/fs"
2019-02-02 21:24:29 +00:00
"code.dumpstack.io/tools/out-of-tree/qemu"
2018-11-17 19:37:04 +00:00
)
2023-01-31 07:13:33 +00:00
type PewCmd struct {
Max int64 `help:"test no more than X kernels" default:"100500"`
Runs int64 `help:"runs per each kernel" default:"1"`
Kernel string `help:"override kernel regex"`
2023-05-08 22:21:28 +00:00
RootFS string `help:"override rootfs image" type:"existingfile"`
2023-01-31 07:13:33 +00:00
Guess bool `help:"try all defined kernels"`
Shuffle bool `help:"randomize kernels test order"`
2023-01-31 07:13:33 +00:00
Binary string `help:"use binary, do not build"`
Test string `help:"override path for test"`
Dist string `help:"build result path" default:"/dev/null"`
2023-01-31 09:05:43 +00:00
Threads int `help:"threads" default:"1"`
2023-01-31 07:13:33 +00:00
Tag string `help:"log tagging"`
Timeout time.Duration `help:"timeout after tool will not spawn new tests"`
2023-01-31 09:05:43 +00:00
2023-02-01 07:37:08 +00:00
ArtifactConfig string `help:"path to artifact config" type:"path"`
QemuTimeout time.Duration `help:"timeout for qemu"`
QemuAfterStartTimeout time.Duration `help:"timeout after starting of the qemu vm before tests"`
DockerTimeout time.Duration `help:"timeout for docker"`
2023-01-31 09:34:12 +00:00
2023-01-31 09:57:53 +00:00
Threshold float64 `help:"reliablity threshold for exit code" default:"1.00"`
2023-05-08 21:19:06 +00:00
Endless bool `help:"endless tests"`
EndlessTimeout time.Duration `help:"timeout between tests" default:"1m"`
EndlessStress string `help:"endless stress script" type:"existingfile"`
2023-05-08 21:19:06 +00:00
db *sql.DB
kcfg config.KernelConfig
timeoutDeadline time.Time
2023-01-31 07:13:33 +00:00
}
func (cmd *PewCmd) Run(g *Globals) (err error) {
cmd.kcfg, err = config.ReadKernelConfig(g.Config.Kernels)
2023-01-31 07:13:33 +00:00
if err != nil {
log.Fatal().Err(err).Msg("read kernels config")
2023-01-31 07:13:33 +00:00
}
if cmd.Timeout != 0 {
2023-03-18 22:34:30 +00:00
log.Info().Msgf("Set global timeout to %s", cmd.Timeout)
cmd.timeoutDeadline = time.Now().Add(cmd.Timeout)
2023-01-31 07:13:33 +00:00
}
cmd.db, err = openDatabase(g.Config.Database)
2023-01-31 07:13:33 +00:00
if err != nil {
2023-03-18 22:34:30 +00:00
log.Fatal().Err(err).
Msgf("Cannot open database %s", g.Config.Database)
2023-01-31 07:13:33 +00:00
}
defer cmd.db.Close()
2023-01-31 07:13:33 +00:00
2023-02-01 07:37:08 +00:00
var configPath string
if cmd.ArtifactConfig == "" {
configPath = g.WorkDir + "/.out-of-tree.toml"
} else {
configPath = cmd.ArtifactConfig
}
ka, err := config.ReadArtifactConfig(configPath)
2023-01-31 09:05:43 +00:00
if err != nil {
return
}
if ka.SourcePath == "" {
ka.SourcePath = g.WorkDir
}
if cmd.Kernel != "" {
var km config.KernelMask
km, err = kernelMask(cmd.Kernel)
if err != nil {
return
}
ka.SupportedKernels = []config.KernelMask{km}
}
if cmd.Guess {
ka.SupportedKernels, err = genAllKernels()
if err != nil {
return
}
}
if cmd.QemuTimeout != 0 {
2023-03-18 22:34:30 +00:00
log.Info().Msgf("Set qemu timeout to %s", cmd.QemuTimeout)
} else {
cmd.QemuTimeout = g.Config.Qemu.Timeout.Duration
2023-01-31 09:05:43 +00:00
}
if cmd.DockerTimeout != 0 {
2023-03-18 22:34:30 +00:00
log.Info().Msgf("Set docker timeout to %s", cmd.DockerTimeout)
} else {
cmd.DockerTimeout = g.Config.Docker.Timeout.Duration
2023-01-31 09:05:43 +00:00
}
2023-03-16 09:41:49 +00:00
if cmd.Tag == "" {
cmd.Tag = fmt.Sprintf("%d", time.Now().Unix())
}
log.Info().Str("tag", cmd.Tag).Msg("")
2023-03-16 09:41:49 +00:00
err = cmd.performCI(ka)
2023-01-31 09:05:43 +00:00
if err != nil {
return
}
2023-03-18 22:34:30 +00:00
log.Info().Msgf("Success rate: %.02f, Threshold: %.02f",
successRate(state), cmd.Threshold)
2023-01-31 09:56:49 +00:00
if successRate(state) < cmd.Threshold {
2023-01-31 09:34:12 +00:00
err = errors.New("reliability threshold not met")
}
2023-01-31 09:05:43 +00:00
return
2023-01-31 07:13:33 +00:00
}
type runstate struct {
Overall, Success float64
}
var (
state runstate
)
func successRate(state runstate) float64 {
return state.Success / state.Overall
}
2019-08-17 09:05:06 +00:00
const pathDevNull = "/dev/null"
2023-04-06 18:14:24 +00:00
func sh(workdir, command string) (output string, err error) {
flog := log.With().
Str("workdir", workdir).
Str("command", command).
Logger()
2023-03-19 13:14:14 +00:00
2023-04-06 18:14:24 +00:00
cmd := exec.Command("sh", "-c", "cd "+workdir+" && "+command)
flog.Debug().Msgf("%v", cmd)
stdout, err := cmd.StdoutPipe()
if err != nil {
return
}
cmd.Stderr = cmd.Stdout
err = cmd.Start()
if err != nil {
return
}
go func() {
scanner := bufio.NewScanner(stdout)
for scanner.Scan() {
m := scanner.Text()
output += m + "\n"
flog.Trace().Str("stdout", m).Msg("")
}
}()
err = cmd.Wait()
2023-03-19 13:14:14 +00:00
2023-02-16 10:21:44 +00:00
if err != nil {
e := fmt.Sprintf("%v %v output: %v", cmd, err, output)
err = errors.New(e)
}
return
}
func applyPatches(src string, ka config.Artifact) (err error) {
for i, patch := range ka.Patches {
name := fmt.Sprintf("patch_%02d", i)
path := src + "/" + name + ".diff"
if patch.Source != "" && patch.Path != "" {
err = errors.New("path and source are mutually exclusive")
return
} else if patch.Source != "" {
err = os.WriteFile(path, []byte(patch.Source), 0644)
if err != nil {
return
}
} else if patch.Path != "" {
err = copy.Copy(patch.Path, path)
if err != nil {
return
}
}
if patch.Source != "" || patch.Path != "" {
_, err = sh(src, "patch < "+path)
if err != nil {
return
}
}
if patch.Script != "" {
script := src + "/" + name + ".sh"
err = os.WriteFile(script, []byte(patch.Script), 0755)
if err != nil {
return
}
_, err = sh(src, script)
if err != nil {
return
}
}
}
return
}
func build(flog zerolog.Logger, tmp string, ka config.Artifact,
ki config.KernelInfo, dockerTimeout time.Duration) (
outdir, outpath, output string, err error) {
2018-11-17 19:37:04 +00:00
2023-05-17 16:55:22 +00:00
target := fmt.Sprintf("%d", rand.Int())
2018-11-17 19:37:04 +00:00
outdir = tmp + "/source"
2018-11-17 19:37:04 +00:00
err = copy.Copy(ka.SourcePath, outdir)
2018-11-17 19:37:04 +00:00
if err != nil {
return
}
err = applyPatches(outdir, ka)
2023-02-16 10:21:44 +00:00
if err != nil {
return
}
outpath = outdir + "/" + target
2018-11-17 20:18:50 +00:00
if ka.Type == config.KernelModule {
outpath += ".ko"
2018-11-17 19:37:04 +00:00
}
2023-05-14 22:00:29 +00:00
if ki.KernelVersion == "" {
ki.KernelVersion = ki.KernelRelease
}
kernel := "/lib/modules/" + ki.KernelVersion + "/build"
if ki.KernelSource != "" {
kernel = ki.KernelSource
}
2018-11-17 19:37:04 +00:00
2023-02-15 16:54:46 +00:00
buildCommand := "make KERNEL=" + kernel + " TARGET=" + target
if ka.Make.Target != "" {
buildCommand += " " + ka.Make.Target
}
if ki.ContainerName != "" {
var c container.Container
c, err = container.NewFromKernelInfo(ki, dockerTimeout)
c.Log = flog
if err != nil {
log.Fatal().Err(err).Msg("container creation failure")
}
output, err = c.Run(outdir, buildCommand+" && chmod -R 777 /work")
} else {
cmd := exec.Command("bash", "-c", "cd "+outdir+" && "+
2023-02-15 16:54:46 +00:00
buildCommand)
2023-03-19 13:14:14 +00:00
log.Debug().Msgf("%v", cmd)
timer := time.AfterFunc(dockerTimeout, func() {
cmd.Process.Kill()
})
defer timer.Stop()
var raw []byte
raw, err = cmd.CombinedOutput()
if err != nil {
e := fmt.Sprintf("error `%v` for cmd `%v` with output `%v`",
2023-02-15 16:54:46 +00:00
err, buildCommand, string(raw))
err = errors.New(e)
return
}
output = string(raw)
}
2018-11-17 19:37:04 +00:00
return
}
2023-03-22 20:56:44 +00:00
func runScript(q *qemu.System, script string) (output string, err error) {
return q.Command("root", script)
}
2019-08-17 09:05:06 +00:00
func testKernelModule(q *qemu.System, ka config.Artifact,
2018-11-17 20:18:50 +00:00
test string) (output string, err error) {
2018-11-17 19:37:04 +00:00
output, err = q.Command("root", test)
// TODO generic checks for WARNING's and so on
return
}
2019-08-17 09:05:06 +00:00
func testKernelExploit(q *qemu.System, ka config.Artifact,
2018-11-17 20:18:50 +00:00
test, exploit string) (output string, err error) {
2018-11-17 19:37:04 +00:00
output, err = q.Command("user", "chmod +x "+exploit)
if err != nil {
return
}
randFilePath := fmt.Sprintf("/root/%d", rand.Int())
cmd := fmt.Sprintf("%s %s %s", test, exploit, randFilePath)
output, err = q.Command("user", cmd)
if err != nil {
return
}
_, err = q.Command("root", "stat "+randFilePath)
if err != nil {
return
}
return
}
2018-12-10 02:41:45 +00:00
func genOkFail(name string, ok bool) (aurv aurora.Value) {
state.Overall += 1
2023-03-22 20:56:44 +00:00
s := " " + name
if name == "" {
s = ""
}
2018-11-17 19:37:04 +00:00
if ok {
state.Success += 1
2023-03-22 20:56:44 +00:00
s += " SUCCESS "
2018-12-10 02:41:45 +00:00
aurv = aurora.BgGreen(aurora.Black(s))
2018-11-17 19:37:04 +00:00
} else {
2023-03-22 20:56:44 +00:00
s += " FAILURE "
aurv = aurora.BgRed(aurora.White(aurora.Bold(s)))
2018-11-17 19:37:04 +00:00
}
2018-12-10 02:41:45 +00:00
return
2018-11-17 19:37:04 +00:00
}
2019-08-13 21:54:59 +00:00
type phasesResult struct {
BuildDir string
BuildArtifact string
2019-08-13 21:54:59 +00:00
Build, Run, Test struct {
Output string
Ok bool
}
}
func copyFile(sourcePath, destinationPath string) (err error) {
sourceFile, err := os.Open(sourcePath)
if err != nil {
return
}
defer sourceFile.Close()
destinationFile, err := os.Create(destinationPath)
if err != nil {
return err
}
if _, err := io.Copy(destinationFile, sourceFile); err != nil {
destinationFile.Close()
return err
}
return destinationFile.Close()
}
2019-08-17 09:05:06 +00:00
func dumpResult(q *qemu.System, ka config.Artifact, ki config.KernelInfo,
2019-08-16 18:30:46 +00:00
res *phasesResult, dist, tag, binary string, db *sql.DB) {
2018-11-17 20:18:50 +00:00
2018-11-17 19:37:04 +00:00
colored := ""
2023-03-22 20:56:44 +00:00
switch ka.Type {
case config.KernelExploit:
colored = aurora.Sprintf("%s %s",
2019-08-13 21:54:59 +00:00
genOkFail("BUILD", res.Build.Ok),
genOkFail("LPE", res.Test.Ok))
2023-03-22 20:56:44 +00:00
case config.KernelModule:
colored = aurora.Sprintf("%s %s %s",
2019-08-13 21:54:59 +00:00
genOkFail("BUILD", res.Build.Ok),
genOkFail("INSMOD", res.Run.Ok),
genOkFail("TEST", res.Test.Ok))
2023-03-22 20:56:44 +00:00
case config.Script:
colored = aurora.Sprintf("%s",
2023-03-22 20:56:44 +00:00
genOkFail("", res.Test.Ok))
2018-11-17 19:37:04 +00:00
}
additional := ""
if q.KernelPanic {
additional = "(panic)"
} else if q.KilledByTimeout {
additional = "(timeout)"
}
if additional != "" {
q.Log.Info().Msgf("%v %v", colored, additional)
2018-11-17 19:37:04 +00:00
} else {
q.Log.Info().Msgf("%v", colored)
2018-11-17 19:37:04 +00:00
}
2019-08-13 21:54:59 +00:00
2019-08-16 18:30:46 +00:00
err := addToLog(db, q, ka, ki, res, tag)
2019-08-13 21:54:59 +00:00
if err != nil {
q.Log.Warn().Err(err).Msgf("[db] addToLog (%v)", ka)
2019-08-13 21:54:59 +00:00
}
2019-08-17 09:05:06 +00:00
if binary == "" && dist != pathDevNull {
err = os.MkdirAll(dist, os.ModePerm)
if err != nil {
2023-03-19 12:36:19 +00:00
log.Warn().Err(err).Msgf("os.MkdirAll (%v)", ka)
}
path := fmt.Sprintf("%s/%s-%s-%s", dist, ki.DistroType,
ki.DistroRelease, ki.KernelRelease)
if ka.Type != config.KernelExploit {
path += ".ko"
}
err = copyFile(res.BuildArtifact, path)
if err != nil {
2023-03-19 12:36:19 +00:00
log.Warn().Err(err).Msgf("copy file (%v)", ka)
}
}
2018-11-17 19:37:04 +00:00
}
2023-03-22 18:32:40 +00:00
func copyArtifactAndTest(slog zerolog.Logger, q *qemu.System, ka config.Artifact,
2019-08-17 10:00:01 +00:00
res *phasesResult, remoteTest string) (err error) {
2023-03-22 20:56:44 +00:00
// Copy all test files to the remote machine
for _, f := range ka.TestFiles {
if f.Local[0] != '/' {
if res.BuildDir != "" {
f.Local = res.BuildDir + "/" + f.Local
}
}
err = q.CopyFile(f.User, f.Local, f.Remote)
if err != nil {
slog.Error().Err(err).Msg("copy test file")
return
}
}
2019-08-17 10:00:01 +00:00
switch ka.Type {
case config.KernelModule:
res.Run.Output, err = q.CopyAndInsmod(res.BuildArtifact)
if err != nil {
2023-03-22 18:32:40 +00:00
slog.Error().Err(err).Msg(res.Run.Output)
2019-08-17 10:00:01 +00:00
return
}
res.Run.Ok = true
res.Test.Output, err = testKernelModule(q, ka, remoteTest)
if err != nil {
2023-03-22 18:32:40 +00:00
slog.Error().Err(err).Msg(res.Test.Output)
2019-08-17 10:00:01 +00:00
return
}
res.Test.Ok = true
case config.KernelExploit:
remoteExploit := fmt.Sprintf("/tmp/exploit_%d", rand.Int())
err = q.CopyFile("user", res.BuildArtifact, remoteExploit)
if err != nil {
return
}
res.Test.Output, err = testKernelExploit(q, ka, remoteTest,
remoteExploit)
if err != nil {
2023-03-22 18:32:40 +00:00
slog.Error().Err(err).Msg(res.Test.Output)
2019-08-17 10:00:01 +00:00
return
}
res.Run.Ok = true // does not really used
res.Test.Ok = true
2023-03-22 20:56:44 +00:00
case config.Script:
res.Test.Output, err = runScript(q, remoteTest)
if err != nil {
slog.Error().Err(err).Msg(res.Test.Output)
return
}
2023-05-08 19:18:49 +00:00
slog.Debug().Msg(res.Test.Output)
2023-03-22 20:56:44 +00:00
res.Run.Ok = true
res.Test.Ok = true
2019-08-17 10:00:01 +00:00
default:
2023-03-22 18:32:40 +00:00
slog.Fatal().Msg("Unsupported artifact type")
2019-08-17 10:00:01 +00:00
}
return
}
func copyTest(q *qemu.System, testPath string, ka config.Artifact) (
remoteTest string, err error) {
remoteTest = fmt.Sprintf("/tmp/test_%d", rand.Int())
err = q.CopyFile("user", testPath, remoteTest)
if err != nil {
if ka.Type == config.KernelExploit {
q.Command("user",
"echo -e '#!/bin/sh\necho touch $2 | $1' "+
"> "+remoteTest+
" && chmod +x "+remoteTest)
} else {
q.Command("user", "echo '#!/bin/sh' "+
"> "+remoteTest+" && chmod +x "+remoteTest)
}
}
_, err = q.Command("root", "chmod +x "+remoteTest)
return
}
2023-02-15 11:48:25 +00:00
func copyStandardModules(q *qemu.System, ki config.KernelInfo) (err error) {
2023-05-07 18:14:59 +00:00
_, err = q.Command("root", "mkdir -p /lib/modules/"+ki.KernelRelease)
if err != nil {
return
}
remotePath := "/lib/modules/" + ki.KernelRelease + "/"
err = q.CopyDirectory("root", ki.ModulesPath+"/kernel", remotePath+"/kernel")
2023-02-15 11:48:25 +00:00
if err != nil {
return
}
files, err := ioutil.ReadDir(ki.ModulesPath)
if err != nil {
return
}
for _, f := range files {
if f.Mode()&os.ModeSymlink == os.ModeSymlink {
continue
}
2023-05-07 18:14:59 +00:00
if !strings.HasPrefix(f.Name(), "modules") {
continue
}
2023-05-07 18:14:59 +00:00
err = q.CopyFile("root", ki.ModulesPath+"/"+f.Name(), remotePath)
}
return
2023-02-15 11:48:25 +00:00
}
func (cmd PewCmd) testArtifact(swg *sizedwaitgroup.SizedWaitGroup,
ka config.Artifact, ki config.KernelInfo) {
2018-11-17 19:37:04 +00:00
defer swg.Done()
logdir := "logs/" + cmd.Tag
err := os.MkdirAll(logdir, os.ModePerm)
if err != nil {
log.Error().Err(err).Msgf("mkdir %s", logdir)
return
}
logfile := fmt.Sprintf("logs/%s/%s-%s-%s.log",
cmd.Tag,
ki.DistroType.String(),
ki.DistroRelease,
ki.KernelRelease,
)
f, err := os.Create(logfile)
if err != nil {
log.Error().Err(err).Msgf("create %s", logfile)
return
}
defer f.Close()
slog := zerolog.New(zerolog.MultiLevelWriter(
&consoleWriter,
&fileWriter,
&zerolog.ConsoleWriter{
Out: f,
FieldsExclude: []string{
"distro_release",
"distro_type",
"kernel",
},
2023-05-02 17:40:21 +00:00
NoColor: true,
},
))
switch loglevel {
case zerolog.TraceLevel, zerolog.DebugLevel:
slog = slog.With().Caller().Logger()
}
slog = slog.With().Timestamp().
2023-03-19 12:36:19 +00:00
Str("distro_type", ki.DistroType.String()).
Str("distro_release", ki.DistroRelease).
Str("kernel", ki.KernelRelease).
Logger()
slog.Info().Msg("start")
2018-11-17 19:37:04 +00:00
kernel := qemu.Kernel{KernelPath: ki.KernelPath, InitrdPath: ki.InitrdPath}
2023-05-08 22:21:28 +00:00
if cmd.RootFS != "" {
ki.RootFS = cmd.RootFS
}
2019-08-17 09:05:06 +00:00
q, err := qemu.NewSystem(qemu.X86x64, kernel, ki.RootFS)
2018-11-17 19:37:04 +00:00
if err != nil {
2023-03-19 12:36:19 +00:00
slog.Error().Err(err).Msg("qemu init")
2018-11-17 19:37:04 +00:00
return
}
q.Log = slog
q.Timeout = cmd.QemuTimeout
2018-11-17 19:37:04 +00:00
if ka.Qemu.Timeout.Duration != 0 {
q.Timeout = ka.Qemu.Timeout.Duration
}
2019-08-19 19:03:59 +00:00
if ka.Qemu.Cpus != 0 {
q.Cpus = ka.Qemu.Cpus
}
if ka.Qemu.Memory != 0 {
q.Memory = ka.Qemu.Memory
}
if ka.Docker.Timeout.Duration != 0 {
cmd.DockerTimeout = ka.Docker.Timeout.Duration
}
2019-08-19 19:03:59 +00:00
q.SetKASLR(!ka.Mitigations.DisableKaslr)
q.SetSMEP(!ka.Mitigations.DisableSmep)
q.SetSMAP(!ka.Mitigations.DisableSmap)
q.SetKPTI(!ka.Mitigations.DisableKpti)
2023-05-08 21:19:06 +00:00
if cmd.Endless {
q.Timeout = 0
}
2018-11-17 19:37:04 +00:00
err = q.Start()
if err != nil {
2023-03-19 12:36:19 +00:00
slog.Error().Err(err).Msg("qemu start")
2018-11-17 19:37:04 +00:00
return
}
defer q.Stop()
slog.Debug().Msgf("wait %v", cmd.QemuAfterStartTimeout)
time.Sleep(cmd.QemuAfterStartTimeout)
2023-03-18 22:39:24 +00:00
go func() {
2023-05-07 16:05:11 +00:00
time.Sleep(time.Minute)
2023-03-18 22:39:24 +00:00
for !q.Died {
2023-03-19 12:36:19 +00:00
slog.Debug().Msg("still alive")
2023-05-07 16:05:11 +00:00
time.Sleep(time.Minute)
2023-03-18 22:39:24 +00:00
}
}()
2019-11-14 15:38:16 +00:00
tmp, err := fs.TempDir()
2018-11-17 19:37:04 +00:00
if err != nil {
2023-03-19 12:36:19 +00:00
slog.Error().Err(err).Msg("making tmp directory")
2018-11-17 19:37:04 +00:00
return
}
defer os.RemoveAll(tmp)
2019-08-13 21:54:59 +00:00
result := phasesResult{}
2023-05-08 21:19:06 +00:00
if !cmd.Endless {
defer dumpResult(q, ka, ki, &result, cmd.Dist, cmd.Tag, cmd.Binary, cmd.db)
}
2018-11-17 19:37:04 +00:00
2023-03-22 20:56:44 +00:00
if ka.Type == config.Script {
result.Build.Ok = true
cmd.Test = ka.Script
} else if cmd.Binary == "" {
// TODO: build should return structure
2023-03-18 22:55:38 +00:00
start := time.Now()
result.BuildDir, result.BuildArtifact, result.Build.Output, err =
build(slog, tmp, ka, ki, cmd.DockerTimeout)
2023-03-19 12:36:19 +00:00
slog.Debug().Str("duration", time.Now().Sub(start).String()).
Msg("build done")
2018-11-17 19:37:04 +00:00
if err != nil {
2023-03-19 12:36:19 +00:00
log.Error().Err(err).Msg("build")
2018-11-17 19:37:04 +00:00
return
}
2019-08-13 21:54:59 +00:00
result.Build.Ok = true
2018-11-17 19:37:04 +00:00
} else {
result.BuildArtifact = cmd.Binary
2019-08-13 21:54:59 +00:00
result.Build.Ok = true
2018-11-17 19:37:04 +00:00
}
if cmd.Test == "" {
cmd.Test = result.BuildArtifact + "_test"
if !fs.PathExists(cmd.Test) {
slog.Debug().Msgf("%s does not exist", cmd.Test)
cmd.Test = tmp + "/source/" + "test.sh"
} else {
slog.Debug().Msgf("%s exist", cmd.Test)
2019-08-18 15:04:24 +00:00
}
2018-11-17 19:37:04 +00:00
}
err = q.WaitForSSH(cmd.QemuTimeout)
2023-02-16 06:27:17 +00:00
if err != nil {
return
}
remoteTest, err := copyTest(q, cmd.Test, ka)
2018-11-17 19:37:04 +00:00
if err != nil {
2023-05-01 14:51:58 +00:00
slog.Error().Err(err).Msg("copy test script")
2019-08-17 10:00:01 +00:00
return
2018-11-17 19:37:04 +00:00
}
2023-02-15 11:48:25 +00:00
if ka.StandardModules {
// Module depends on one of the standard modules
2023-03-18 22:48:24 +00:00
start := time.Now()
2023-02-15 11:48:25 +00:00
err = copyStandardModules(q, ki)
if err != nil {
2023-05-01 14:52:16 +00:00
slog.Error().Err(err).Msg("copy standard modules")
2023-02-15 11:48:25 +00:00
return
}
2023-03-19 12:36:19 +00:00
slog.Debug().Str("duration", time.Now().Sub(start).String()).
Msg("copy standard modules")
2023-02-15 11:48:25 +00:00
}
err = preloadModules(q, ka, ki, cmd.DockerTimeout)
2020-06-14 20:14:59 +00:00
if err != nil {
2023-03-19 12:36:19 +00:00
slog.Error().Err(err).Msg("preload modules")
2020-06-14 20:14:59 +00:00
return
}
2023-03-18 22:55:38 +00:00
start := time.Now()
2023-03-22 18:32:40 +00:00
copyArtifactAndTest(slog, q, ka, &result, remoteTest)
2023-03-19 12:36:19 +00:00
slog.Debug().Str("duration", time.Now().Sub(start).String()).
Msgf("test completed (success: %v)", result.Test.Ok)
2023-05-08 21:19:06 +00:00
if !cmd.Endless {
return
}
dumpResult(q, ka, ki, &result, cmd.Dist, cmd.Tag, cmd.Binary, cmd.db)
if !result.Build.Ok || !result.Run.Ok || !result.Test.Ok {
return
}
slog.Info().Msg("start endless tests")
if cmd.EndlessStress != "" {
slog.Debug().Msg("copy and run endless stress script")
err = q.CopyAndRunAsync("root", cmd.EndlessStress)
if err != nil {
q.Stop()
2023-05-08 22:31:46 +00:00
f.Sync()
slog.Fatal().Err(err).Msg("cannot copy/run stress")
return
}
}
2023-05-08 21:19:06 +00:00
for {
output, err := q.Command("root", remoteTest)
if err != nil {
2023-05-08 21:36:25 +00:00
q.Stop()
2023-05-08 22:31:46 +00:00
f.Sync()
2023-05-08 21:36:25 +00:00
slog.Fatal().Err(err).Msg(output)
2023-05-08 21:19:06 +00:00
return
}
slog.Debug().Msg(output)
slog.Info().Msg("test success")
slog.Debug().Msgf("wait %v", cmd.EndlessTimeout)
time.Sleep(cmd.EndlessTimeout)
}
2018-11-17 19:37:04 +00:00
}
2019-08-12 23:21:38 +00:00
func shuffleKernels(a []config.KernelInfo) []config.KernelInfo {
// FisherYates shuffle
for i := len(a) - 1; i > 0; i-- {
j := rand.Intn(i + 1)
a[i], a[j] = a[j], a[i]
}
return a
}
func (cmd PewCmd) performCI(ka config.Artifact) (err error) {
2018-11-17 19:37:04 +00:00
found := false
max := cmd.Max
2018-11-17 19:37:04 +00:00
swg := sizedwaitgroup.New(cmd.Threads)
if cmd.Shuffle {
cmd.kcfg.Kernels = shuffleKernels(cmd.kcfg.Kernels)
}
for _, kernel := range cmd.kcfg.Kernels {
2019-08-12 23:21:38 +00:00
if max <= 0 {
break
}
2018-11-17 19:37:04 +00:00
var supported bool
supported, err = ka.Supported(kernel)
if err != nil {
return
}
if supported {
found = true
2019-08-17 09:35:36 +00:00
max--
for i := int64(0); i < cmd.Runs; i++ {
if !cmd.timeoutDeadline.IsZero() &&
time.Now().After(cmd.timeoutDeadline) {
2019-08-30 00:34:14 +00:00
break
}
swg.Add()
go cmd.testArtifact(&swg, ka, kernel)
}
2018-11-17 19:37:04 +00:00
}
}
swg.Wait()
if !found {
err = errors.New("No supported kernels found")
}
return
}
func kernelMask(kernel string) (km config.KernelMask, err error) {
parts := strings.Split(kernel, ":")
if len(parts) != 2 {
err = errors.New("Kernel is not 'distroType:regex'")
return
}
dt, err := config.NewDistroType(parts[0])
if err != nil {
return
}
km = config.KernelMask{DistroType: dt, ReleaseMask: parts[1]}
return
}
func genAllKernels() (sk []config.KernelMask, err error) {
for _, dType := range config.DistroTypeStrings {
var dt config.DistroType
dt, err = config.NewDistroType(dType)
if err != nil {
return
}
sk = append(sk, config.KernelMask{
DistroType: dt,
ReleaseMask: ".*",
})
}
return
}