tools/out-of-tree
1
0
Fork 0
Code Releases Activity

feat: initial daemon implementation

Browse Source
This commit is contained in:
dump_stack()
2024-02-20 13:25:31 +00:00
parent 820208d079
commit 0314b5ca93
45 changed files with 2989 additions and 1041 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
cmd/daemon.go Normal file
View File

@ -0,0 +1,120 @@
// Copyright 2024 Mikhail Klementev. All rights reserved.
// Use of this source code is governed by a AGPLv3 license
// (or later) that can be found in the LICENSE file.
package cmd
import (
"encoding/json"
"fmt"
"github.com/rs/zerolog/log"
"code.dumpstack.io/tools/out-of-tree/client"
"code.dumpstack.io/tools/out-of-tree/daemon"
)
type DaemonCmd struct {
Addr string `default:":63527"`
Serve DaemonServeCmd `cmd:"" help:"start daemon"`
Job DaemonJobCmd `cmd:"" aliases:"jobs" help:"manage jobs"`
Repo DaemonRepoCmd `cmd:"" aliases:"repos" help:"manage repositories"`
}
type DaemonServeCmd struct{}
func (cmd *DaemonServeCmd) Run(dm *DaemonCmd, g *Globals) (err error) {
d, err := daemon.Init(g.Config.Kernels)
if err != nil {
log.Fatal().Err(err).Msg("")
}
defer d.Kill()
go d.Daemon()
d.Listen(dm.Addr)
return
}
type DaemonJobCmd struct {
List DaemonJobsListCmd `cmd:"" help:"list jobs"`
Status DaemonJobsStatusCmd `cmd:"" help:"show job status"`
Log DaemonJobsLogsCmd `cmd:"" help:"job logs"`
}
type DaemonJobsListCmd struct{}
func (cmd *DaemonJobsListCmd) Run(dm *DaemonCmd, g *Globals) (err error) {
c := client.Client{RemoteAddr: g.RemoteAddr}
jobs, err := c.Jobs()
if err != nil {
log.Error().Err(err).Msg("")
return
}
b, err := json.MarshalIndent(jobs, "", " ")
if err != nil {
log.Error().Err(err).Msg("")
}
fmt.Println(string(b))
return
}
type DaemonJobsStatusCmd struct {
UUID string `arg:""`
}
func (cmd *DaemonJobsStatusCmd) Run(dm *DaemonCmd, g *Globals) (err error) {
c := client.Client{RemoteAddr: g.RemoteAddr}
st, err := c.JobStatus(cmd.UUID)
if err != nil {
log.Error().Err(err).Msg("")
return
}
fmt.Println(st)
return
}
type DaemonJobsLogsCmd struct {
UUID string `arg:""`
}
func (cmd *DaemonJobsLogsCmd) Run(dm *DaemonCmd, g *Globals) (err error) {
c := client.Client{RemoteAddr: g.RemoteAddr}
logs, err := c.JobLogs(cmd.UUID)
if err != nil {
log.Error().Err(err).Msg("")
return
}
for _, l := range logs {
log.Info().Msg(l.Name)
fmt.Println(l.Text)
}
return
}
type DaemonRepoCmd struct {
List DaemonRepoListCmd `cmd:"" help:"list repos"`
}
type DaemonRepoListCmd struct{}
func (cmd *DaemonRepoListCmd) Run(dm *DaemonCmd, g *Globals) (err error) {
c := client.Client{RemoteAddr: g.RemoteAddr}
repos, err := c.Repos()
if err != nil {
return
}
b, err := json.MarshalIndent(repos, "", " ")
if err != nil {
log.Error().Err(err).Msg("")
}
fmt.Println(string(b))
return
}

12
cmd/db.go
View File

@ -12,7 +12,7 @@ import (
_ "github.com/mattn/go-sqlite3"
"code.dumpstack.io/tools/out-of-tree/config"
"code.dumpstack.io/tools/out-of-tree/artifact"
"code.dumpstack.io/tools/out-of-tree/distro"
"code.dumpstack.io/tools/out-of-tree/qemu"
)
@ -28,9 +28,9 @@ type logEntry struct {
Timestamp time.Time
qemu.System
config.Artifact
artifact.Artifact
distro.KernelInfo
phasesResult
artifact.Result
}
func createLogTable(db *sql.DB) (err error) {
@ -123,8 +123,8 @@ func getVersion(db *sql.DB) (version int, err error) {
return
}
func addToLog(db *sql.DB, q *qemu.System, ka config.Artifact,
ki distro.KernelInfo, res *phasesResult, tag string) (err error) {
func addToLog(db *sql.DB, q *qemu.System, ka artifact.Artifact,
ki distro.KernelInfo, res *artifact.Result, tag string) (err error) {
stmt, err := db.Prepare("INSERT INTO log (name, type, tag, " +
"distro_type, distro_release, kernel_release, " +
@ -201,7 +201,7 @@ func getAllLogs(db *sql.DB, tag string, num int) (les []logEntry, err error) {
return
}
func getAllArtifactLogs(db *sql.DB, tag string, num int, ka config.Artifact) (
func getAllArtifactLogs(db *sql.DB, tag string, num int, ka artifact.Artifact) (
les []logEntry, err error) {
stmt, err := db.Prepare("SELECT id, time, name, type, tag, " +

17
cmd/debug.go
View File

@ -14,6 +14,7 @@ import (
"github.com/rs/zerolog/log"
"gopkg.in/logrusorgru/aurora.v2"
"code.dumpstack.io/tools/out-of-tree/artifact"
"code.dumpstack.io/tools/out-of-tree/config"
"code.dumpstack.io/tools/out-of-tree/distro"
"code.dumpstack.io/tools/out-of-tree/fs"
@ -53,7 +54,7 @@ func (cmd *DebugCmd) Run(g *Globals) (err error) {
} else {
configPath = cmd.ArtifactConfig
}
ka, err := config.ReadArtifactConfig(configPath)
ka, err := artifact.Artifact{}.Read(configPath)
if err != nil {
return
}
@ -158,14 +159,14 @@ func (cmd *DebugCmd) Run(g *Globals) (err error) {
if ka.StandardModules {
// Module depends on one of the standard modules
err = copyStandardModules(q, ki)
err = artifact.CopyStandardModules(q, ki)
if err != nil {
log.Print(err)
return
}
}
err = preloadModules(q, ka, ki, g.Config.Docker.Timeout.Duration)
err = artifact.PreloadModules(q, ka, ki, g.Config.Docker.Timeout.Duration)
if err != nil {
log.Print(err)
return
@ -173,20 +174,20 @@ func (cmd *DebugCmd) Run(g *Globals) (err error) {
var buildDir, outFile, output, remoteFile string
if ka.Type == config.Script {
if ka.Type == artifact.Script {
err = q.CopyFile("root", ka.Script, ka.Script)
if err != nil {
return
}
} else {
buildDir, outFile, output, err = build(log.Logger, tmp, ka, ki, g.Config.Docker.Timeout.Duration)
buildDir, outFile, output, err = artifact.Build(log.Logger, tmp, ka, ki, g.Config.Docker.Timeout.Duration)
if err != nil {
log.Print(err, output)
return
}
remoteFile = "/tmp/" + strings.Replace(ka.Name, " ", "_", -1)
if ka.Type == config.KernelModule {
if ka.Type == artifact.KernelModule {
remoteFile += ".ko"
}
@ -222,7 +223,7 @@ func (cmd *DebugCmd) Run(g *Globals) (err error) {
return
}
func firstSupported(kcfg config.KernelConfig, ka config.Artifact,
func firstSupported(kcfg config.KernelConfig, ka artifact.Artifact,
kernel string) (ki distro.KernelInfo, err error) {
km, err := kernelMask(kernel)
@ -230,7 +231,7 @@ func firstSupported(kcfg config.KernelConfig, ka config.Artifact,
return
}
ka.Targets = []config.Target{km}
ka.Targets = []artifact.Target{km}
for _, ki = range kcfg.Kernels {
var supported bool

22
cmd/gen.go
View File

@ -9,7 +9,7 @@ import (
"github.com/naoina/toml"
"code.dumpstack.io/tools/out-of-tree/config"
"code.dumpstack.io/tools/out-of-tree/artifact"
"code.dumpstack.io/tools/out-of-tree/distro"
)
@ -20,30 +20,30 @@ type GenCmd struct {
func (cmd *GenCmd) Run(g *Globals) (err error) {
switch cmd.Type {
case "module":
err = genConfig(config.KernelModule)
err = genConfig(artifact.KernelModule)
case "exploit":
err = genConfig(config.KernelExploit)
err = genConfig(artifact.KernelExploit)
}
return
}
func genConfig(at config.ArtifactType) (err error) {
a := config.Artifact{
func genConfig(at artifact.ArtifactType) (err error) {
a := artifact.Artifact{
Name: "Put name here",
Type: at,
}
a.Targets = append(a.Targets, config.Target{
a.Targets = append(a.Targets, artifact.Target{
Distro: distro.Distro{ID: distro.Ubuntu, Release: "18.04"},
Kernel: config.Kernel{Regex: ".*"},
Kernel: artifact.Kernel{Regex: ".*"},
})
a.Targets = append(a.Targets, config.Target{
a.Targets = append(a.Targets, artifact.Target{
Distro: distro.Distro{ID: distro.Debian, Release: "8"},
Kernel: config.Kernel{Regex: ".*"},
Kernel: artifact.Kernel{Regex: ".*"},
})
a.Preload = append(a.Preload, config.PreloadModule{
a.Preload = append(a.Preload, artifact.PreloadModule{
Repo: "Repo name (e.g. https://github.com/openwall/lkrg)",
})
a.Patches = append(a.Patches, config.Patch{
a.Patches = append(a.Patches, artifact.Patch{
Path: "/path/to/profiling.patch",
})

5
cmd/globals.go
View File

@ -9,7 +9,10 @@ import (
type Globals struct {
Config config.OutOfTree `help:"path to out-of-tree configuration" default:"~/.out-of-tree/out-of-tree.toml"`
WorkDir string `help:"path to work directory" default:"./" type:"path"`
WorkDir string `help:"path to work directory" default:"./" type:"path" existingdir:""`
CacheURL url.URL
Remote bool `help:"run at remote server"`
RemoteAddr string `default:"localhost:63527"`
}

5
cmd/images.go
View File

@ -13,6 +13,7 @@ import (
"time"
"code.dumpstack.io/tools/out-of-tree/config"
"code.dumpstack.io/tools/out-of-tree/config/dotfiles"
"code.dumpstack.io/tools/out-of-tree/distro"
"code.dumpstack.io/tools/out-of-tree/fs"
"code.dumpstack.io/tools/out-of-tree/qemu"
@ -26,7 +27,7 @@ type ImageCmd struct {
type ImageListCmd struct{}
func (cmd *ImageListCmd) Run(g *Globals) (err error) {
entries, err := os.ReadDir(config.Dir("images"))
entries, err := os.ReadDir(dotfiles.Dir("images"))
if err != nil {
return
}
@ -44,7 +45,7 @@ type ImageEditCmd struct {
}
func (cmd *ImageEditCmd) Run(g *Globals) (err error) {
image := filepath.Join(config.Dir("images"), cmd.Name)
image := filepath.Join(dotfiles.Dir("images"), cmd.Name)
if !fs.PathExists(image) {
fmt.Println("image does not exist")
}

22
cmd/kernel.go
View File

@ -15,7 +15,9 @@ import (
"github.com/remeh/sizedwaitgroup"
"github.com/rs/zerolog/log"
"code.dumpstack.io/tools/out-of-tree/artifact"
"code.dumpstack.io/tools/out-of-tree/config"
"code.dumpstack.io/tools/out-of-tree/config/dotfiles"
"code.dumpstack.io/tools/out-of-tree/container"
"code.dumpstack.io/tools/out-of-tree/distro"
"code.dumpstack.io/tools/out-of-tree/kernel"
@ -83,7 +85,7 @@ func (cmd KernelCmd) UpdateConfig() (err error) {
return
}
err = os.WriteFile(config.File("kernels.toml"), buf, os.ModePerm)
err = os.WriteFile(dotfiles.File("kernels.toml"), buf, os.ModePerm)
if err != nil {
return
}
@ -92,7 +94,7 @@ func (cmd KernelCmd) UpdateConfig() (err error) {
return
}
func (cmd *KernelCmd) GenKernel(km config.Target, pkg string) {
func (cmd *KernelCmd) GenKernel(km artifact.Target, pkg string) {
flog := log.With().
Str("kernel", pkg).
Str("distro", km.Distro.String()).
@ -156,7 +158,7 @@ func (cmd *KernelCmd) GenKernel(km config.Target, pkg string) {
}
}
func (cmd *KernelCmd) Generate(g *Globals, km config.Target) (err error) {
func (cmd *KernelCmd) Generate(g *Globals, km artifact.Target) (err error) {
if cmd.Update {
container.UseCache = false
}
@ -263,9 +265,9 @@ func (cmd *KernelListRemoteCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error
return
}
km := config.Target{
km := artifact.Target{
Distro: distro.Distro{ID: distroType, Release: cmd.Ver},
Kernel: config.Kernel{Regex: ".*"},
Kernel: artifact.Kernel{Regex: ".*"},
}
_, err = kernel.GenRootfsImage(km.Distro.RootFS(), false)
@ -289,7 +291,7 @@ func (cmd *KernelListRemoteCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error
type KernelAutogenCmd struct{}
func (cmd *KernelAutogenCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
ka, err := config.ReadArtifactConfig(g.WorkDir + "/.out-of-tree.toml")
ka, err := artifact.Artifact{}.Read(g.WorkDir + "/.out-of-tree.toml")
if err != nil {
return
}
@ -340,9 +342,9 @@ func (cmd *KernelGenallCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
continue
}
target := config.Target{
target := artifact.Target{
Distro: dist,
Kernel: config.Kernel{Regex: ".*"},
Kernel: artifact.Kernel{Regex: ".*"},
}
err = kernelCmd.Generate(g, target)
@ -368,9 +370,9 @@ func (cmd *KernelInstallCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
kernel.SetSigintHandler(&kernelCmd.shutdown)
km := config.Target{
km := artifact.Target{
Distro: distro.Distro{ID: distroType, Release: cmd.Ver},
Kernel: config.Kernel{Regex: cmd.Kernel},
Kernel: artifact.Kernel{Regex: cmd.Kernel},
}
err = kernelCmd.Generate(g, km)
if err != nil {

12
cmd/log.go
View File

@ -15,7 +15,7 @@ import (
"github.com/rs/zerolog/log"
"gopkg.in/logrusorgru/aurora.v2"
"code.dumpstack.io/tools/out-of-tree/config"
"code.dumpstack.io/tools/out-of-tree/artifact"
)
type LogCmd struct {
@ -40,7 +40,7 @@ func (cmd *LogQueryCmd) Run(g *Globals) (err error) {
var les []logEntry
ka, kaErr := config.ReadArtifactConfig(g.WorkDir + "/.out-of-tree.toml")
ka, kaErr := artifact.Artifact{}.Read(g.WorkDir + "/.out-of-tree.toml")
if kaErr == nil {
log.Print(".out-of-tree.toml found, filter by artifact name")
les, err = getAllArtifactLogs(db, cmd.Tag, cmd.Num, ka)
@ -119,7 +119,7 @@ func (cmd *LogDumpCmd) Run(g *Globals) (err error) {
fmt.Println()
fmt.Println("Build ok:", l.Build.Ok)
if l.Type == config.KernelModule {
if l.Type == artifact.KernelModule {
fmt.Println("Insmod ok:", l.Run.Ok)
}
fmt.Println("Test ok:", l.Test.Ok)
@ -128,7 +128,7 @@ func (cmd *LogDumpCmd) Run(g *Globals) (err error) {
fmt.Printf("Build output:\n%s\n", l.Build.Output)
fmt.Println()
if l.Type == config.KernelModule {
if l.Type == artifact.KernelModule {
fmt.Printf("Insmod output:\n%s\n", l.Run.Output)
fmt.Println()
}
@ -232,7 +232,7 @@ func logLogEntry(l logEntry) {
var status aurora.Value
if l.InternalErrorString != "" {
status = genOkFailCentered("INTERNAL", false)
} else if l.Type == config.KernelExploit {
} else if l.Type == artifact.KernelExploit {
if l.Build.Ok {
status = genOkFailCentered("LPE", l.Test.Ok)
} else {
@ -273,7 +273,7 @@ func getStats(db *sql.DB, path, tag string) (
var les []logEntry
ka, kaErr := config.ReadArtifactConfig(path + "/.out-of-tree.toml")
ka, kaErr := artifact.Artifact{}.Read(path + "/.out-of-tree.toml")
if kaErr == nil {
les, err = getAllArtifactLogs(db, tag, -1, ka)
} else {

958
cmd/pew.go
View File

File diff suppressed because it is too large Load Diff

164
cmd/preload.go
View File

@ -1,164 +0,0 @@
// Copyright 2020 Mikhail Klementev. All rights reserved.
// Use of this source code is governed by a AGPLv3 license
// (or later) that can be found in the LICENSE file.
package cmd
import (
"crypto/sha1"
"encoding/hex"
"errors"
"os"
"path/filepath"
"time"
"github.com/go-git/go-git/v5"
"github.com/rs/zerolog/log"
"code.dumpstack.io/tools/out-of-tree/config"
"code.dumpstack.io/tools/out-of-tree/distro"
"code.dumpstack.io/tools/out-of-tree/fs"
"code.dumpstack.io/tools/out-of-tree/qemu"
)
func preloadModules(q *qemu.System, ka config.Artifact, ki distro.KernelInfo,
dockerTimeout time.Duration) (err error) {
for _, pm := range ka.Preload {
err = preload(q, ki, pm, dockerTimeout)
if err != nil {
return
}
}
return
}
func preload(q *qemu.System, ki distro.KernelInfo, pm config.PreloadModule,
dockerTimeout time.Duration) (err error) {
var workPath, cache string
if pm.Path != "" {
log.Print("Use non-git path for preload module (no cache)")
workPath = pm.Path
} else if pm.Repo != "" {
workPath, cache, err = cloneOrPull(pm.Repo, ki)
if err != nil {
return
}
} else {
errors.New("No repo/path in preload entry")
}
err = buildAndInsmod(workPath, q, ki, dockerTimeout, cache)
if err != nil {
return
}
time.Sleep(pm.TimeoutAfterLoad.Duration)
return
}
func buildAndInsmod(workPath string, q *qemu.System, ki distro.KernelInfo,
dockerTimeout time.Duration, cache string) (err error) {
tmp, err := fs.TempDir()
if err != nil {
return
}
defer os.RemoveAll(tmp)
var artifact string
if fs.PathExists(cache) {
artifact = cache
} else {
artifact, err = buildPreload(workPath, tmp, ki, dockerTimeout)
if err != nil {
return
}
if cache != "" {
err = copyFile(artifact, cache)
if err != nil {
return
}
}
}
output, err := q.CopyAndInsmod(artifact)
if err != nil {
log.Print(output)
return
}
return
}
func buildPreload(workPath, tmp string, ki distro.KernelInfo,
dockerTimeout time.Duration) (artifact string, err error) {
ka, err := config.ReadArtifactConfig(workPath + "/.out-of-tree.toml")
if err != nil {
log.Warn().Err(err).Msg("preload")
}
ka.SourcePath = workPath
km := config.Target{
Distro: ki.Distro,
Kernel: config.Kernel{Regex: ki.KernelRelease},
}
ka.Targets = []config.Target{km}
if ka.Docker.Timeout.Duration != 0 {
dockerTimeout = ka.Docker.Timeout.Duration
}
_, artifact, _, err = build(log.Logger, tmp, ka, ki, dockerTimeout)
return
}
func cloneOrPull(repo string, ki distro.KernelInfo) (workPath, cache string,
err error) {
base := config.Dir("preload")
workPath = filepath.Join(base, "/repos/", sha1sum(repo))
var r *git.Repository
if fs.PathExists(workPath) {
r, err = git.PlainOpen(workPath)
if err != nil {
return
}
var w *git.Worktree
w, err = r.Worktree()
if err != nil {
return
}
err = w.Pull(&git.PullOptions{})
if err != nil && err != git.NoErrAlreadyUpToDate {
log.Print(repo, "pull error:", err)
}
} else {
r, err = git.PlainClone(workPath, false, &git.CloneOptions{URL: repo})
if err != nil {
return
}
}
ref, err := r.Head()
if err != nil {
return
}
cachedir := filepath.Join(base, "/cache/")
os.MkdirAll(cachedir, 0700)
filename := sha1sum(repo + ki.KernelPath + ref.Hash().String())
cache = filepath.Join(cachedir, filename)
return
}
func sha1sum(data string) string {
h := sha1.Sum([]byte(data))
return hex.EncodeToString(h[:])
}