tools/out-of-tree
1
0
Fork 0
Code Releases Activity

Implements tagging

Browse Source
This commit is contained in:
dump_stack() 2019-08-16 18:30:46 +00:00
parent ddec4adf57
commit 574d5d45c3
Signed by: dump_stack
GPG Key ID: BE44DA8C062D87DC
4 changed files with 62 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
db.go
View File

@ -16,12 +16,14 @@ import (
"code.dumpstack.io/tools/out-of-tree/qemu" "code.dumpstack.io/tools/out-of-tree/qemu"
) )
const CURRENT_DATABASE_VERSION = 1 // Change on ANY database update
const CURRENT_DATABASE_VERSION = 2
const VERSION_FIELD = "db_version" const VERSION_FIELD = "db_version"
type logEntry struct { type logEntry struct {
ID int ID int
Tag string
Timestamp time.Time Timestamp time.Time
qemu.QemuSystem qemu.QemuSystem
@ -35,6 +37,7 @@ func createLogTable(db *sql.DB) (err error) {
CREATE TABLE IF NOT EXISTS log ( CREATE TABLE IF NOT EXISTS log (
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
time DATETIME DEFAULT CURRENT_TIMESTAMP, time DATETIME DEFAULT CURRENT_TIMESTAMP,
tag TEXT,
name TEXT, name TEXT,
type TEXT, type TEXT,
@ -65,7 +68,7 @@ func createMetadataTable(db *sql.DB) (err error) {
_, err = db.Exec(` _, err = db.Exec(`
CREATE TABLE IF NOT EXISTS metadata ( CREATE TABLE IF NOT EXISTS metadata (
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
key TEXT, key TEXT UNIQUE,
value TEXT value TEXT
)`) )`)
return return
@ -96,7 +99,7 @@ func metaGetValue(db *sql.DB, key string) (value string, err error) {
} }
func metaSetValue(db *sql.DB, key, value string) (err error) { func metaSetValue(db *sql.DB, key, value string) (err error) {
stmt, err := db.Prepare("INSERT INTO metadata " + stmt, err := db.Prepare("INSERT OR REPLACE INTO metadata " +
"(key, value) VALUES ($1, $2)") "(key, value) VALUES ($1, $2)")
if err != nil { if err != nil {
return return
@ -118,9 +121,9 @@ func getVersion(db *sql.DB) (version int, err error) {
} }
func addToLog(db *sql.DB, q *qemu.QemuSystem, ka config.Artifact, func addToLog(db *sql.DB, q *qemu.QemuSystem, ka config.Artifact,
ki config.KernelInfo, res *phasesResult) (err error) { ki config.KernelInfo, res *phasesResult, tag string) (err error) {
stmt, err := db.Prepare("INSERT INTO log (name, type, " + stmt, err := db.Prepare("INSERT INTO log (name, type, tag, " +
"distro_type, distro_release, kernel_release, " + "distro_type, distro_release, kernel_release, " +
"build_output, build_ok, " + "build_output, build_ok, " +
"run_output, run_ok, " + "run_output, run_ok, " +
@ -128,7 +131,7 @@ func addToLog(db *sql.DB, q *qemu.QemuSystem, ka config.Artifact,
"qemu_stdout, qemu_stderr, " + "qemu_stdout, qemu_stderr, " +
"kernel_panic, timeout_kill) " + "kernel_panic, timeout_kill) " +
"VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, " + "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, " +
"$10, $11, $12, $13, $14, $15);") "$10, $11, $12, $13, $14, $15, $16);")
if err != nil { if err != nil {
return return
} }
@ -136,7 +139,7 @@ func addToLog(db *sql.DB, q *qemu.QemuSystem, ka config.Artifact,
defer stmt.Close() defer stmt.Close()
_, err = stmt.Exec( _, err = stmt.Exec(
ka.Name, ka.Type, ka.Name, ka.Type, tag,
ki.DistroType, ki.DistroRelease, ki.KernelRelease, ki.DistroType, ki.DistroRelease, ki.KernelRelease,
res.Build.Output, res.Build.Ok, res.Build.Output, res.Build.Ok,
res.Run.Output, res.Run.Ok, res.Run.Output, res.Run.Ok,
@ -151,8 +154,8 @@ func addToLog(db *sql.DB, q *qemu.QemuSystem, ka config.Artifact,
return return
} }
func getAllLogs(db *sql.DB, num int) (les []logEntry, err error) { func getAllLogs(db *sql.DB, tag string, num int) (les []logEntry, err error) {
stmt, err := db.Prepare("SELECT id, time, name, type, " + stmt, err := db.Prepare("SELECT id, time, name, type, tag, " +
"distro_type, distro_release, kernel_release, " + "distro_type, distro_release, kernel_release, " +
"build_ok, run_ok, test_ok, kernel_panic, " + "build_ok, run_ok, test_ok, kernel_panic, " +
"timeout_kill FROM log ORDER BY datetime(time) DESC " + "timeout_kill FROM log ORDER BY datetime(time) DESC " +
@ -171,7 +174,7 @@ func getAllLogs(db *sql.DB, num int) (les []logEntry, err error) {
for rows.Next() { for rows.Next() {
le := logEntry{} le := logEntry{}
err = rows.Scan(&le.ID, &le.Timestamp, err = rows.Scan(&le.ID, &le.Timestamp,
&le.Name, &le.Type, &le.Name, &le.Type, &le.Tag,
&le.DistroType, &le.DistroRelease, &le.KernelRelease, &le.DistroType, &le.DistroRelease, &le.KernelRelease,
&le.Build.Ok, &le.Run.Ok, &le.Test.Ok, &le.Build.Ok, &le.Run.Ok, &le.Test.Ok,
&le.KernelPanic, &le.KilledByTimeout, &le.KernelPanic, &le.KilledByTimeout,
@ -180,16 +183,18 @@ func getAllLogs(db *sql.DB, num int) (les []logEntry, err error) {
return return
} }
if tag == "" || tag == le.Tag {
les = append(les, le) les = append(les, le)
} }
}
return return
} }
func getAllArtifactLogs(db *sql.DB, num int, ka config.Artifact) ( func getAllArtifactLogs(db *sql.DB, tag string, num int, ka config.Artifact) (
les []logEntry, err error) { les []logEntry, err error) {
stmt, err := db.Prepare("SELECT id, time, name, type, " + stmt, err := db.Prepare("SELECT id, time, name, type, tag, " +
"distro_type, distro_release, kernel_release, " + "distro_type, distro_release, kernel_release, " +
"build_ok, run_ok, test_ok, kernel_panic, " + "build_ok, run_ok, test_ok, kernel_panic, " +
"timeout_kill FROM log WHERE name=$1 AND type=$2 " + "timeout_kill FROM log WHERE name=$1 AND type=$2 " +
@ -208,7 +213,7 @@ func getAllArtifactLogs(db *sql.DB, num int, ka config.Artifact) (
for rows.Next() { for rows.Next() {
le := logEntry{} le := logEntry{}
err = rows.Scan(&le.ID, &le.Timestamp, err = rows.Scan(&le.ID, &le.Timestamp,
&le.Name, &le.Type, &le.Name, &le.Type, &le.Tag,
&le.DistroType, &le.DistroRelease, &le.KernelRelease, &le.DistroType, &le.DistroRelease, &le.KernelRelease,
&le.Build.Ok, &le.Run.Ok, &le.Test.Ok, &le.Build.Ok, &le.Run.Ok, &le.Test.Ok,
&le.KernelPanic, &le.KilledByTimeout, &le.KernelPanic, &le.KilledByTimeout,
@ -217,14 +222,16 @@ func getAllArtifactLogs(db *sql.DB, num int, ka config.Artifact) (
return return
} }
if tag == "" || tag == le.Tag {
les = append(les, le) les = append(les, le)
} }
}
return return
} }
func getLogByID(db *sql.DB, id int) (le logEntry, err error) { func getLogByID(db *sql.DB, id int) (le logEntry, err error) {
stmt, err := db.Prepare("SELECT id, time, name, type, " + stmt, err := db.Prepare("SELECT id, time, name, type, tag, " +
"distro_type, distro_release, kernel_release, " + "distro_type, distro_release, kernel_release, " +
"build_ok, run_ok, test_ok, " + "build_ok, run_ok, test_ok, " +
"build_output, run_output, test_output, " + "build_output, run_output, test_output, " +
@ -237,7 +244,7 @@ func getLogByID(db *sql.DB, id int) (le logEntry, err error) {
defer stmt.Close() defer stmt.Close()
err = stmt.QueryRow(id).Scan(&le.ID, &le.Timestamp, err = stmt.QueryRow(id).Scan(&le.ID, &le.Timestamp,
&le.Name, &le.Type, &le.Name, &le.Type, &le.Tag,
&le.DistroType, &le.DistroRelease, &le.KernelRelease, &le.DistroType, &le.DistroRelease, &le.KernelRelease,
&le.Build.Ok, &le.Run.Ok, &le.Test.Ok, &le.Build.Ok, &le.Run.Ok, &le.Test.Ok,
&le.Build.Output, &le.Run.Output, &le.Test.Output, &le.Build.Output, &le.Run.Output, &le.Test.Output,
@ -286,6 +293,20 @@ func openDatabase(path string) (db *sql.DB, err error) {
return return
} }
if version == 1 {
_, err = db.Exec(`ALTER TABLE log ADD tag TEXT`)
if err != nil {
return
}
err = metaSetValue(db, VERSION_FIELD, "2")
if err != nil {
return
}
version = 2
}
if version != CURRENT_DATABASE_VERSION { if version != CURRENT_DATABASE_VERSION {
err = fmt.Errorf("Database is not supported (%d instead of %d)", err = fmt.Errorf("Database is not supported (%d instead of %d)",
version, CURRENT_DATABASE_VERSION) version, CURRENT_DATABASE_VERSION)

17
log.go
View File

@ -21,13 +21,13 @@ func logLogEntry(l logEntry) {
colored := "" colored := ""
if l.Type == config.KernelExploit { if l.Type == config.KernelExploit {
colored = aurora.Sprintf("[%8d] [%s] %40s %40s: %s %s", colored = aurora.Sprintf("[%4d %4s] [%s] %40s %40s: %s %s",
l.ID, l.Timestamp, artifactInfo, distroInfo, l.ID, l.Tag, l.Timestamp, artifactInfo, distroInfo,
genOkFail("BUILD", l.Build.Ok), genOkFail("BUILD", l.Build.Ok),
genOkFail("LPE", l.Test.Ok)) genOkFail("LPE", l.Test.Ok))
} else { } else {
colored = aurora.Sprintf("[%8d] [%s] %40s %40s: %s %s %s", colored = aurora.Sprintf("[%4d %4s] [%s] %40s %40s: %s %s %s",
l.ID, l.Timestamp, artifactInfo, distroInfo, l.ID, l.Tag, l.Timestamp, artifactInfo, distroInfo,
genOkFail("BUILD", l.Build.Ok), genOkFail("BUILD", l.Build.Ok),
genOkFail("INSMOD", l.Run.Ok), genOkFail("INSMOD", l.Run.Ok),
genOkFail("TEST", l.Test.Ok)) genOkFail("TEST", l.Test.Ok))
@ -47,14 +47,14 @@ func logLogEntry(l logEntry) {
} }
} }
func logHandler(db *sql.DB, path string, num int, rate bool) (err error) { func logHandler(db *sql.DB, path, tag string, num int, rate bool) (err error) {
var les []logEntry var les []logEntry
ka, kaErr := config.ReadArtifactConfig(path + "/.out-of-tree.toml") ka, kaErr := config.ReadArtifactConfig(path + "/.out-of-tree.toml")
if kaErr == nil { if kaErr == nil {
les, err = getAllArtifactLogs(db, num, ka) les, err = getAllArtifactLogs(db, tag, num, ka)
} else { } else {
les, err = getAllLogs(db, num) les, err = getAllLogs(db, tag, num)
} }
if err != nil { if err != nil {
return return
@ -69,7 +69,7 @@ func logHandler(db *sql.DB, path string, num int, rate bool) (err error) {
s = fmt.Sprintf("{[%s] %s} Overall s", ka.Type, ka.Name) s = fmt.Sprintf("{[%s] %s} Overall s", ka.Type, ka.Name)
les, err = getAllArtifactLogs(db, math.MaxInt64, ka) les, err = getAllArtifactLogs(db, tag, math.MaxInt64, ka)
if err != nil { if err != nil {
return return
} }
@ -101,6 +101,7 @@ func logDumpHandler(db *sql.DB, id int) (err error) {
fmt.Println("ID:", l.ID) fmt.Println("ID:", l.ID)
fmt.Println("Date:", l.Timestamp) fmt.Println("Date:", l.Timestamp)
fmt.Println("Tag:", l.Tag)
fmt.Println() fmt.Println()
fmt.Println("Type:", l.Type.String()) fmt.Println("Type:", l.Type.String())

8
main.go
View File

@ -135,6 +135,9 @@ func main() {
pewThreadsFlag := pewCommand.Flag("threads", "Build result path") pewThreadsFlag := pewCommand.Flag("threads", "Build result path")
pewThreads := pewThreadsFlag.Default(strconv.Itoa(runtime.NumCPU())).Int() pewThreads := pewThreadsFlag.Default(strconv.Itoa(runtime.NumCPU())).Int()
pewTagFlag := pewCommand.Flag("tag", "Log tagging")
pewTag := pewTagFlag.String()
kernelCommand := app.Command("kernel", "Manipulate kernels") kernelCommand := app.Command("kernel", "Manipulate kernels")
kernelUseHost := kernelCommand.Flag("host", "Use also host kernels").Bool() kernelUseHost := kernelCommand.Flag("host", "Use also host kernels").Bool()
kernelListCommand := kernelCommand.Command("list", "List kernels") kernelListCommand := kernelCommand.Command("list", "List kernels")
@ -180,6 +183,7 @@ func main() {
logQueryCommand := logCommand.Command("query", "Query logs") logQueryCommand := logCommand.Command("query", "Query logs")
logNum := logQueryCommand.Flag("num", "How much lines").Default("50").Int() logNum := logQueryCommand.Flag("num", "How much lines").Default("50").Int()
logRate := logQueryCommand.Flag("rate", "Show artifact success rate").Bool() logRate := logQueryCommand.Flag("rate", "Show artifact success rate").Bool()
logTag := logQueryCommand.Flag("tag", "Filter tag").String()
logDumpCommand := logCommand.Command("dump", logDumpCommand := logCommand.Command("dump",
"Show all info for log entry with ID") "Show all info for log entry with ID")
@ -245,7 +249,7 @@ func main() {
case pewCommand.FullCommand(): case pewCommand.FullCommand():
err = pewHandler(kcfg, *path, *pewKernel, *pewBinary, err = pewHandler(kcfg, *path, *pewKernel, *pewBinary,
*pewTest, *pewGuess, *qemuTimeout, *dockerTimeout, *pewTest, *pewGuess, *qemuTimeout, *dockerTimeout,
*pewMax, *pewDist, *pewThreads, db) *pewMax, *pewDist, *pewTag, *pewThreads, db)
case kernelListCommand.FullCommand(): case kernelListCommand.FullCommand():
err = kernelListHandler(kcfg) err = kernelListHandler(kcfg)
case kernelAutogenCommand.FullCommand(): case kernelAutogenCommand.FullCommand():
@ -264,7 +268,7 @@ func main() {
case bootstrapCommand.FullCommand(): case bootstrapCommand.FullCommand():
err = bootstrapHandler() err = bootstrapHandler()
case logQueryCommand.FullCommand(): case logQueryCommand.FullCommand():
err = logHandler(db, *path, *logNum, *logRate) err = logHandler(db, *path, *logTag, *logNum, *logRate)
case logDumpCommand.FullCommand(): case logDumpCommand.FullCommand():
err = logDumpHandler(db, *logDumpID) err = logDumpHandler(db, *logDumpID)
} }

17
pew.go
View File

@ -189,7 +189,7 @@ func copyFile(sourcePath, destinationPath string) (err error) {
} }
func dumpResult(q *qemu.QemuSystem, ka config.Artifact, ki config.KernelInfo, func dumpResult(q *qemu.QemuSystem, ka config.Artifact, ki config.KernelInfo,
res *phasesResult, dist, binary string, db *sql.DB) { res *phasesResult, dist, tag, binary string, db *sql.DB) {
// TODO merge (problem is it's not 100% same) with log.go:logLogEntry // TODO merge (problem is it's not 100% same) with log.go:logLogEntry
@ -221,7 +221,7 @@ func dumpResult(q *qemu.QemuSystem, ka config.Artifact, ki config.KernelInfo,
fmt.Println(colored) fmt.Println(colored)
} }
err := addToLog(db, q, ka, ki, res) err := addToLog(db, q, ka, ki, res, tag)
if err != nil { if err != nil {
log.Println("[db] addToLog (", ka, ") error:", err) log.Println("[db] addToLog (", ka, ") error:", err)
} }
@ -247,7 +247,8 @@ func dumpResult(q *qemu.QemuSystem, ka config.Artifact, ki config.KernelInfo,
func whatever(swg *sizedwaitgroup.SizedWaitGroup, ka config.Artifact, func whatever(swg *sizedwaitgroup.SizedWaitGroup, ka config.Artifact,
ki config.KernelInfo, binaryPath, testPath string, ki config.KernelInfo, binaryPath, testPath string,
qemuTimeout, dockerTimeout time.Duration, dist string, db *sql.DB) { qemuTimeout, dockerTimeout time.Duration, dist, tag string,
db *sql.DB) {
defer swg.Done() defer swg.Done()
@ -281,7 +282,7 @@ func whatever(swg *sizedwaitgroup.SizedWaitGroup, ka config.Artifact,
defer os.RemoveAll(tmp) defer os.RemoveAll(tmp)
result := phasesResult{} result := phasesResult{}
defer dumpResult(q, ka, ki, &result, dist, binaryPath, db) defer dumpResult(q, ka, ki, &result, dist, tag, binaryPath, db)
if binaryPath == "" { if binaryPath == "" {
result.BuildArtifact, result.Build.Output, err = build(tmp, ka, result.BuildArtifact, result.Build.Output, err = build(tmp, ka,
@ -372,7 +373,7 @@ func shuffleKernels(a []config.KernelInfo) []config.KernelInfo {
func performCI(ka config.Artifact, kcfg config.KernelConfig, binaryPath, func performCI(ka config.Artifact, kcfg config.KernelConfig, binaryPath,
testPath string, qemuTimeout, dockerTimeout time.Duration, testPath string, qemuTimeout, dockerTimeout time.Duration,
max int64, dist string, threads int, db *sql.DB) (err error) { max int64, dist, tag string, threads int, db *sql.DB) (err error) {
found := false found := false
@ -393,7 +394,7 @@ func performCI(ka config.Artifact, kcfg config.KernelConfig, binaryPath,
max -= 1 max -= 1
swg.Add() swg.Add()
go whatever(&swg, ka, kernel, binaryPath, testPath, go whatever(&swg, ka, kernel, binaryPath, testPath,
qemuTimeout, dockerTimeout, dist, db) qemuTimeout, dockerTimeout, dist, tag, db)
} }
} }
swg.Wait() swg.Wait()
@ -447,7 +448,7 @@ func genAllKernels() (sk []config.KernelMask, err error) {
func pewHandler(kcfg config.KernelConfig, func pewHandler(kcfg config.KernelConfig,
workPath, ovrrdKrnl, binary, test string, guess bool, workPath, ovrrdKrnl, binary, test string, guess bool,
qemuTimeout, dockerTimeout time.Duration, qemuTimeout, dockerTimeout time.Duration,
max int64, dist string, threads int, db *sql.DB) (err error) { max int64, dist, tag string, threads int, db *sql.DB) (err error) {
ka, err := config.ReadArtifactConfig(workPath + "/.out-of-tree.toml") ka, err := config.ReadArtifactConfig(workPath + "/.out-of-tree.toml")
if err != nil { if err != nil {
@ -476,7 +477,7 @@ func pewHandler(kcfg config.KernelConfig,
} }
err = performCI(ka, kcfg, binary, test, qemuTimeout, dockerTimeout, err = performCI(ka, kcfg, binary, test, qemuTimeout, dockerTimeout,
max, dist, threads, db) max, dist, tag, threads, db)
if err != nil { if err != nil {
return return
} }