diff --git a/db.go b/db.go index 138ffaf..ebbe1cf 100644 --- a/db.go +++ b/db.go @@ -16,12 +16,14 @@ import ( "code.dumpstack.io/tools/out-of-tree/qemu" ) -const CURRENT_DATABASE_VERSION = 1 +// Change on ANY database update +const CURRENT_DATABASE_VERSION = 2 const VERSION_FIELD = "db_version" type logEntry struct { ID int + Tag string Timestamp time.Time qemu.QemuSystem @@ -35,6 +37,7 @@ func createLogTable(db *sql.DB) (err error) { CREATE TABLE IF NOT EXISTS log ( id INTEGER PRIMARY KEY, time DATETIME DEFAULT CURRENT_TIMESTAMP, + tag TEXT, name TEXT, type TEXT, @@ -65,7 +68,7 @@ func createMetadataTable(db *sql.DB) (err error) { _, err = db.Exec(` CREATE TABLE IF NOT EXISTS metadata ( id INTEGER PRIMARY KEY, - key TEXT, + key TEXT UNIQUE, value TEXT )`) return @@ -96,7 +99,7 @@ func metaGetValue(db *sql.DB, key string) (value string, err error) { } func metaSetValue(db *sql.DB, key, value string) (err error) { - stmt, err := db.Prepare("INSERT INTO metadata " + + stmt, err := db.Prepare("INSERT OR REPLACE INTO metadata " + "(key, value) VALUES ($1, $2)") if err != nil { return @@ -118,9 +121,9 @@ func getVersion(db *sql.DB) (version int, err error) { } func addToLog(db *sql.DB, q *qemu.QemuSystem, ka config.Artifact, - ki config.KernelInfo, res *phasesResult) (err error) { + ki config.KernelInfo, res *phasesResult, tag string) (err error) { - stmt, err := db.Prepare("INSERT INTO log (name, type, " + + stmt, err := db.Prepare("INSERT INTO log (name, type, tag, " + "distro_type, distro_release, kernel_release, " + "build_output, build_ok, " + "run_output, run_ok, " + @@ -128,7 +131,7 @@ func addToLog(db *sql.DB, q *qemu.QemuSystem, ka config.Artifact, "qemu_stdout, qemu_stderr, " + "kernel_panic, timeout_kill) " + "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, " + - "$10, $11, $12, $13, $14, $15);") + "$10, $11, $12, $13, $14, $15, $16);") if err != nil { return } @@ -136,7 +139,7 @@ func addToLog(db *sql.DB, q *qemu.QemuSystem, ka config.Artifact, defer stmt.Close() _, err = stmt.Exec( - ka.Name, ka.Type, + ka.Name, ka.Type, tag, ki.DistroType, ki.DistroRelease, ki.KernelRelease, res.Build.Output, res.Build.Ok, res.Run.Output, res.Run.Ok, @@ -151,8 +154,8 @@ func addToLog(db *sql.DB, q *qemu.QemuSystem, ka config.Artifact, return } -func getAllLogs(db *sql.DB, num int) (les []logEntry, err error) { - stmt, err := db.Prepare("SELECT id, time, name, type, " + +func getAllLogs(db *sql.DB, tag string, num int) (les []logEntry, err error) { + stmt, err := db.Prepare("SELECT id, time, name, type, tag, " + "distro_type, distro_release, kernel_release, " + "build_ok, run_ok, test_ok, kernel_panic, " + "timeout_kill FROM log ORDER BY datetime(time) DESC " + @@ -171,7 +174,7 @@ func getAllLogs(db *sql.DB, num int) (les []logEntry, err error) { for rows.Next() { le := logEntry{} err = rows.Scan(&le.ID, &le.Timestamp, - &le.Name, &le.Type, + &le.Name, &le.Type, &le.Tag, &le.DistroType, &le.DistroRelease, &le.KernelRelease, &le.Build.Ok, &le.Run.Ok, &le.Test.Ok, &le.KernelPanic, &le.KilledByTimeout, @@ -180,16 +183,18 @@ func getAllLogs(db *sql.DB, num int) (les []logEntry, err error) { return } - les = append(les, le) + if tag == "" || tag == le.Tag { + les = append(les, le) + } } return } -func getAllArtifactLogs(db *sql.DB, num int, ka config.Artifact) ( +func getAllArtifactLogs(db *sql.DB, tag string, num int, ka config.Artifact) ( les []logEntry, err error) { - stmt, err := db.Prepare("SELECT id, time, name, type, " + + stmt, err := db.Prepare("SELECT id, time, name, type, tag, " + "distro_type, distro_release, kernel_release, " + "build_ok, run_ok, test_ok, kernel_panic, " + "timeout_kill FROM log WHERE name=$1 AND type=$2 " + @@ -208,7 +213,7 @@ func getAllArtifactLogs(db *sql.DB, num int, ka config.Artifact) ( for rows.Next() { le := logEntry{} err = rows.Scan(&le.ID, &le.Timestamp, - &le.Name, &le.Type, + &le.Name, &le.Type, &le.Tag, &le.DistroType, &le.DistroRelease, &le.KernelRelease, &le.Build.Ok, &le.Run.Ok, &le.Test.Ok, &le.KernelPanic, &le.KilledByTimeout, @@ -217,14 +222,16 @@ func getAllArtifactLogs(db *sql.DB, num int, ka config.Artifact) ( return } - les = append(les, le) + if tag == "" || tag == le.Tag { + les = append(les, le) + } } return } func getLogByID(db *sql.DB, id int) (le logEntry, err error) { - stmt, err := db.Prepare("SELECT id, time, name, type, " + + stmt, err := db.Prepare("SELECT id, time, name, type, tag, " + "distro_type, distro_release, kernel_release, " + "build_ok, run_ok, test_ok, " + "build_output, run_output, test_output, " + @@ -237,7 +244,7 @@ func getLogByID(db *sql.DB, id int) (le logEntry, err error) { defer stmt.Close() err = stmt.QueryRow(id).Scan(&le.ID, &le.Timestamp, - &le.Name, &le.Type, + &le.Name, &le.Type, &le.Tag, &le.DistroType, &le.DistroRelease, &le.KernelRelease, &le.Build.Ok, &le.Run.Ok, &le.Test.Ok, &le.Build.Output, &le.Run.Output, &le.Test.Output, @@ -286,6 +293,20 @@ func openDatabase(path string) (db *sql.DB, err error) { return } + if version == 1 { + _, err = db.Exec(`ALTER TABLE log ADD tag TEXT`) + if err != nil { + return + } + + err = metaSetValue(db, VERSION_FIELD, "2") + if err != nil { + return + } + + version = 2 + } + if version != CURRENT_DATABASE_VERSION { err = fmt.Errorf("Database is not supported (%d instead of %d)", version, CURRENT_DATABASE_VERSION) diff --git a/log.go b/log.go index 3e5de44..ca56edb 100644 --- a/log.go +++ b/log.go @@ -21,13 +21,13 @@ func logLogEntry(l logEntry) { colored := "" if l.Type == config.KernelExploit { - colored = aurora.Sprintf("[%8d] [%s] %40s %40s: %s %s", - l.ID, l.Timestamp, artifactInfo, distroInfo, + colored = aurora.Sprintf("[%4d %4s] [%s] %40s %40s: %s %s", + l.ID, l.Tag, l.Timestamp, artifactInfo, distroInfo, genOkFail("BUILD", l.Build.Ok), genOkFail("LPE", l.Test.Ok)) } else { - colored = aurora.Sprintf("[%8d] [%s] %40s %40s: %s %s %s", - l.ID, l.Timestamp, artifactInfo, distroInfo, + colored = aurora.Sprintf("[%4d %4s] [%s] %40s %40s: %s %s %s", + l.ID, l.Tag, l.Timestamp, artifactInfo, distroInfo, genOkFail("BUILD", l.Build.Ok), genOkFail("INSMOD", l.Run.Ok), genOkFail("TEST", l.Test.Ok)) @@ -47,14 +47,14 @@ func logLogEntry(l logEntry) { } } -func logHandler(db *sql.DB, path string, num int, rate bool) (err error) { +func logHandler(db *sql.DB, path, tag string, num int, rate bool) (err error) { var les []logEntry ka, kaErr := config.ReadArtifactConfig(path + "/.out-of-tree.toml") if kaErr == nil { - les, err = getAllArtifactLogs(db, num, ka) + les, err = getAllArtifactLogs(db, tag, num, ka) } else { - les, err = getAllLogs(db, num) + les, err = getAllLogs(db, tag, num) } if err != nil { return @@ -69,7 +69,7 @@ func logHandler(db *sql.DB, path string, num int, rate bool) (err error) { s = fmt.Sprintf("{[%s] %s} Overall s", ka.Type, ka.Name) - les, err = getAllArtifactLogs(db, math.MaxInt64, ka) + les, err = getAllArtifactLogs(db, tag, math.MaxInt64, ka) if err != nil { return } @@ -101,6 +101,7 @@ func logDumpHandler(db *sql.DB, id int) (err error) { fmt.Println("ID:", l.ID) fmt.Println("Date:", l.Timestamp) + fmt.Println("Tag:", l.Tag) fmt.Println() fmt.Println("Type:", l.Type.String()) diff --git a/main.go b/main.go index 5b5c85f..6f1dc97 100644 --- a/main.go +++ b/main.go @@ -135,6 +135,9 @@ func main() { pewThreadsFlag := pewCommand.Flag("threads", "Build result path") pewThreads := pewThreadsFlag.Default(strconv.Itoa(runtime.NumCPU())).Int() + pewTagFlag := pewCommand.Flag("tag", "Log tagging") + pewTag := pewTagFlag.String() + kernelCommand := app.Command("kernel", "Manipulate kernels") kernelUseHost := kernelCommand.Flag("host", "Use also host kernels").Bool() kernelListCommand := kernelCommand.Command("list", "List kernels") @@ -180,6 +183,7 @@ func main() { logQueryCommand := logCommand.Command("query", "Query logs") logNum := logQueryCommand.Flag("num", "How much lines").Default("50").Int() logRate := logQueryCommand.Flag("rate", "Show artifact success rate").Bool() + logTag := logQueryCommand.Flag("tag", "Filter tag").String() logDumpCommand := logCommand.Command("dump", "Show all info for log entry with ID") @@ -245,7 +249,7 @@ func main() { case pewCommand.FullCommand(): err = pewHandler(kcfg, *path, *pewKernel, *pewBinary, *pewTest, *pewGuess, *qemuTimeout, *dockerTimeout, - *pewMax, *pewDist, *pewThreads, db) + *pewMax, *pewDist, *pewTag, *pewThreads, db) case kernelListCommand.FullCommand(): err = kernelListHandler(kcfg) case kernelAutogenCommand.FullCommand(): @@ -264,7 +268,7 @@ func main() { case bootstrapCommand.FullCommand(): err = bootstrapHandler() case logQueryCommand.FullCommand(): - err = logHandler(db, *path, *logNum, *logRate) + err = logHandler(db, *path, *logTag, *logNum, *logRate) case logDumpCommand.FullCommand(): err = logDumpHandler(db, *logDumpID) } diff --git a/pew.go b/pew.go index 41d5169..deb7913 100644 --- a/pew.go +++ b/pew.go @@ -189,7 +189,7 @@ func copyFile(sourcePath, destinationPath string) (err error) { } func dumpResult(q *qemu.QemuSystem, ka config.Artifact, ki config.KernelInfo, - res *phasesResult, dist, binary string, db *sql.DB) { + res *phasesResult, dist, tag, binary string, db *sql.DB) { // TODO merge (problem is it's not 100% same) with log.go:logLogEntry @@ -221,7 +221,7 @@ func dumpResult(q *qemu.QemuSystem, ka config.Artifact, ki config.KernelInfo, fmt.Println(colored) } - err := addToLog(db, q, ka, ki, res) + err := addToLog(db, q, ka, ki, res, tag) if err != nil { log.Println("[db] addToLog (", ka, ") error:", err) } @@ -247,7 +247,8 @@ func dumpResult(q *qemu.QemuSystem, ka config.Artifact, ki config.KernelInfo, func whatever(swg *sizedwaitgroup.SizedWaitGroup, ka config.Artifact, ki config.KernelInfo, binaryPath, testPath string, - qemuTimeout, dockerTimeout time.Duration, dist string, db *sql.DB) { + qemuTimeout, dockerTimeout time.Duration, dist, tag string, + db *sql.DB) { defer swg.Done() @@ -281,7 +282,7 @@ func whatever(swg *sizedwaitgroup.SizedWaitGroup, ka config.Artifact, defer os.RemoveAll(tmp) result := phasesResult{} - defer dumpResult(q, ka, ki, &result, dist, binaryPath, db) + defer dumpResult(q, ka, ki, &result, dist, tag, binaryPath, db) if binaryPath == "" { result.BuildArtifact, result.Build.Output, err = build(tmp, ka, @@ -372,7 +373,7 @@ func shuffleKernels(a []config.KernelInfo) []config.KernelInfo { func performCI(ka config.Artifact, kcfg config.KernelConfig, binaryPath, testPath string, qemuTimeout, dockerTimeout time.Duration, - max int64, dist string, threads int, db *sql.DB) (err error) { + max int64, dist, tag string, threads int, db *sql.DB) (err error) { found := false @@ -393,7 +394,7 @@ func performCI(ka config.Artifact, kcfg config.KernelConfig, binaryPath, max -= 1 swg.Add() go whatever(&swg, ka, kernel, binaryPath, testPath, - qemuTimeout, dockerTimeout, dist, db) + qemuTimeout, dockerTimeout, dist, tag, db) } } swg.Wait() @@ -447,7 +448,7 @@ func genAllKernels() (sk []config.KernelMask, err error) { func pewHandler(kcfg config.KernelConfig, workPath, ovrrdKrnl, binary, test string, guess bool, qemuTimeout, dockerTimeout time.Duration, - max int64, dist string, threads int, db *sql.DB) (err error) { + max int64, dist, tag string, threads int, db *sql.DB) (err error) { ka, err := config.ReadArtifactConfig(workPath + "/.out-of-tree.toml") if err != nil { @@ -476,7 +477,7 @@ func pewHandler(kcfg config.KernelConfig, } err = performCI(ka, kcfg, binary, test, qemuTimeout, dockerTimeout, - max, dist, threads, db) + max, dist, tag, threads, db) if err != nil { return }