From 9175305cb9933dce723f79d1ce5d3584a79353fb Mon Sep 17 00:00:00 2001 From: Mikhail Klementev Date: Tue, 13 Aug 2019 23:33:52 +0000 Subject: [PATCH] Implements basic logs query, success rate calculation Closes #10 Closes #11 --- db.go | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++-- log.go | 94 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ main.go | 6 ++++ pew.go | 2 ++ 4 files changed, 187 insertions(+), 2 deletions(-) create mode 100644 log.go diff --git a/db.go b/db.go index a379af3..d3980f0 100644 --- a/db.go +++ b/db.go @@ -6,6 +6,7 @@ package main import ( "database/sql" + "time" _ "github.com/mattn/go-sqlite3" @@ -13,6 +14,16 @@ import ( "code.dumpstack.io/tools/out-of-tree/qemu" ) +type logEntry struct { + ID int + Timestamp time.Time + + qemu.QemuSystem + config.Artifact + config.KernelInfo + phasesResult +} + func createLogTable(db *sql.DB) (err error) { _, err = db.Exec(` CREATE TABLE IF NOT EXISTS log ( @@ -59,8 +70,8 @@ func addToLog(db *sql.DB, q *qemu.QemuSystem, ka config.Artifact, defer stmt.Close() _, err = stmt.Exec( - ka.Name, ka.Type.String(), - ki.DistroType.String(), ki.DistroRelease, ki.KernelRelease, + ka.Name, ka.Type, + ki.DistroType, ki.DistroRelease, ki.KernelRelease, res.Build.Output, res.Build.Ok, res.Run.Output, res.Run.Ok, res.Test.Output, res.Test.Ok, @@ -73,6 +84,78 @@ func addToLog(db *sql.DB, q *qemu.QemuSystem, ka config.Artifact, return } +func getAllLogs(db *sql.DB, num int) (les []logEntry, err error) { + stmt, err := db.Prepare("SELECT id, time, name, type, " + + "distro_type, distro_release, kernel_release, " + + "build_ok, run_ok, test_ok, kernel_panic, " + + "timeout_kill FROM log ORDER BY datetime(time) DESC " + + "LIMIT $1") + if err != nil { + return + } + defer stmt.Close() + + rows, err := stmt.Query(num) + if err != nil { + return + } + defer rows.Close() + + for rows.Next() { + le := logEntry{} + err = rows.Scan(&le.ID, &le.Timestamp, + &le.Name, &le.Type, + &le.DistroType, &le.DistroRelease, &le.KernelRelease, + &le.Build.Ok, &le.Run.Ok, &le.Test.Ok, + &le.KernelPanic, &le.KilledByTimeout, + ) + if err != nil { + return + } + + les = append(les, le) + } + + return +} + +func getAllArtifactLogs(db *sql.DB, num int, ka config.Artifact) ( + les []logEntry, err error) { + + stmt, err := db.Prepare("SELECT id, time, name, type, " + + "distro_type, distro_release, kernel_release, " + + "build_ok, run_ok, test_ok, kernel_panic, " + + "timeout_kill FROM log WHERE name=$1 AND type=$2 " + + "ORDER BY datetime(time) DESC LIMIT $3") + if err != nil { + return + } + defer stmt.Close() + + rows, err := stmt.Query(ka.Name, ka.Type, num) + if err != nil { + return + } + defer rows.Close() + + for rows.Next() { + le := logEntry{} + err = rows.Scan(&le.ID, &le.Timestamp, + &le.Name, &le.Type, + &le.DistroType, &le.DistroRelease, &le.KernelRelease, + &le.Build.Ok, &le.Run.Ok, &le.Test.Ok, + &le.KernelPanic, &le.KilledByTimeout, + ) + if err != nil { + return + } + + les = append(les, le) + } + + return +} + func createSchema(db *sql.DB) (err error) { err = createLogTable(db) if err != nil { diff --git a/log.go b/log.go new file mode 100644 index 0000000..27053a2 --- /dev/null +++ b/log.go @@ -0,0 +1,94 @@ +// Copyright 2019 Mikhail Klementev. All rights reserved. +// Use of this source code is governed by a AGPLv3 license +// (or later) that can be found in the LICENSE file. + +package main + +import ( + "database/sql" + "fmt" + "math" + + "code.dumpstack.io/tools/out-of-tree/config" + "gopkg.in/logrusorgru/aurora.v1" +) + +func logLogEntry(l logEntry) { + distroInfo := fmt.Sprintf("%s-%s {%s}", l.DistroType, + l.DistroRelease, l.KernelRelease) + + artifactInfo := fmt.Sprintf("{[%s] %s}", l.Type, l.Name) + + colored := "" + if l.Type == config.KernelExploit { + colored = aurora.Sprintf("[%s] %40s %40s: %s %s", + l.Timestamp, artifactInfo, distroInfo, + genOkFail("BUILD", l.Build.Ok), + genOkFail("LPE", l.Test.Ok)) + } else { + colored = aurora.Sprintf("[%s] %40s %40s: %s %s %s", + l.Timestamp, artifactInfo, distroInfo, + genOkFail("BUILD", l.Build.Ok), + genOkFail("INSMOD", l.Run.Ok), + genOkFail("TEST", l.Test.Ok)) + } + + additional := "" + if l.KernelPanic { + additional = "(panic)" + } else if l.KilledByTimeout { + additional = "(timeout)" + } + + if additional != "" { + fmt.Println(colored, additional) + } else { + fmt.Println(colored) + } +} + +func logHandler(db *sql.DB, path string, num int, rate bool) (err error) { + var les []logEntry + + ka, kaErr := config.ReadArtifactConfig(path + "/.out-of-tree.toml") + if kaErr == nil { + les, err = getAllArtifactLogs(db, num, ka) + } else { + les, err = getAllLogs(db, num) + } + if err != nil { + return + } + + s := "\nS" + if rate { + if kaErr != nil { + err = kaErr + return + } + + s = fmt.Sprintf("{[%s] %s} Overall s", ka.Type, ka.Name) + + les, err = getAllArtifactLogs(db, math.MaxInt64, ka) + if err != nil { + return + } + } else { + for _, l := range les { + logLogEntry(l) + } + } + + success := 0 + for _, l := range les { + if l.Test.Ok { + success += 1 + } + } + + overall := float64(success) / float64(len(les)) + fmt.Printf("%success rate: %.04f (~%.0f%%)\n", + s, overall, overall*100) + + return +} diff --git a/main.go b/main.go index 530c98f..f3411d2 100644 --- a/main.go +++ b/main.go @@ -158,6 +158,10 @@ func main() { bootstrapCommand := app.Command("bootstrap", "Create directories && download images") + logCommand := app.Command("log", "Query logs") + logNum := logCommand.Flag("num", "How much lines").Default("50").Int() + logRate := logCommand.Flag("rate", "Show artifact success rate").Bool() + err = checkRequiredUtils() if err != nil { log.Fatalln(err) @@ -236,6 +240,8 @@ func main() { *dockerTimeout) case bootstrapCommand.FullCommand(): err = bootstrapHandler() + case logCommand.FullCommand(): + err = logHandler(db, *path, *logNum, *logRate) } if err != nil { diff --git a/pew.go b/pew.go index f019f0c..83076a6 100644 --- a/pew.go +++ b/pew.go @@ -150,6 +150,8 @@ type phasesResult struct { func dumpResult(q *qemu.QemuSystem, ka config.Artifact, ki config.KernelInfo, res *phasesResult, db *sql.DB) { + // TODO merge (problem is it's not 100% same) with log.go:logLogEntry + distroInfo := fmt.Sprintf("%s-%s {%s}", ki.DistroType, ki.DistroRelease, ki.KernelRelease)