tools/out-of-tree
1
0
Fork 0
Code Releases Activity

Implements tagging

Browse Source
This commit is contained in:
dump_stack() 2019-08-16 18:30:46 +00:00
parent ddec4adf57
commit 574d5d45c3
Signed by: dump_stack
GPG Key ID: BE44DA8C062D87DC
4 changed files with 62 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
db.go
View File

@ -16,12 +16,14 @@ import (
"code.dumpstack.io/tools/out-of-tree/qemu"
)
const CURRENT_DATABASE_VERSION = 1
// Change on ANY database update
const CURRENT_DATABASE_VERSION = 2
const VERSION_FIELD = "db_version"
type logEntry struct {
ID int
Tag string
Timestamp time.Time
qemu.QemuSystem
@ -35,6 +37,7 @@ func createLogTable(db *sql.DB) (err error) {
CREATE TABLE IF NOT EXISTS log (
id INTEGER PRIMARY KEY,
time DATETIME DEFAULT CURRENT_TIMESTAMP,
tag TEXT,
name TEXT,
type TEXT,
@ -65,7 +68,7 @@ func createMetadataTable(db *sql.DB) (err error) {
_, err = db.Exec(`
CREATE TABLE IF NOT EXISTS metadata (
id INTEGER PRIMARY KEY,
key TEXT,
key TEXT UNIQUE,
value TEXT
)`)
return
@ -96,7 +99,7 @@ func metaGetValue(db *sql.DB, key string) (value string, err error) {
}
func metaSetValue(db *sql.DB, key, value string) (err error) {
stmt, err := db.Prepare("INSERT INTO metadata " +
stmt, err := db.Prepare("INSERT OR REPLACE INTO metadata " +
"(key, value) VALUES ($1, $2)")
if err != nil {
return
@ -118,9 +121,9 @@ func getVersion(db *sql.DB) (version int, err error) {
}
func addToLog(db *sql.DB, q *qemu.QemuSystem, ka config.Artifact,
ki config.KernelInfo, res *phasesResult) (err error) {
ki config.KernelInfo, res *phasesResult, tag string) (err error) {
stmt, err := db.Prepare("INSERT INTO log (name, type, " +
stmt, err := db.Prepare("INSERT INTO log (name, type, tag, " +
"distro_type, distro_release, kernel_release, " +
"build_output, build_ok, " +
"run_output, run_ok, " +
@ -128,7 +131,7 @@ func addToLog(db *sql.DB, q *qemu.QemuSystem, ka config.Artifact,
"qemu_stdout, qemu_stderr, " +
"kernel_panic, timeout_kill) " +
"VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, " +
"$10, $11, $12, $13, $14, $15);")
"$10, $11, $12, $13, $14, $15, $16);")
if err != nil {
return
}
@ -136,7 +139,7 @@ func addToLog(db *sql.DB, q *qemu.QemuSystem, ka config.Artifact,
defer stmt.Close()
_, err = stmt.Exec(
ka.Name, ka.Type,
ka.Name, ka.Type, tag,
ki.DistroType, ki.DistroRelease, ki.KernelRelease,
res.Build.Output, res.Build.Ok,
res.Run.Output, res.Run.Ok,
@ -151,8 +154,8 @@ func addToLog(db *sql.DB, q *qemu.QemuSystem, ka config.Artifact,
return
}
func getAllLogs(db *sql.DB, num int) (les []logEntry, err error) {
stmt, err := db.Prepare("SELECT id, time, name, type, " +
func getAllLogs(db *sql.DB, tag string, num int) (les []logEntry, err error) {
stmt, err := db.Prepare("SELECT id, time, name, type, tag, " +
"distro_type, distro_release, kernel_release, " +
"build_ok, run_ok, test_ok, kernel_panic, " +
"timeout_kill FROM log ORDER BY datetime(time) DESC " +
@ -171,7 +174,7 @@ func getAllLogs(db *sql.DB, num int) (les []logEntry, err error) {
for rows.Next() {
le := logEntry{}
err = rows.Scan(&le.ID, &le.Timestamp,
&le.Name, &le.Type,
&le.Name, &le.Type, &le.Tag,
&le.DistroType, &le.DistroRelease, &le.KernelRelease,
&le.Build.Ok, &le.Run.Ok, &le.Test.Ok,
&le.KernelPanic, &le.KilledByTimeout,
@ -180,16 +183,18 @@ func getAllLogs(db *sql.DB, num int) (les []logEntry, err error) {
return
}
les = append(les, le)
if tag == "" || tag == le.Tag {
les = append(les, le)
}
}
return
}
func getAllArtifactLogs(db *sql.DB, num int, ka config.Artifact) (
func getAllArtifactLogs(db *sql.DB, tag string, num int, ka config.Artifact) (
les []logEntry, err error) {
stmt, err := db.Prepare("SELECT id, time, name, type, " +
stmt, err := db.Prepare("SELECT id, time, name, type, tag, " +
"distro_type, distro_release, kernel_release, " +
"build_ok, run_ok, test_ok, kernel_panic, " +
"timeout_kill FROM log WHERE name=$1 AND type=$2 " +
@ -208,7 +213,7 @@ func getAllArtifactLogs(db *sql.DB, num int, ka config.Artifact) (
for rows.Next() {
le := logEntry{}
err = rows.Scan(&le.ID, &le.Timestamp,
&le.Name, &le.Type,
&le.Name, &le.Type, &le.Tag,
&le.DistroType, &le.DistroRelease, &le.KernelRelease,
&le.Build.Ok, &le.Run.Ok, &le.Test.Ok,
&le.KernelPanic, &le.KilledByTimeout,
@ -217,14 +222,16 @@ func getAllArtifactLogs(db *sql.DB, num int, ka config.Artifact) (
return
}
les = append(les, le)
if tag == "" || tag == le.Tag {
les = append(les, le)
}
}
return
}
func getLogByID(db *sql.DB, id int) (le logEntry, err error) {
stmt, err := db.Prepare("SELECT id, time, name, type, " +
stmt, err := db.Prepare("SELECT id, time, name, type, tag, " +
"distro_type, distro_release, kernel_release, " +
"build_ok, run_ok, test_ok, " +
"build_output, run_output, test_output, " +
@ -237,7 +244,7 @@ func getLogByID(db *sql.DB, id int) (le logEntry, err error) {
defer stmt.Close()
err = stmt.QueryRow(id).Scan(&le.ID, &le.Timestamp,
&le.Name, &le.Type,
&le.Name, &le.Type, &le.Tag,
&le.DistroType, &le.DistroRelease, &le.KernelRelease,
&le.Build.Ok, &le.Run.Ok, &le.Test.Ok,
&le.Build.Output, &le.Run.Output, &le.Test.Output,
@ -286,6 +293,20 @@ func openDatabase(path string) (db *sql.DB, err error) {
return
}
if version == 1 {
_, err = db.Exec(`ALTER TABLE log ADD tag TEXT`)
if err != nil {
return
}
err = metaSetValue(db, VERSION_FIELD, "2")
if err != nil {
return
}
version = 2
}
if version != CURRENT_DATABASE_VERSION {
err = fmt.Errorf("Database is not supported (%d instead of %d)",
version, CURRENT_DATABASE_VERSION)

17
log.go
View File

@ -21,13 +21,13 @@ func logLogEntry(l logEntry) {
colored := ""
if l.Type == config.KernelExploit {
colored = aurora.Sprintf("[%8d] [%s] %40s %40s: %s %s",
l.ID, l.Timestamp, artifactInfo, distroInfo,
colored = aurora.Sprintf("[%4d %4s] [%s] %40s %40s: %s %s",
l.ID, l.Tag, l.Timestamp, artifactInfo, distroInfo,
genOkFail("BUILD", l.Build.Ok),
genOkFail("LPE", l.Test.Ok))
} else {
colored = aurora.Sprintf("[%8d] [%s] %40s %40s: %s %s %s",
l.ID, l.Timestamp, artifactInfo, distroInfo,
colored = aurora.Sprintf("[%4d %4s] [%s] %40s %40s: %s %s %s",
l.ID, l.Tag, l.Timestamp, artifactInfo, distroInfo,
genOkFail("BUILD", l.Build.Ok),
genOkFail("INSMOD", l.Run.Ok),
genOkFail("TEST", l.Test.Ok))
@ -47,14 +47,14 @@ func logLogEntry(l logEntry) {
}
}
func logHandler(db *sql.DB, path string, num int, rate bool) (err error) {
func logHandler(db *sql.DB, path, tag string, num int, rate bool) (err error) {
var les []logEntry
ka, kaErr := config.ReadArtifactConfig(path + "/.out-of-tree.toml")
if kaErr == nil {
les, err = getAllArtifactLogs(db, num, ka)
les, err = getAllArtifactLogs(db, tag, num, ka)
} else {
les, err = getAllLogs(db, num)
les, err = getAllLogs(db, tag, num)
}
if err != nil {
return
@ -69,7 +69,7 @@ func logHandler(db *sql.DB, path string, num int, rate bool) (err error) {
s = fmt.Sprintf("{[%s] %s} Overall s", ka.Type, ka.Name)
les, err = getAllArtifactLogs(db, math.MaxInt64, ka)
les, err = getAllArtifactLogs(db, tag, math.MaxInt64, ka)
if err != nil {
return
}
@ -101,6 +101,7 @@ func logDumpHandler(db *sql.DB, id int) (err error) {
fmt.Println("ID:", l.ID)
fmt.Println("Date:", l.Timestamp)
fmt.Println("Tag:", l.Tag)
fmt.Println()
fmt.Println("Type:", l.Type.String())

8
main.go
View File

@ -135,6 +135,9 @@ func main() {
pewThreadsFlag := pewCommand.Flag("threads", "Build result path")
pewThreads := pewThreadsFlag.Default(strconv.Itoa(runtime.NumCPU())).Int()
pewTagFlag := pewCommand.Flag("tag", "Log tagging")
pewTag := pewTagFlag.String()
kernelCommand := app.Command("kernel", "Manipulate kernels")
kernelUseHost := kernelCommand.Flag("host", "Use also host kernels").Bool()
kernelListCommand := kernelCommand.Command("list", "List kernels")
@ -180,6 +183,7 @@ func main() {
logQueryCommand := logCommand.Command("query", "Query logs")
logNum := logQueryCommand.Flag("num", "How much lines").Default("50").Int()
logRate := logQueryCommand.Flag("rate", "Show artifact success rate").Bool()
logTag := logQueryCommand.Flag("tag", "Filter tag").String()
logDumpCommand := logCommand.Command("dump",
"Show all info for log entry with ID")
@ -245,7 +249,7 @@ func main() {
case pewCommand.FullCommand():
err = pewHandler(kcfg, *path, *pewKernel, *pewBinary,
*pewTest, *pewGuess, *qemuTimeout, *dockerTimeout,
*pewMax, *pewDist, *pewThreads, db)
*pewMax, *pewDist, *pewTag, *pewThreads, db)
case kernelListCommand.FullCommand():
err = kernelListHandler(kcfg)
case kernelAutogenCommand.FullCommand():
@ -264,7 +268,7 @@ func main() {
case bootstrapCommand.FullCommand():
err = bootstrapHandler()
case logQueryCommand.FullCommand():
err = logHandler(db, *path, *logNum, *logRate)
err = logHandler(db, *path, *logTag, *logNum, *logRate)
case logDumpCommand.FullCommand():
err = logDumpHandler(db, *logDumpID)
}

17
pew.go
View File

@ -189,7 +189,7 @@ func copyFile(sourcePath, destinationPath string) (err error) {
}
func dumpResult(q *qemu.QemuSystem, ka config.Artifact, ki config.KernelInfo,
res *phasesResult, dist, binary string, db *sql.DB) {
res *phasesResult, dist, tag, binary string, db *sql.DB) {
// TODO merge (problem is it's not 100% same) with log.go:logLogEntry
@ -221,7 +221,7 @@ func dumpResult(q *qemu.QemuSystem, ka config.Artifact, ki config.KernelInfo,
fmt.Println(colored)
}
err := addToLog(db, q, ka, ki, res)
err := addToLog(db, q, ka, ki, res, tag)
if err != nil {
log.Println("[db] addToLog (", ka, ") error:", err)
}
@ -247,7 +247,8 @@ func dumpResult(q *qemu.QemuSystem, ka config.Artifact, ki config.KernelInfo,
func whatever(swg *sizedwaitgroup.SizedWaitGroup, ka config.Artifact,
ki config.KernelInfo, binaryPath, testPath string,
qemuTimeout, dockerTimeout time.Duration, dist string, db *sql.DB) {
qemuTimeout, dockerTimeout time.Duration, dist, tag string,
db *sql.DB) {
defer swg.Done()
@ -281,7 +282,7 @@ func whatever(swg *sizedwaitgroup.SizedWaitGroup, ka config.Artifact,
defer os.RemoveAll(tmp)
result := phasesResult{}
defer dumpResult(q, ka, ki, &result, dist, binaryPath, db)
defer dumpResult(q, ka, ki, &result, dist, tag, binaryPath, db)
if binaryPath == "" {
result.BuildArtifact, result.Build.Output, err = build(tmp, ka,
@ -372,7 +373,7 @@ func shuffleKernels(a []config.KernelInfo) []config.KernelInfo {
func performCI(ka config.Artifact, kcfg config.KernelConfig, binaryPath,
testPath string, qemuTimeout, dockerTimeout time.Duration,
max int64, dist string, threads int, db *sql.DB) (err error) {
max int64, dist, tag string, threads int, db *sql.DB) (err error) {
found := false
@ -393,7 +394,7 @@ func performCI(ka config.Artifact, kcfg config.KernelConfig, binaryPath,
max -= 1
swg.Add()
go whatever(&swg, ka, kernel, binaryPath, testPath,
qemuTimeout, dockerTimeout, dist, db)
qemuTimeout, dockerTimeout, dist, tag, db)
}
}
swg.Wait()
@ -447,7 +448,7 @@ func genAllKernels() (sk []config.KernelMask, err error) {
func pewHandler(kcfg config.KernelConfig,
workPath, ovrrdKrnl, binary, test string, guess bool,
qemuTimeout, dockerTimeout time.Duration,
max int64, dist string, threads int, db *sql.DB) (err error) {
max int64, dist, tag string, threads int, db *sql.DB) (err error) {
ka, err := config.ReadArtifactConfig(workPath + "/.out-of-tree.toml")
if err != nil {
@ -476,7 +477,7 @@ func pewHandler(kcfg config.KernelConfig,
}
err = performCI(ka, kcfg, binary, test, qemuTimeout, dockerTimeout,
max, dist, threads, db)
max, dist, tag, threads, db)
if err != nil {
return
}