From 986a6f55e0f0701e67bf465b8826746233106fbb Mon Sep 17 00:00:00 2001 From: Mikhail Klementev Date: Sat, 17 Aug 2019 09:05:06 +0000 Subject: [PATCH] Refactor [2] --- CHANGELOG.md | 16 ------------- db.go | 22 ++++++++--------- debug.go | 10 ++++---- main.go | 2 +- pew.go | 14 +++++------ qemu/README.md | 2 +- qemu/qemu-kernel.go | 51 ++++++++++++++++++++-------------------- qemu/qemu-kernel_test.go | 44 +++++++++++++++++----------------- 8 files changed, 73 insertions(+), 88 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a55e42f..157e541 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,33 +12,22 @@ base on `.out-of-tree.toml` definitions) and `pew` (automated runs) and allows to specify a maximum number of runs per each supported kernel in module/exploit definition. - - New command `genall` -- generate all kernels for specified distro/version. - - All logs stores in sqlite3 database. Implemented specific commands for making simple queries and export data to markdown and json. - - Implemented success rate calculation for previous runs. - - Save of build results supported by parameter `--dist` for `pew`. - - Support for generating kernels info from host system. - - Support for build on host. - - Support for custom kernels. - - Now debugging environment is automatically looking for debug kernel on the host system. - - Added ability to enable/disable kaslr/smep/smap for debugging by command line flags. - - New parameter `--threads=N` is added for `pew` and allows to specify maximum number of threads that will be used for parallel build/run/test. - - Tagging for runs. Tags write to log and can be used for statistics. ### Changed @@ -46,13 +35,10 @@ - Now if there's no base image found — out-of-tree will try to use an image from closest previous version, e.g. image from Ubuntu 18.04 for Ubuntu 18.10. - - Kernel modules tests will not be failed if there are no tests exists. - - Now *out-of-tree* will return negative error code if at least one of the stage was failed. - - Project is switch to use Go modules. ### Removed @@ -63,9 +49,7 @@ ### Fixed - Command `timeout` is not required anymore. - - Errors is more meaningful. - - Temporary files is moved to `~/.out-of-tree/tmp/` to avoid docker mounting issues on some systems. ## [0.2.0] - 2019-12-01 diff --git a/db.go b/db.go index ebbe1cf..94b32a4 100644 --- a/db.go +++ b/db.go @@ -17,16 +17,16 @@ import ( ) // Change on ANY database update -const CURRENT_DATABASE_VERSION = 2 +const currentDatabaseVersion = 2 -const VERSION_FIELD = "db_version" +const versionField = "db_version" type logEntry struct { ID int Tag string Timestamp time.Time - qemu.QemuSystem + qemu.System config.Artifact config.KernelInfo phasesResult @@ -111,7 +111,7 @@ func metaSetValue(db *sql.DB, key, value string) (err error) { } func getVersion(db *sql.DB) (version int, err error) { - s, err := metaGetValue(db, VERSION_FIELD) + s, err := metaGetValue(db, versionField) if err != nil { return } @@ -120,7 +120,7 @@ func getVersion(db *sql.DB) (version int, err error) { return } -func addToLog(db *sql.DB, q *qemu.QemuSystem, ka config.Artifact, +func addToLog(db *sql.DB, q *qemu.System, ka config.Artifact, ki config.KernelInfo, res *phasesResult, tag string) (err error) { stmt, err := db.Prepare("INSERT INTO log (name, type, tag, " + @@ -276,15 +276,15 @@ func openDatabase(path string) (db *sql.DB, err error) { db.SetMaxOpenConns(1) - exists, _ := metaChkValue(db, VERSION_FIELD) + exists, _ := metaChkValue(db, versionField) if !exists { err = createSchema(db) if err != nil { return } - err = metaSetValue(db, VERSION_FIELD, - strconv.Itoa(CURRENT_DATABASE_VERSION)) + err = metaSetValue(db, versionField, + strconv.Itoa(currentDatabaseVersion)) return } @@ -299,7 +299,7 @@ func openDatabase(path string) (db *sql.DB, err error) { return } - err = metaSetValue(db, VERSION_FIELD, "2") + err = metaSetValue(db, versionField, "2") if err != nil { return } @@ -307,9 +307,9 @@ func openDatabase(path string) (db *sql.DB, err error) { version = 2 } - if version != CURRENT_DATABASE_VERSION { + if version != currentDatabaseVersion { err = fmt.Errorf("Database is not supported (%d instead of %d)", - version, CURRENT_DATABASE_VERSION) + version, currentDatabaseVersion) return } diff --git a/debug.go b/debug.go index a31c356..48314f7 100644 --- a/debug.go +++ b/debug.go @@ -41,7 +41,7 @@ func firstSupported(kcfg config.KernelConfig, ka config.Artifact, return } -func handleLine(q *qemu.QemuSystem) (err error) { +func handleLine(q *qemu.System) (err error) { fmt.Print("out-of-tree> ") rawLine := "help" fmt.Scanf("%s", &rawLine) @@ -64,7 +64,7 @@ func handleLine(q *qemu.QemuSystem) (err error) { case "c", "cleanup": q.Stdout = []byte{} case "s", "ssh": - fmt.Println(q.GetSshCommand()) + fmt.Println(q.GetSSHCommand()) case "q", "quit": return errors.New("end of session") default: @@ -73,7 +73,7 @@ func handleLine(q *qemu.QemuSystem) (err error) { return } -func interactive(q *qemu.QemuSystem) (err error) { +func interactive(q *qemu.System) (err error) { for { err = handleLine(q) if err != nil { @@ -100,7 +100,7 @@ func debugHandler(kcfg config.KernelConfig, workPath, kernRegex, gdb string, } kernel := qemu.Kernel{KernelPath: ki.KernelPath, InitrdPath: ki.InitrdPath} - q, err := qemu.NewQemuSystem(qemu.X86_64, kernel, ki.RootFS) + q, err := qemu.NewSystem(qemu.X86x64, kernel, ki.RootFS) if err != nil { return } @@ -155,7 +155,7 @@ func debugHandler(kcfg config.KernelConfig, workPath, kernRegex, gdb string, coloredRemoteFile := aurora.BgGreen(aurora.Black(remoteFile)) fmt.Printf("[*] build result copied to %s\n", coloredRemoteFile) - fmt.Printf("\n%s\n", q.GetSshCommand()) + fmt.Printf("\n%s\n", q.GetSSHCommand()) fmt.Printf("gdb %s -ex 'target remote %s'\n\n", ki.VmlinuxPath, gdb) // TODO set substitute-path /build/.../linux-... /path/to/linux-source diff --git a/main.go b/main.go index fffe902..c799b7a 100644 --- a/main.go +++ b/main.go @@ -136,7 +136,7 @@ func main() { pewTest := pewTestFlag.String() pewDistFlag := pewCommand.Flag("dist", "Build result path") - pewDist := pewDistFlag.Default(PATH_DEV_NULL).String() + pewDist := pewDistFlag.Default(pathDevNull).String() pewThreadsFlag := pewCommand.Flag("threads", "Build result path") pewThreads := pewThreadsFlag.Default(strconv.Itoa(runtime.NumCPU())).Int() diff --git a/pew.go b/pew.go index 796ec7d..46c3a82 100644 --- a/pew.go +++ b/pew.go @@ -28,7 +28,7 @@ import ( var somethingFailed = false -const PATH_DEV_NULL = "/dev/null" +const pathDevNull = "/dev/null" func dockerRun(timeout time.Duration, container, workdir, command string) ( output string, err error) { @@ -101,7 +101,7 @@ func build(tmp string, ka config.Artifact, ki config.KernelInfo, return } -func cleanDmesg(q *qemu.QemuSystem) (err error) { +func cleanDmesg(q *qemu.System) (err error) { start := time.Now() for { _, err = q.Command("root", "dmesg -c") @@ -118,7 +118,7 @@ func cleanDmesg(q *qemu.QemuSystem) (err error) { return } -func testKernelModule(q *qemu.QemuSystem, ka config.Artifact, +func testKernelModule(q *qemu.System, ka config.Artifact, test string) (output string, err error) { output, err = q.Command("root", test) @@ -126,7 +126,7 @@ func testKernelModule(q *qemu.QemuSystem, ka config.Artifact, return } -func testKernelExploit(q *qemu.QemuSystem, ka config.Artifact, +func testKernelExploit(q *qemu.System, ka config.Artifact, test, exploit string) (output string, err error) { output, err = q.Command("user", "chmod +x "+exploit) @@ -188,7 +188,7 @@ func copyFile(sourcePath, destinationPath string) (err error) { return destinationFile.Close() } -func dumpResult(q *qemu.QemuSystem, ka config.Artifact, ki config.KernelInfo, +func dumpResult(q *qemu.System, ka config.Artifact, ki config.KernelInfo, res *phasesResult, dist, tag, binary string, db *sql.DB) { // TODO merge (problem is it's not 100% same) with log.go:logLogEntry @@ -226,7 +226,7 @@ func dumpResult(q *qemu.QemuSystem, ka config.Artifact, ki config.KernelInfo, log.Println("[db] addToLog (", ka, ") error:", err) } - if binary == "" && dist != PATH_DEV_NULL { + if binary == "" && dist != pathDevNull { err = os.MkdirAll(dist, os.ModePerm) if err != nil { log.Println("os.MkdirAll (", ka, ") error:", err) @@ -253,7 +253,7 @@ func whatever(swg *sizedwaitgroup.SizedWaitGroup, ka config.Artifact, defer swg.Done() kernel := qemu.Kernel{KernelPath: ki.KernelPath, InitrdPath: ki.InitrdPath} - q, err := qemu.NewQemuSystem(qemu.X86_64, kernel, ki.RootFS) + q, err := qemu.NewSystem(qemu.X86x64, kernel, ki.RootFS) if err != nil { log.Println("Qemu creation error:", err) return diff --git a/qemu/README.md b/qemu/README.md index ff53d32..67aea76 100644 --- a/qemu/README.md +++ b/qemu/README.md @@ -52,7 +52,7 @@ Minimal example: KernelPath: "/path/to/vmlinuz", InitrdPath: "/path/to/initrd", // if required } - q, err := qemu.NewQemuSystem(qemu.X86_64, kernel, "/path/to/qcow2") + q, err := qemu.NewSystem(qemu.X86_64, kernel, "/path/to/qcow2") if err != nil { log.Fatal(err) } diff --git a/qemu/qemu-kernel.go b/qemu/qemu-kernel.go index 82e3c04..3846b68 100644 --- a/qemu/qemu-kernel.go +++ b/qemu/qemu-kernel.go @@ -44,9 +44,10 @@ func readUntilEOF(pipe io.ReadCloser, buf *[]byte) (err error) { type arch string const ( - // X86_64 must be exactly same as in qemu-system-${HERE} - X86_64 arch = "x86_64" - I386 = "i386" + // X86x64 is the qemu-system-x86_64 + X86x64 arch = "x86_64" + // X86x32 is the qemu-system-i386 + X86x32 = "i386" // TODO add other unsupported = "unsupported" // for test purposes @@ -59,8 +60,8 @@ type Kernel struct { InitrdPath string } -// QemuSystem describe qemu parameters and runned process -type QemuSystem struct { +// System describe qemu parameters and runned process +type System struct { arch arch kernel Kernel drivePath string @@ -98,12 +99,12 @@ type QemuSystem struct { exitErr error } -// NewQemuSystem constructor -func NewQemuSystem(arch arch, kernel Kernel, drivePath string) (q *QemuSystem, err error) { +// NewSystem constructor +func NewSystem(arch arch, kernel Kernel, drivePath string) (q *System, err error) { if _, err = exec.LookPath("qemu-system-" + string(arch)); err != nil { return } - q = &QemuSystem{} + q = &System{} q.arch = arch if _, err = os.Stat(kernel.KernelPath); err != nil { @@ -164,7 +165,7 @@ func kvmExists() bool { return true } -func (q *QemuSystem) panicWatcher() { +func (q *System) panicWatcher() { for { time.Sleep(time.Second) if bytes.Contains(q.Stdout, []byte("Kernel panic")) { @@ -178,7 +179,7 @@ func (q *QemuSystem) panicWatcher() { } // Start qemu process -func (q *QemuSystem) Start() (err error) { +func (q *System) Start() (err error) { rand.Seed(time.Now().UnixNano()) // Are you sure? q.sshAddrPort = getFreeAddrPort() hostfwd := fmt.Sprintf("hostfwd=tcp:%s-:22", q.sshAddrPort) @@ -213,11 +214,11 @@ func (q *QemuSystem) Start() (err error) { qemuArgs = append(qemuArgs, "-initrd", q.kernel.InitrdPath) } - if (q.arch == X86_64 || q.arch == I386) && kvmExists() { + if (q.arch == X86x64 || q.arch == X86x32) && kvmExists() { qemuArgs = append(qemuArgs, "-enable-kvm") } - if q.arch == X86_64 && runtime.GOOS == "darwin" { + if q.arch == X86x64 && runtime.GOOS == "darwin" { qemuArgs = append(qemuArgs, "-accel", "hvf", "-cpu", "host") } @@ -270,7 +271,7 @@ func (q *QemuSystem) Start() (err error) { } // Stop qemu process -func (q *QemuSystem) Stop() { +func (q *System) Stop() { // 1 00/01 01 01 SOH (Ctrl-A) START OF HEADING fmt.Fprintf(q.pipe.stdin, "%cx", 1) // wait for die @@ -282,7 +283,7 @@ func (q *QemuSystem) Stop() { } } -func (q QemuSystem) ssh(user string) (client *ssh.Client, err error) { +func (q System) ssh(user string) (client *ssh.Client, err error) { cfg := &ssh.ClientConfig{ User: user, HostKeyCallback: ssh.InsecureIgnoreHostKey(), @@ -293,7 +294,7 @@ func (q QemuSystem) ssh(user string) (client *ssh.Client, err error) { } // Command executes shell commands on qemu system -func (q QemuSystem) Command(user, cmd string) (output string, err error) { +func (q System) Command(user, cmd string) (output string, err error) { client, err := q.ssh(user) if err != nil { return @@ -311,7 +312,7 @@ func (q QemuSystem) Command(user, cmd string) (output string, err error) { } // AsyncCommand executes command on qemu system but does not wait for exit -func (q QemuSystem) AsyncCommand(user, cmd string) (err error) { +func (q System) AsyncCommand(user, cmd string) (err error) { client, err := q.ssh(user) if err != nil { return @@ -328,7 +329,7 @@ func (q QemuSystem) AsyncCommand(user, cmd string) (err error) { } // CopyFile is copy file from local machine to remote through ssh/scp -func (q *QemuSystem) CopyFile(user, localPath, remotePath string) (err error) { +func (q *System) CopyFile(user, localPath, remotePath string) (err error) { addrPort := strings.Split(q.sshAddrPort, ":") addr := addrPort[0] port := addrPort[1] @@ -346,7 +347,7 @@ func (q *QemuSystem) CopyFile(user, localPath, remotePath string) (err error) { } // CopyAndInsmod copy kernel module to temporary file on qemu then insmod it -func (q *QemuSystem) CopyAndInsmod(localKoPath string) (output string, err error) { +func (q *System) CopyAndInsmod(localKoPath string) (output string, err error) { remoteKoPath := fmt.Sprintf("/tmp/module_%d.ko", rand.Int()) err = q.CopyFile("root", localKoPath, remoteKoPath) if err != nil { @@ -357,7 +358,7 @@ func (q *QemuSystem) CopyAndInsmod(localKoPath string) (output string, err error } // CopyAndRun is copy local file to qemu vm then run it -func (q *QemuSystem) CopyAndRun(user, path string) (output string, err error) { +func (q *System) CopyAndRun(user, path string) (output string, err error) { remotePath := fmt.Sprintf("/tmp/executable_%d", rand.Int()) err = q.CopyFile(user, path, remotePath) if err != nil { @@ -368,28 +369,28 @@ func (q *QemuSystem) CopyAndRun(user, path string) (output string, err error) { } // Debug is for enable qemu debug and set hostname and port for listen -func (q *QemuSystem) Debug(conn string) { +func (q *System) Debug(conn string) { q.debug = true q.gdb = conn } // SetKASLR is changing KASLR state through kernel boot args -func (q *QemuSystem) SetKASLR(state bool) { +func (q *System) SetKASLR(state bool) { q.noKASLR = !state } // SetSMEP is changing SMEP state through kernel boot args -func (q *QemuSystem) SetSMEP(state bool) { +func (q *System) SetSMEP(state bool) { q.noSMEP = !state } // SetSMAP is changing SMAP state through kernel boot args -func (q *QemuSystem) SetSMAP(state bool) { +func (q *System) SetSMAP(state bool) { q.noSMAP = !state } -// GetSshCommand returns command for connect to qemu machine over ssh -func (q QemuSystem) GetSshCommand() (cmd string) { +// GetSSHCommand returns command for connect to qemu machine over ssh +func (q System) GetSSHCommand() (cmd string) { addrPort := strings.Split(q.sshAddrPort, ":") addr := addrPort[0] port := addrPort[1] diff --git a/qemu/qemu-kernel_test.go b/qemu/qemu-kernel_test.go index 31bf47c..76ffaf3 100644 --- a/qemu/qemu-kernel_test.go +++ b/qemu/qemu-kernel_test.go @@ -20,37 +20,37 @@ func init() { rand.Seed(time.Now().UnixNano()) } -func TestQemuSystemNew_InvalidKernelPath(t *testing.T) { +func TestSystemNew_InvalidKernelPath(t *testing.T) { kernel := Kernel{Name: "Invalid", KernelPath: "/invalid/path"} - if _, err := NewQemuSystem(X86_64, kernel, "/bin/sh"); err == nil { + if _, err := NewSystem(X86_64, kernel, "/bin/sh"); err == nil { t.Fatal(err) } } -func TestQemuSystemNew_InvalidQemuArch(t *testing.T) { +func TestSystemNew_InvalidQemuArch(t *testing.T) { kernel := Kernel{Name: "Valid path", KernelPath: testConfigVmlinuz} - if _, err := NewQemuSystem(unsupported, kernel, "/bin/sh"); err == nil { + if _, err := NewSystem(unsupported, kernel, "/bin/sh"); err == nil { t.Fatal(err) } } -func TestQemuSystemNew_InvalidQemuDrivePath(t *testing.T) { +func TestSystemNew_InvalidQemuDrivePath(t *testing.T) { kernel := Kernel{Name: "Valid path", KernelPath: testConfigVmlinuz} - if _, err := NewQemuSystem(X86_64, kernel, "/invalid/path"); err == nil { + if _, err := NewSystem(X86_64, kernel, "/invalid/path"); err == nil { t.Fatal(err) } } -func TestQemuSystemNew(t *testing.T) { +func TestSystemNew(t *testing.T) { kernel := Kernel{Name: "Valid path", KernelPath: testConfigVmlinuz} - if _, err := NewQemuSystem(X86_64, kernel, "/bin/sh"); err != nil { + if _, err := NewSystem(X86_64, kernel, "/bin/sh"); err != nil { t.Fatal(err) } } -func TestQemuSystemStart(t *testing.T) { +func TestSystemStart(t *testing.T) { kernel := Kernel{Name: "Test kernel", KernelPath: testConfigVmlinuz} - q, err := NewQemuSystem(X86_64, kernel, "/bin/sh") + q, err := NewSystem(X86_64, kernel, "/bin/sh") if err != nil { t.Fatal(err) } @@ -71,10 +71,10 @@ func TestGetFreeAddrPort(t *testing.T) { ln.Close() } -func TestQemuSystemStart_Timeout(t *testing.T) { +func TestSystemStart_Timeout(t *testing.T) { t.Parallel() kernel := Kernel{Name: "Test kernel", KernelPath: testConfigVmlinuz} - q, err := NewQemuSystem(X86_64, kernel, "/bin/sh") + q, err := NewSystem(X86_64, kernel, "/bin/sh") if err != nil { t.Fatal(err) } @@ -96,14 +96,14 @@ func TestQemuSystemStart_Timeout(t *testing.T) { } } -func startTestQemu(t *testing.T, timeout time.Duration) (q *QemuSystem, err error) { +func startTestQemu(t *testing.T, timeout time.Duration) (q *System, err error) { t.Parallel() kernel := Kernel{ Name: "Test kernel", KernelPath: testConfigVmlinuz, InitrdPath: testConfigInitrd, } - q, err = NewQemuSystem(X86_64, kernel, testConfigRootfs) + q, err = NewSystem(X86_64, kernel, testConfigRootfs) if err != nil { return } @@ -119,7 +119,7 @@ func startTestQemu(t *testing.T, timeout time.Duration) (q *QemuSystem, err erro return } -func TestQemuSystemCommand(t *testing.T) { +func TestSystemCommand(t *testing.T) { q, err := startTestQemu(t, 0) if err != nil { t.Fatal(err) @@ -149,7 +149,7 @@ func TestQemuSystemCommand(t *testing.T) { } } -func TestQemuSystemCopyFile(t *testing.T) { +func TestSystemCopyFile(t *testing.T) { q, err := startTestQemu(t, 0) if err != nil { t.Fatal(err) @@ -182,7 +182,7 @@ func TestQemuSystemCopyFile(t *testing.T) { } } -func TestQemuSystemCopyAndRun(t *testing.T) { +func TestSystemCopyAndRun(t *testing.T) { q, err := startTestQemu(t, 0) if err != nil { t.Fatal(err) @@ -216,7 +216,7 @@ func TestQemuSystemCopyAndRun(t *testing.T) { } } -func TestQemuSystemCopyAndInsmod(t *testing.T) { +func TestSystemCopyAndInsmod(t *testing.T) { q, err := startTestQemu(t, 0) if err != nil { t.Fatal(err) @@ -243,7 +243,7 @@ func TestQemuSystemCopyAndInsmod(t *testing.T) { } } -func TestQemuSystemKernelPanic(t *testing.T) { +func TestSystemKernelPanic(t *testing.T) { q, err := startTestQemu(t, 5*time.Minute) if err != nil { t.Fatal(err) @@ -278,7 +278,7 @@ func TestQemuSystemKernelPanic(t *testing.T) { } } -func TestQemuSystemRun(t *testing.T) { +func TestSystemRun(t *testing.T) { q, err := startTestQemu(t, 0) if err != nil { t.Fatal(err) @@ -313,13 +313,13 @@ func openedPort(port int) bool { return true } -func TestQemuSystemDebug(t *testing.T) { +func TestSystemDebug(t *testing.T) { t.Parallel() kernel := Kernel{ KernelPath: testConfigVmlinuz, InitrdPath: testConfigInitrd, } - q, err := NewQemuSystem(X86_64, kernel, testConfigRootfs) + q, err := NewSystem(X86_64, kernel, testConfigRootfs) if err != nil { return }