tools/out-of-tree
1
0
Fork 0
Code Releases Activity

Refactor [2]

Browse Source
This commit is contained in:
dump_stack()
2019-08-17 09:05:06 +00:00
parent e0c0d3a072
commit 986a6f55e0
8 changed files with 73 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
CHANGELOG.md
View File

@ -12,33 +12,22 @@
base on `.out-of-tree.toml` definitions) and `pew` (automated runs)
and allows to specify a maximum number of runs per each supported
kernel in module/exploit definition.
- New command `genall` -- generate all kernels for specified
distro/version.
- All logs stores in sqlite3 database. Implemented specific commands
for making simple queries and export data to markdown and json.
- Implemented success rate calculation for previous runs.
- Save of build results supported by parameter `--dist` for `pew`.
- Support for generating kernels info from host system.
- Support for build on host.
- Support for custom kernels.
- Now debugging environment is automatically looking for debug kernel
on the host system.
- Added ability to enable/disable kaslr/smep/smap for debugging by
command line flags.
- New parameter `--threads=N` is added for `pew` and allows to specify
maximum number of threads that will be used for parallel
build/run/test.
- Tagging for runs. Tags write to log and can be used for statistics.
### Changed
@ -46,13 +35,10 @@
- Now if there's no base image found — out-of-tree will try to use an
image from closest previous version, e.g. image from Ubuntu 18.04
for Ubuntu 18.10.
- Kernel modules tests will not be failed if there are no tests
exists.
- Now *out-of-tree* will return negative error code if at least one of
the stage was failed.
- Project is switch to use Go modules.
### Removed
@ -63,9 +49,7 @@
### Fixed
- Command `timeout` is not required anymore.
- Errors is more meaningful.
- Temporary files is moved to `~/.out-of-tree/tmp/` to avoid docker mounting issues on some systems.
## [0.2.0] - 2019-12-01

22
db.go
View File

@ -17,16 +17,16 @@ import (
)
// Change on ANY database update
const CURRENT_DATABASE_VERSION = 2
const currentDatabaseVersion = 2
const VERSION_FIELD = "db_version"
const versionField = "db_version"
type logEntry struct {
ID int
Tag string
Timestamp time.Time
qemu.QemuSystem
qemu.System
config.Artifact
config.KernelInfo
phasesResult
@ -111,7 +111,7 @@ func metaSetValue(db *sql.DB, key, value string) (err error) {
}
func getVersion(db *sql.DB) (version int, err error) {
s, err := metaGetValue(db, VERSION_FIELD)
s, err := metaGetValue(db, versionField)
if err != nil {
return
}
@ -120,7 +120,7 @@ func getVersion(db *sql.DB) (version int, err error) {
return
}
func addToLog(db *sql.DB, q *qemu.QemuSystem, ka config.Artifact,
func addToLog(db *sql.DB, q *qemu.System, ka config.Artifact,
ki config.KernelInfo, res *phasesResult, tag string) (err error) {
stmt, err := db.Prepare("INSERT INTO log (name, type, tag, " +
@ -276,15 +276,15 @@ func openDatabase(path string) (db *sql.DB, err error) {
db.SetMaxOpenConns(1)
exists, _ := metaChkValue(db, VERSION_FIELD)
exists, _ := metaChkValue(db, versionField)
if !exists {
err = createSchema(db)
if err != nil {
return
}
err = metaSetValue(db, VERSION_FIELD,
strconv.Itoa(CURRENT_DATABASE_VERSION))
err = metaSetValue(db, versionField,
strconv.Itoa(currentDatabaseVersion))
return
}
@ -299,7 +299,7 @@ func openDatabase(path string) (db *sql.DB, err error) {
return
}
err = metaSetValue(db, VERSION_FIELD, "2")
err = metaSetValue(db, versionField, "2")
if err != nil {
return
}
@ -307,9 +307,9 @@ func openDatabase(path string) (db *sql.DB, err error) {
version = 2
}
if version != CURRENT_DATABASE_VERSION {
if version != currentDatabaseVersion {
err = fmt.Errorf("Database is not supported (%d instead of %d)",
version, CURRENT_DATABASE_VERSION)
version, currentDatabaseVersion)
return
}

10
debug.go
View File

@ -41,7 +41,7 @@ func firstSupported(kcfg config.KernelConfig, ka config.Artifact,
return
}
func handleLine(q *qemu.QemuSystem) (err error) {
func handleLine(q *qemu.System) (err error) {
fmt.Print("out-of-tree> ")
rawLine := "help"
fmt.Scanf("%s", &rawLine)
@ -64,7 +64,7 @@ func handleLine(q *qemu.QemuSystem) (err error) {
case "c", "cleanup":
q.Stdout = []byte{}
case "s", "ssh":
fmt.Println(q.GetSshCommand())
fmt.Println(q.GetSSHCommand())
case "q", "quit":
return errors.New("end of session")
default:
@ -73,7 +73,7 @@ func handleLine(q *qemu.QemuSystem) (err error) {
return
}
func interactive(q *qemu.QemuSystem) (err error) {
func interactive(q *qemu.System) (err error) {
for {
err = handleLine(q)
if err != nil {
@ -100,7 +100,7 @@ func debugHandler(kcfg config.KernelConfig, workPath, kernRegex, gdb string,
}
kernel := qemu.Kernel{KernelPath: ki.KernelPath, InitrdPath: ki.InitrdPath}
q, err := qemu.NewQemuSystem(qemu.X86_64, kernel, ki.RootFS)
q, err := qemu.NewSystem(qemu.X86x64, kernel, ki.RootFS)
if err != nil {
return
}
@ -155,7 +155,7 @@ func debugHandler(kcfg config.KernelConfig, workPath, kernRegex, gdb string,
coloredRemoteFile := aurora.BgGreen(aurora.Black(remoteFile))
fmt.Printf("[*] build result copied to %s\n", coloredRemoteFile)
fmt.Printf("\n%s\n", q.GetSshCommand())
fmt.Printf("\n%s\n", q.GetSSHCommand())
fmt.Printf("gdb %s -ex 'target remote %s'\n\n", ki.VmlinuxPath, gdb)
// TODO set substitute-path /build/.../linux-... /path/to/linux-source

2
main.go
View File

@ -136,7 +136,7 @@ func main() {
pewTest := pewTestFlag.String()
pewDistFlag := pewCommand.Flag("dist", "Build result path")
pewDist := pewDistFlag.Default(PATH_DEV_NULL).String()
pewDist := pewDistFlag.Default(pathDevNull).String()
pewThreadsFlag := pewCommand.Flag("threads", "Build result path")
pewThreads := pewThreadsFlag.Default(strconv.Itoa(runtime.NumCPU())).Int()

14
pew.go
View File

@ -28,7 +28,7 @@ import (
var somethingFailed = false
const PATH_DEV_NULL = "/dev/null"
const pathDevNull = "/dev/null"
func dockerRun(timeout time.Duration, container, workdir, command string) (
output string, err error) {
@ -101,7 +101,7 @@ func build(tmp string, ka config.Artifact, ki config.KernelInfo,
return
}
func cleanDmesg(q *qemu.QemuSystem) (err error) {
func cleanDmesg(q *qemu.System) (err error) {
start := time.Now()
for {
_, err = q.Command("root", "dmesg -c")
@ -118,7 +118,7 @@ func cleanDmesg(q *qemu.QemuSystem) (err error) {
return
}
func testKernelModule(q *qemu.QemuSystem, ka config.Artifact,
func testKernelModule(q *qemu.System, ka config.Artifact,
test string) (output string, err error) {
output, err = q.Command("root", test)
@ -126,7 +126,7 @@ func testKernelModule(q *qemu.QemuSystem, ka config.Artifact,
return
}
func testKernelExploit(q *qemu.QemuSystem, ka config.Artifact,
func testKernelExploit(q *qemu.System, ka config.Artifact,
test, exploit string) (output string, err error) {
output, err = q.Command("user", "chmod +x "+exploit)
@ -188,7 +188,7 @@ func copyFile(sourcePath, destinationPath string) (err error) {
return destinationFile.Close()
}
func dumpResult(q *qemu.QemuSystem, ka config.Artifact, ki config.KernelInfo,
func dumpResult(q *qemu.System, ka config.Artifact, ki config.KernelInfo,
res *phasesResult, dist, tag, binary string, db *sql.DB) {
// TODO merge (problem is it's not 100% same) with log.go:logLogEntry
@ -226,7 +226,7 @@ func dumpResult(q *qemu.QemuSystem, ka config.Artifact, ki config.KernelInfo,
log.Println("[db] addToLog (", ka, ") error:", err)
}
if binary == "" && dist != PATH_DEV_NULL {
if binary == "" && dist != pathDevNull {
err = os.MkdirAll(dist, os.ModePerm)
if err != nil {
log.Println("os.MkdirAll (", ka, ") error:", err)
@ -253,7 +253,7 @@ func whatever(swg *sizedwaitgroup.SizedWaitGroup, ka config.Artifact,
defer swg.Done()
kernel := qemu.Kernel{KernelPath: ki.KernelPath, InitrdPath: ki.InitrdPath}
q, err := qemu.NewQemuSystem(qemu.X86_64, kernel, ki.RootFS)
q, err := qemu.NewSystem(qemu.X86x64, kernel, ki.RootFS)
if err != nil {
log.Println("Qemu creation error:", err)
return

2
qemu/README.md
View File

@ -52,7 +52,7 @@ Minimal example:
KernelPath: "/path/to/vmlinuz",
InitrdPath: "/path/to/initrd", // if required
}
q, err := qemu.NewQemuSystem(qemu.X86_64, kernel, "/path/to/qcow2")
q, err := qemu.NewSystem(qemu.X86_64, kernel, "/path/to/qcow2")
if err != nil {
log.Fatal(err)
}

51
qemu/qemu-kernel.go
View File

@ -44,9 +44,10 @@ func readUntilEOF(pipe io.ReadCloser, buf *[]byte) (err error) {
type arch string
const (
// X86_64 must be exactly same as in qemu-system-${HERE}
X86_64 arch = "x86_64"
I386 = "i386"
// X86x64 is the qemu-system-x86_64
X86x64 arch = "x86_64"
// X86x32 is the qemu-system-i386
X86x32 = "i386"
// TODO add other
unsupported = "unsupported" // for test purposes
@ -59,8 +60,8 @@ type Kernel struct {
InitrdPath string
}
// QemuSystem describe qemu parameters and runned process
type QemuSystem struct {
// System describe qemu parameters and runned process
type System struct {
arch arch
kernel Kernel
drivePath string
@ -98,12 +99,12 @@ type QemuSystem struct {
exitErr error
}
// NewQemuSystem constructor
func NewQemuSystem(arch arch, kernel Kernel, drivePath string) (q *QemuSystem, err error) {
// NewSystem constructor
func NewSystem(arch arch, kernel Kernel, drivePath string) (q *System, err error) {
if _, err = exec.LookPath("qemu-system-" + string(arch)); err != nil {
return
}
q = &QemuSystem{}
q = &System{}
q.arch = arch
if _, err = os.Stat(kernel.KernelPath); err != nil {
@ -164,7 +165,7 @@ func kvmExists() bool {
return true
}
func (q *QemuSystem) panicWatcher() {
func (q *System) panicWatcher() {
for {
time.Sleep(time.Second)
if bytes.Contains(q.Stdout, []byte("Kernel panic")) {
@ -178,7 +179,7 @@ func (q *QemuSystem) panicWatcher() {
}
// Start qemu process
func (q *QemuSystem) Start() (err error) {
func (q *System) Start() (err error) {
rand.Seed(time.Now().UnixNano()) // Are you sure?
q.sshAddrPort = getFreeAddrPort()
hostfwd := fmt.Sprintf("hostfwd=tcp:%s-:22", q.sshAddrPort)
@ -213,11 +214,11 @@ func (q *QemuSystem) Start() (err error) {
qemuArgs = append(qemuArgs, "-initrd", q.kernel.InitrdPath)
}
if (q.arch == X86_64 || q.arch == I386) && kvmExists() {
if (q.arch == X86x64 || q.arch == X86x32) && kvmExists() {
qemuArgs = append(qemuArgs, "-enable-kvm")
}
if q.arch == X86_64 && runtime.GOOS == "darwin" {
if q.arch == X86x64 && runtime.GOOS == "darwin" {
qemuArgs = append(qemuArgs, "-accel", "hvf", "-cpu", "host")
}
@ -270,7 +271,7 @@ func (q *QemuSystem) Start() (err error) {
}
// Stop qemu process
func (q *QemuSystem) Stop() {
func (q *System) Stop() {
// 1 00/01 01 01 SOH (Ctrl-A) START OF HEADING
fmt.Fprintf(q.pipe.stdin, "%cx", 1)
// wait for die
@ -282,7 +283,7 @@ func (q *QemuSystem) Stop() {
}
}
func (q QemuSystem) ssh(user string) (client *ssh.Client, err error) {
func (q System) ssh(user string) (client *ssh.Client, err error) {
cfg := &ssh.ClientConfig{
User: user,
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
@ -293,7 +294,7 @@ func (q QemuSystem) ssh(user string) (client *ssh.Client, err error) {
}
// Command executes shell commands on qemu system
func (q QemuSystem) Command(user, cmd string) (output string, err error) {
func (q System) Command(user, cmd string) (output string, err error) {
client, err := q.ssh(user)
if err != nil {
return
@ -311,7 +312,7 @@ func (q QemuSystem) Command(user, cmd string) (output string, err error) {
}
// AsyncCommand executes command on qemu system but does not wait for exit
func (q QemuSystem) AsyncCommand(user, cmd string) (err error) {
func (q System) AsyncCommand(user, cmd string) (err error) {
client, err := q.ssh(user)
if err != nil {
return
@ -328,7 +329,7 @@ func (q QemuSystem) AsyncCommand(user, cmd string) (err error) {
}
// CopyFile is copy file from local machine to remote through ssh/scp
func (q *QemuSystem) CopyFile(user, localPath, remotePath string) (err error) {
func (q *System) CopyFile(user, localPath, remotePath string) (err error) {
addrPort := strings.Split(q.sshAddrPort, ":")
addr := addrPort[0]
port := addrPort[1]
@ -346,7 +347,7 @@ func (q *QemuSystem) CopyFile(user, localPath, remotePath string) (err error) {
}
// CopyAndInsmod copy kernel module to temporary file on qemu then insmod it
func (q *QemuSystem) CopyAndInsmod(localKoPath string) (output string, err error) {
func (q *System) CopyAndInsmod(localKoPath string) (output string, err error) {
remoteKoPath := fmt.Sprintf("/tmp/module_%d.ko", rand.Int())
err = q.CopyFile("root", localKoPath, remoteKoPath)
if err != nil {
@ -357,7 +358,7 @@ func (q *QemuSystem) CopyAndInsmod(localKoPath string) (output string, err error
}
// CopyAndRun is copy local file to qemu vm then run it
func (q *QemuSystem) CopyAndRun(user, path string) (output string, err error) {
func (q *System) CopyAndRun(user, path string) (output string, err error) {
remotePath := fmt.Sprintf("/tmp/executable_%d", rand.Int())
err = q.CopyFile(user, path, remotePath)
if err != nil {
@ -368,28 +369,28 @@ func (q *QemuSystem) CopyAndRun(user, path string) (output string, err error) {
}
// Debug is for enable qemu debug and set hostname and port for listen
func (q *QemuSystem) Debug(conn string) {
func (q *System) Debug(conn string) {
q.debug = true
q.gdb = conn
}
// SetKASLR is changing KASLR state through kernel boot args
func (q *QemuSystem) SetKASLR(state bool) {
func (q *System) SetKASLR(state bool) {
q.noKASLR = !state
}
// SetSMEP is changing SMEP state through kernel boot args
func (q *QemuSystem) SetSMEP(state bool) {
func (q *System) SetSMEP(state bool) {
q.noSMEP = !state
}
// SetSMAP is changing SMAP state through kernel boot args
func (q *QemuSystem) SetSMAP(state bool) {
func (q *System) SetSMAP(state bool) {
q.noSMAP = !state
}
// GetSshCommand returns command for connect to qemu machine over ssh
func (q QemuSystem) GetSshCommand() (cmd string) {
// GetSSHCommand returns command for connect to qemu machine over ssh
func (q System) GetSSHCommand() (cmd string) {
addrPort := strings.Split(q.sshAddrPort, ":")
addr := addrPort[0]
port := addrPort[1]

44
qemu/qemu-kernel_test.go
View File

@ -20,37 +20,37 @@ func init() {
rand.Seed(time.Now().UnixNano())
}
func TestQemuSystemNew_InvalidKernelPath(t *testing.T) {
func TestSystemNew_InvalidKernelPath(t *testing.T) {
kernel := Kernel{Name: "Invalid", KernelPath: "/invalid/path"}
if _, err := NewQemuSystem(X86_64, kernel, "/bin/sh"); err == nil {
if _, err := NewSystem(X86_64, kernel, "/bin/sh"); err == nil {
t.Fatal(err)
}
}
func TestQemuSystemNew_InvalidQemuArch(t *testing.T) {
func TestSystemNew_InvalidQemuArch(t *testing.T) {
kernel := Kernel{Name: "Valid path", KernelPath: testConfigVmlinuz}
if _, err := NewQemuSystem(unsupported, kernel, "/bin/sh"); err == nil {
if _, err := NewSystem(unsupported, kernel, "/bin/sh"); err == nil {
t.Fatal(err)
}
}
func TestQemuSystemNew_InvalidQemuDrivePath(t *testing.T) {
func TestSystemNew_InvalidQemuDrivePath(t *testing.T) {
kernel := Kernel{Name: "Valid path", KernelPath: testConfigVmlinuz}
if _, err := NewQemuSystem(X86_64, kernel, "/invalid/path"); err == nil {
if _, err := NewSystem(X86_64, kernel, "/invalid/path"); err == nil {
t.Fatal(err)
}
}
func TestQemuSystemNew(t *testing.T) {
func TestSystemNew(t *testing.T) {
kernel := Kernel{Name: "Valid path", KernelPath: testConfigVmlinuz}
if _, err := NewQemuSystem(X86_64, kernel, "/bin/sh"); err != nil {
if _, err := NewSystem(X86_64, kernel, "/bin/sh"); err != nil {
t.Fatal(err)
}
}
func TestQemuSystemStart(t *testing.T) {
func TestSystemStart(t *testing.T) {
kernel := Kernel{Name: "Test kernel", KernelPath: testConfigVmlinuz}
q, err := NewQemuSystem(X86_64, kernel, "/bin/sh")
q, err := NewSystem(X86_64, kernel, "/bin/sh")
if err != nil {
t.Fatal(err)
}
@ -71,10 +71,10 @@ func TestGetFreeAddrPort(t *testing.T) {
ln.Close()
}
func TestQemuSystemStart_Timeout(t *testing.T) {
func TestSystemStart_Timeout(t *testing.T) {
t.Parallel()
kernel := Kernel{Name: "Test kernel", KernelPath: testConfigVmlinuz}
q, err := NewQemuSystem(X86_64, kernel, "/bin/sh")
q, err := NewSystem(X86_64, kernel, "/bin/sh")
if err != nil {
t.Fatal(err)
}
@ -96,14 +96,14 @@ func TestQemuSystemStart_Timeout(t *testing.T) {
}
}
func startTestQemu(t *testing.T, timeout time.Duration) (q *QemuSystem, err error) {
func startTestQemu(t *testing.T, timeout time.Duration) (q *System, err error) {
t.Parallel()
kernel := Kernel{
Name: "Test kernel",
KernelPath: testConfigVmlinuz,
InitrdPath: testConfigInitrd,
}
q, err = NewQemuSystem(X86_64, kernel, testConfigRootfs)
q, err = NewSystem(X86_64, kernel, testConfigRootfs)
if err != nil {
return
}
@ -119,7 +119,7 @@ func startTestQemu(t *testing.T, timeout time.Duration) (q *QemuSystem, err erro
return
}
func TestQemuSystemCommand(t *testing.T) {
func TestSystemCommand(t *testing.T) {
q, err := startTestQemu(t, 0)
if err != nil {
t.Fatal(err)
@ -149,7 +149,7 @@ func TestQemuSystemCommand(t *testing.T) {
}
}
func TestQemuSystemCopyFile(t *testing.T) {
func TestSystemCopyFile(t *testing.T) {
q, err := startTestQemu(t, 0)
if err != nil {
t.Fatal(err)
@ -182,7 +182,7 @@ func TestQemuSystemCopyFile(t *testing.T) {
}
}
func TestQemuSystemCopyAndRun(t *testing.T) {
func TestSystemCopyAndRun(t *testing.T) {
q, err := startTestQemu(t, 0)
if err != nil {
t.Fatal(err)
@ -216,7 +216,7 @@ func TestQemuSystemCopyAndRun(t *testing.T) {
}
}
func TestQemuSystemCopyAndInsmod(t *testing.T) {
func TestSystemCopyAndInsmod(t *testing.T) {
q, err := startTestQemu(t, 0)
if err != nil {
t.Fatal(err)
@ -243,7 +243,7 @@ func TestQemuSystemCopyAndInsmod(t *testing.T) {
}
}
func TestQemuSystemKernelPanic(t *testing.T) {
func TestSystemKernelPanic(t *testing.T) {
q, err := startTestQemu(t, 5*time.Minute)
if err != nil {
t.Fatal(err)
@ -278,7 +278,7 @@ func TestQemuSystemKernelPanic(t *testing.T) {
}
}
func TestQemuSystemRun(t *testing.T) {
func TestSystemRun(t *testing.T) {
q, err := startTestQemu(t, 0)
if err != nil {
t.Fatal(err)
@ -313,13 +313,13 @@ func openedPort(port int) bool {
return true
}
func TestQemuSystemDebug(t *testing.T) {
func TestSystemDebug(t *testing.T) {
t.Parallel()
kernel := Kernel{
KernelPath: testConfigVmlinuz,
InitrdPath: testConfigInitrd,
}
q, err := NewQemuSystem(X86_64, kernel, testConfigRootfs)
q, err := NewSystem(X86_64, kernel, testConfigRootfs)
if err != nil {
return
}