1
0
out-of-tree/distro.go

186 lines
3.9 KiB
Go
Raw Normal View History

2023-05-11 20:08:08 +00:00
package main
import (
"context"
2023-05-17 12:33:44 +00:00
"fmt"
"os"
"path/filepath"
"regexp"
"time"
"github.com/cavaliergopher/grab/v3"
2023-05-17 12:33:44 +00:00
"github.com/davecgh/go-spew/spew"
"github.com/remeh/sizedwaitgroup"
2023-05-11 20:08:08 +00:00
"github.com/rs/zerolog/log"
"code.dumpstack.io/tools/out-of-tree/cache"
"code.dumpstack.io/tools/out-of-tree/config"
2023-05-11 20:08:08 +00:00
"code.dumpstack.io/tools/out-of-tree/distro/debian"
"code.dumpstack.io/tools/out-of-tree/distro/debian/snapshot"
"code.dumpstack.io/tools/out-of-tree/fs"
2023-05-11 20:08:08 +00:00
)
type DistroCmd struct {
Debian DebianCmd `cmd:""`
}
2023-05-11 20:08:08 +00:00
type DebianCmd struct {
Cache DebianCacheCmd `cmd:"" help:"populate cache"`
Fetch DebianFetchCmd `cmd:"" help:"download deb packages"`
2023-05-17 12:33:44 +00:00
Regex string `help:"match deb pkg names by regex" default:".*"`
2023-05-11 20:08:08 +00:00
}
type DebianCacheCmd struct {
Path string `help:"path to cache"`
2023-05-12 00:07:51 +00:00
Refetch int `help:"days before refetch versions without deb package" default:"7"`
2023-05-17 12:33:44 +00:00
Dump bool `help:"dump cache"`
2023-05-11 20:08:08 +00:00
}
2023-05-17 12:33:44 +00:00
func (cmd *DebianCacheCmd) Run(dcmd *DebianCmd) (err error) {
if cmd.Path != "" {
debian.CachePath = cmd.Path
}
2023-05-14 12:37:45 +00:00
debian.RefetchDays = cmd.Refetch
2023-05-11 20:08:08 +00:00
2023-05-12 15:00:50 +00:00
log.Info().Msg("Fetching kernels...")
2023-05-17 12:33:44 +00:00
kernels, err := debian.GetKernels()
2023-05-11 20:08:08 +00:00
if err != nil {
2023-05-12 15:00:50 +00:00
log.Error().Err(err).Msg("")
2023-05-11 20:08:08 +00:00
return
}
2023-05-17 12:33:44 +00:00
if cmd.Dump {
re, err := regexp.Compile(dcmd.Regex)
if err != nil {
log.Fatal().Err(err).Msg("regex")
}
for _, kernel := range kernels {
if !re.MatchString(kernel.Image.Deb.Name) {
continue
}
fmt.Println(spew.Sdump(kernel))
}
}
2023-05-12 15:00:50 +00:00
log.Info().Msg("Success")
2023-05-11 20:08:08 +00:00
return
}
type DebianFetchCmd struct {
2023-05-17 12:33:44 +00:00
Path string `help:"path to download directory" type:"existingdir" default:"./"`
IgnoreMirror bool `help:"ignore check if packages on the mirror"`
Max int `help:"do not download more than X" default:"100500"`
Threads int `help:"parallel download threads" default:"8"`
Timeout time.Duration `help:"timeout for each download" default:"1m"`
swg sizedwaitgroup.SizedWaitGroup
hasResults bool
}
func (cmd *DebianFetchCmd) fetch(pkg snapshot.Package) {
flog := log.With().
Str("pkg", pkg.Deb.Name).
Logger()
defer cmd.swg.Done()
if !cmd.IgnoreMirror {
flog.Debug().Msg("check mirror")
found, _ := cache.PackageURL(config.Debian, pkg.Deb.URL)
if found {
2023-05-17 11:25:07 +00:00
flog.Debug().Msg("found on the mirror")
return
}
}
target := filepath.Join(cmd.Path, filepath.Base(pkg.Deb.URL))
if fs.PathExists(target) {
flog.Debug().Msg("already exists")
return
}
tmp, err := os.MkdirTemp(cmd.Path, "tmp-")
if err != nil {
flog.Fatal().Err(err).Msg("mkdir")
return
}
defer os.RemoveAll(tmp)
flog.Info().Msg("fetch")
flog.Debug().Msg(pkg.Deb.URL)
ctx, cancel := context.WithTimeout(context.Background(), cmd.Timeout)
defer cancel()
req, err := grab.NewRequest(tmp, pkg.Deb.URL)
if err != nil {
flog.Warn().Err(err).Msg("cannot create request")
return
}
req = req.WithContext(ctx)
resp := grab.DefaultClient.Do(req)
if err := resp.Err(); err != nil {
flog.Warn().Err(err).Msg("request cancelled")
return
}
err = os.Rename(resp.Filename, target)
if err != nil {
flog.Fatal().Err(err).Msg("mv")
}
cmd.hasResults = true
cmd.Max--
}
2023-05-17 12:33:44 +00:00
func (cmd *DebianFetchCmd) Run(dcmd *DebianCmd) (err error) {
re, err := regexp.Compile(dcmd.Regex)
2023-05-15 11:50:54 +00:00
if err != nil {
2023-05-17 12:33:44 +00:00
log.Fatal().Err(err).Msg("regex")
2023-05-15 11:50:54 +00:00
}
2023-05-17 12:33:44 +00:00
log.Info().Msg("will not download packages that exist on the mirror")
log.Info().Msg("use --ignore-mirror if you really need it")
kernels, err := debian.GetKernels()
if err != nil {
log.Error().Err(err).Msg("")
return
}
var packages []snapshot.Package
for _, kernel := range kernels {
for _, pkg := range kernel.Packages() {
if !re.MatchString(pkg.Deb.Name) {
continue
}
packages = append(packages, pkg)
}
}
cmd.swg = sizedwaitgroup.New(cmd.Threads)
for _, pkg := range packages {
if cmd.Max <= 0 {
break
}
cmd.swg.Add()
go cmd.fetch(pkg)
}
cmd.swg.Wait()
if !cmd.hasResults {
2023-05-15 14:26:38 +00:00
log.Fatal().Msg("no packages found to download")
}
return
}