1
0
out-of-tree/distro.go

186 lines
3.9 KiB
Go

package main
import (
"context"
"fmt"
"os"
"path/filepath"
"regexp"
"time"
"github.com/cavaliergopher/grab/v3"
"github.com/davecgh/go-spew/spew"
"github.com/remeh/sizedwaitgroup"
"github.com/rs/zerolog/log"
"code.dumpstack.io/tools/out-of-tree/cache"
"code.dumpstack.io/tools/out-of-tree/config"
"code.dumpstack.io/tools/out-of-tree/distro/debian"
"code.dumpstack.io/tools/out-of-tree/distro/debian/snapshot"
"code.dumpstack.io/tools/out-of-tree/fs"
)
type DistroCmd struct {
Debian DebianCmd `cmd:""`
}
type DebianCmd struct {
Cache DebianCacheCmd `cmd:"" help:"populate cache"`
Fetch DebianFetchCmd `cmd:"" help:"download deb packages"`
Regex string `help:"match deb pkg names by regex" default:".*"`
}
type DebianCacheCmd struct {
Path string `help:"path to cache"`
Refetch int `help:"days before refetch versions without deb package" default:"7"`
Dump bool `help:"dump cache"`
}
func (cmd *DebianCacheCmd) Run(dcmd *DebianCmd) (err error) {
if cmd.Path != "" {
debian.CachePath = cmd.Path
}
debian.RefetchDays = cmd.Refetch
log.Info().Msg("Fetching kernels...")
kernels, err := debian.GetKernels()
if err != nil {
log.Error().Err(err).Msg("")
return
}
if cmd.Dump {
re, err := regexp.Compile(dcmd.Regex)
if err != nil {
log.Fatal().Err(err).Msg("regex")
}
for _, kernel := range kernels {
if !re.MatchString(kernel.Image.Deb.Name) {
continue
}
fmt.Println(spew.Sdump(kernel))
}
}
log.Info().Msg("Success")
return
}
type DebianFetchCmd struct {
Path string `help:"path to download directory" type:"existingdir" default:"./"`
IgnoreMirror bool `help:"ignore check if packages on the mirror"`
Max int `help:"do not download more than X" default:"100500"`
Threads int `help:"parallel download threads" default:"8"`
Timeout time.Duration `help:"timeout for each download" default:"1m"`
swg sizedwaitgroup.SizedWaitGroup
hasResults bool
}
func (cmd *DebianFetchCmd) fetch(pkg snapshot.Package) {
flog := log.With().
Str("pkg", pkg.Deb.Name).
Logger()
defer cmd.swg.Done()
if !cmd.IgnoreMirror {
flog.Debug().Msg("check mirror")
found, _ := cache.PackageURL(config.Debian, pkg.Deb.URL)
if found {
flog.Debug().Msg("found on the mirror")
return
}
}
target := filepath.Join(cmd.Path, filepath.Base(pkg.Deb.URL))
if fs.PathExists(target) {
flog.Debug().Msg("already exists")
return
}
tmp, err := os.MkdirTemp(cmd.Path, "tmp-")
if err != nil {
flog.Fatal().Err(err).Msg("mkdir")
return
}
defer os.RemoveAll(tmp)
flog.Info().Msg("fetch")
flog.Debug().Msg(pkg.Deb.URL)
ctx, cancel := context.WithTimeout(context.Background(), cmd.Timeout)
defer cancel()
req, err := grab.NewRequest(tmp, pkg.Deb.URL)
if err != nil {
flog.Warn().Err(err).Msg("cannot create request")
return
}
req = req.WithContext(ctx)
resp := grab.DefaultClient.Do(req)
if err := resp.Err(); err != nil {
flog.Warn().Err(err).Msg("request cancelled")
return
}
err = os.Rename(resp.Filename, target)
if err != nil {
flog.Fatal().Err(err).Msg("mv")
}
cmd.hasResults = true
cmd.Max--
}
func (cmd *DebianFetchCmd) Run(dcmd *DebianCmd) (err error) {
re, err := regexp.Compile(dcmd.Regex)
if err != nil {
log.Fatal().Err(err).Msg("regex")
}
log.Info().Msg("will not download packages that exist on the mirror")
log.Info().Msg("use --ignore-mirror if you really need it")
kernels, err := debian.GetKernels()
if err != nil {
log.Error().Err(err).Msg("")
return
}
var packages []snapshot.Package
for _, kernel := range kernels {
for _, pkg := range kernel.Packages() {
if !re.MatchString(pkg.Deb.Name) {
continue
}
packages = append(packages, pkg)
}
}
cmd.swg = sizedwaitgroup.New(cmd.Threads)
for _, pkg := range packages {
if cmd.Max <= 0 {
break
}
cmd.swg.Add()
go cmd.fetch(pkg)
}
cmd.swg.Wait()
if !cmd.hasResults {
log.Fatal().Msg("no packages found to download")
}
return
}