From 60224243fc1b4ffc85b6b8bb228dd669b02c1e50 Mon Sep 17 00:00:00 2001 From: Ian Eyberg Date: Wed, 11 Sep 2024 07:13:59 -0700 Subject: [PATCH 1/4] handle graceful run shutdown --- cmd/cmd_run.go | 7 ++++++ cmd/run_local_instance.go | 16 +++++++++++++ provider/onprem/onprem_instance.go | 37 ++++-------------------------- provider/onprem/onprem_volume.go | 3 ++- qemu/qemu_unix.go | 29 +++++++++++++++++++++++ qemu/qmp.go | 32 ++++++++++++++++++++++++++ 6 files changed, 90 insertions(+), 34 deletions(-) create mode 100644 qemu/qmp.go diff --git a/cmd/cmd_run.go b/cmd/cmd_run.go index abd49806..5c20a8a6 100644 --- a/cmd/cmd_run.go +++ b/cmd/cmd_run.go @@ -28,6 +28,8 @@ func RunCommand() *cobra.Command { PersistNightlyCommandFlags(persistentFlags) PersistNanosVersionCommandFlags(persistentFlags) + persistentFlags.Bool("qmp", false, "qmp [local only]") + return cmdRun } @@ -61,6 +63,11 @@ func runCommandHandler(cmd *cobra.Command, args []string) { exitWithError(err.Error()) } + qmp, _ := cmd.Flags().GetBool("qmp") + if qmp { + c.RunConfig.QMP = true + } + if c.Mounts != nil { err = onprem.AddVirtfsShares(c) if err != nil { diff --git a/cmd/run_local_instance.go b/cmd/run_local_instance.go index 208b8917..27ead260 100644 --- a/cmd/run_local_instance.go +++ b/cmd/run_local_instance.go @@ -2,6 +2,8 @@ package cmd import ( "fmt" + // "os" + // "os/signal" "github.com/nanovms/ops/lepton" "github.com/nanovms/ops/network" @@ -51,6 +53,20 @@ func RunLocalInstance(c *types.Config) (err error) { c.RunConfig.Kernel = c.Kernel + if c.RunConfig.QMP { + c.RunConfig.Mgmt = qemu.GenMgmtPort() + } + + // this prob needs to happen in the context of the qemu run itself. + /* ch := make(chan os.Signal, 1) + signal.Notify(ch, os.Interrupt) + go func() { + for sig := range ch { + fmt.Println("caught %v - halting vm", sig) + } + }() + */ + fmt.Printf("booting %s ...\n", c.RunConfig.ImageName) hypervisor.Start(&c.RunConfig) diff --git a/provider/onprem/onprem_instance.go b/provider/onprem/onprem_instance.go index 57b9765d..f50d3d55 100644 --- a/provider/onprem/onprem_instance.go +++ b/provider/onprem/onprem_instance.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "math/rand" "net" "os" "os/exec" @@ -32,13 +31,6 @@ func (p *OnPrem) CreateInstancePID(ctx *lepton.Context) (string, error) { return p.createInstance(ctx) } -// genMgmtPort should generate mgmt before launching instance and -// persist in instance metadata -func genMgmtPort() string { - dd := rand.Int31n(10000) + 40000 - return strconv.Itoa(int(dd)) -} - func (p *OnPrem) createInstance(ctx *lepton.Context) (string, error) { c := ctx.Config() @@ -106,7 +98,7 @@ func (p *OnPrem) createInstance(ctx *lepton.Context) (string, error) { c.RunConfig.ImageName = imgpath c.RunConfig.Background = true - c.RunConfig.Mgmt = genMgmtPort() + c.RunConfig.Mgmt = qemu.GenMgmtPort() err := hypervisor.Start(&c.RunConfig) if err != nil { @@ -686,31 +678,10 @@ func (p *OnPrem) StartInstance(ctx *lepton.Context, instancename string) error { `{ "execute": "cont" }`, } - executeQMP(commands, last) + qemu.ExecuteQMP(commands, last) return nil } -func executeQMP(commands []string, last string) { - c, err := net.Dial("tcp", "localhost:"+last) - if err != nil { - fmt.Println(err) - } - defer c.Close() - - for i := 0; i < len(commands); i++ { - _, err := c.Write([]byte(commands[i] + "\n")) - if err != nil { - fmt.Println(err) - } - received := make([]byte, 1024) - _, err = c.Read(received) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - } -} - type qmpResponse struct { qmpReturn `json:"return"` } @@ -779,7 +750,7 @@ func (p *OnPrem) RebootInstance(ctx *lepton.Context, instancename string) error `{ "execute": "system_reset" }`, } - executeQMP(commands, last) + qemu.ExecuteQMP(commands, last) return nil } @@ -798,7 +769,7 @@ func (p *OnPrem) StopInstance(ctx *lepton.Context, instancename string) error { `{ "execute": "stop" }`, } - executeQMP(commands, last) + qemu.ExecuteQMP(commands, last) return nil } diff --git a/provider/onprem/onprem_volume.go b/provider/onprem/onprem_volume.go index 078d75b5..8a6f8390 100644 --- a/provider/onprem/onprem_volume.go +++ b/provider/onprem/onprem_volume.go @@ -11,6 +11,7 @@ import ( "github.com/nanovms/ops/lepton" "github.com/nanovms/ops/log" + "github.com/nanovms/ops/qemu" "github.com/nanovms/ops/types" ) @@ -109,7 +110,7 @@ func (op *OnPrem) AttachVolume(ctx *lepton.Context, instanceName string, volumeN deviceAddCmd, } - executeQMP(commands, last) + qemu.ExecuteQMP(commands, last) return nil } diff --git a/qemu/qemu_unix.go b/qemu/qemu_unix.go index 455f71cb..a4cec92c 100644 --- a/qemu/qemu_unix.go +++ b/qemu/qemu_unix.go @@ -7,6 +7,9 @@ import ( "bufio" "bytes" "crypto/rand" + mrand "math/rand" + "time" + "errors" "fmt" "os" @@ -26,6 +29,13 @@ import ( "github.com/nanovms/ops/types" ) +// GenMgmtPort should generate mgmt before launching instance and +// persist in instance metadata +func GenMgmtPort() string { + dd := mrand.Int31n(10000) + 40000 + return strconv.Itoa(int(dd)) +} + func qemuBaseCommand() string { x86 := "qemu-system-x86_64" arm := "qemu-system-aarch64" @@ -65,6 +75,7 @@ type qemu struct { flags []string kernel string atExitHook string + mgmt string // not sure why we have this as string.. } func newQemu() Hypervisor { @@ -77,6 +88,19 @@ func (q *qemu) SetKernel(kernel string) { func (q *qemu) Stop() { if q.cmd != nil { + fmt.Println("death") + + commands := []string{ + `{ "execute": "qmp_capabilities" }`, + `{ "execute": "stop" }`, + } + + if q.mgmt != "" { + fmt.Println("shutting vm down") + ExecuteQMP(commands, q.mgmt) + time.Sleep(2 * time.Second) + } + if err := q.cmd.Process.Kill(); err != nil { log.Error(err) } @@ -117,6 +141,7 @@ func (q *qemu) Command(rconfig *types.RunConfig) *exec.Cmd { syscall.SIGQUIT) go func(chan os.Signal) { <-c + fmt.Println("caught a kill") q.Stop() }(c) @@ -133,6 +158,10 @@ func (q *qemu) Start(rconfig *types.RunConfig) error { q.cmd.Stderr = os.Stderr } + if rconfig.QMP { + q.mgmt = rconfig.Mgmt + } + if rconfig.Background { err := q.cmd.Start() if err != nil { diff --git a/qemu/qmp.go b/qemu/qmp.go new file mode 100644 index 00000000..3e0d6f02 --- /dev/null +++ b/qemu/qmp.go @@ -0,0 +1,32 @@ +//go:build linux || darwin || freebsd +// +build linux darwin freebsd + +package qemu + +import ( + "fmt" + "net" + "os" +) + +// turn me private and have the actual commands be exportable +func ExecuteQMP(commands []string, last string) { + c, err := net.Dial("tcp", "localhost:"+last) + if err != nil { + fmt.Println(err) + } + defer c.Close() + + for i := 0; i < len(commands); i++ { + _, err := c.Write([]byte(commands[i] + "\n")) + if err != nil { + fmt.Println(err) + } + received := make([]byte, 1024) + _, err = c.Read(received) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + } +} From 35f2835ce88539efc3531d3c74346130a05419a9 Mon Sep 17 00:00:00 2001 From: Ian Eyberg Date: Wed, 11 Sep 2024 07:40:11 -0700 Subject: [PATCH 2/4] . --- qemu/qemu_unix.go | 9 ++++++++- qemu/qmp.go | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/qemu/qemu_unix.go b/qemu/qemu_unix.go index a4cec92c..e4921dfa 100644 --- a/qemu/qemu_unix.go +++ b/qemu/qemu_unix.go @@ -90,14 +90,18 @@ func (q *qemu) Stop() { if q.cmd != nil { fmt.Println("death") + // {"execute": "system_powerdown"} + commands := []string{ `{ "execute": "qmp_capabilities" }`, - `{ "execute": "stop" }`, + `{ "execute": "system_powerdown" }`, + // `{ "execute": "stop" }`, } if q.mgmt != "" { fmt.Println("shutting vm down") ExecuteQMP(commands, q.mgmt) + fmt.Println("done") time.Sleep(2 * time.Second) } @@ -168,6 +172,9 @@ func (q *qemu) Start(rconfig *types.RunConfig) error { log.Error(err) } } else { + q.cmd.SysProcAttr = &syscall.SysProcAttr{ + Setpgid: true, + } if err := q.cmd.Run(); err != nil { log.Error(err) } diff --git a/qemu/qmp.go b/qemu/qmp.go index 3e0d6f02..7c75a5fa 100644 --- a/qemu/qmp.go +++ b/qemu/qmp.go @@ -22,6 +22,7 @@ func ExecuteQMP(commands []string, last string) { if err != nil { fmt.Println(err) } + fmt.Printf("wrote %s\n", commands[i]) received := make([]byte, 1024) _, err = c.Read(received) if err != nil { From ebf4ead104c27885d938af3c42dc2cd0008ae5a0 Mon Sep 17 00:00:00 2001 From: Ian Eyberg Date: Sun, 15 Sep 2024 10:53:59 -0700 Subject: [PATCH 3/4] . --- cmd/run_local_instance.go | 12 ------------ qemu/qemu_unix.go | 7 ------- qemu/qmp.go | 1 - 3 files changed, 20 deletions(-) diff --git a/cmd/run_local_instance.go b/cmd/run_local_instance.go index 27ead260..4448efbb 100644 --- a/cmd/run_local_instance.go +++ b/cmd/run_local_instance.go @@ -2,8 +2,6 @@ package cmd import ( "fmt" - // "os" - // "os/signal" "github.com/nanovms/ops/lepton" "github.com/nanovms/ops/network" @@ -57,16 +55,6 @@ func RunLocalInstance(c *types.Config) (err error) { c.RunConfig.Mgmt = qemu.GenMgmtPort() } - // this prob needs to happen in the context of the qemu run itself. - /* ch := make(chan os.Signal, 1) - signal.Notify(ch, os.Interrupt) - go func() { - for sig := range ch { - fmt.Println("caught %v - halting vm", sig) - } - }() - */ - fmt.Printf("booting %s ...\n", c.RunConfig.ImageName) hypervisor.Start(&c.RunConfig) diff --git a/qemu/qemu_unix.go b/qemu/qemu_unix.go index e4921dfa..4ea51886 100644 --- a/qemu/qemu_unix.go +++ b/qemu/qemu_unix.go @@ -88,20 +88,14 @@ func (q *qemu) SetKernel(kernel string) { func (q *qemu) Stop() { if q.cmd != nil { - fmt.Println("death") - - // {"execute": "system_powerdown"} commands := []string{ `{ "execute": "qmp_capabilities" }`, `{ "execute": "system_powerdown" }`, - // `{ "execute": "stop" }`, } if q.mgmt != "" { - fmt.Println("shutting vm down") ExecuteQMP(commands, q.mgmt) - fmt.Println("done") time.Sleep(2 * time.Second) } @@ -145,7 +139,6 @@ func (q *qemu) Command(rconfig *types.RunConfig) *exec.Cmd { syscall.SIGQUIT) go func(chan os.Signal) { <-c - fmt.Println("caught a kill") q.Stop() }(c) diff --git a/qemu/qmp.go b/qemu/qmp.go index 7c75a5fa..3e0d6f02 100644 --- a/qemu/qmp.go +++ b/qemu/qmp.go @@ -22,7 +22,6 @@ func ExecuteQMP(commands []string, last string) { if err != nil { fmt.Println(err) } - fmt.Printf("wrote %s\n", commands[i]) received := make([]byte, 1024) _, err = c.Read(received) if err != nil { From 159fcc805dcf8d62c85fc6296e8c9d1186e9229b Mon Sep 17 00:00:00 2001 From: Ian Eyberg Date: Sun, 15 Sep 2024 11:00:30 -0700 Subject: [PATCH 4/4] . --- qemu/qmp.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qemu/qmp.go b/qemu/qmp.go index 3e0d6f02..1656b2ec 100644 --- a/qemu/qmp.go +++ b/qemu/qmp.go @@ -9,7 +9,8 @@ import ( "os" ) -// turn me private and have the actual commands be exportable +// ExecuteQMP ships a list of commands to the QMP to execute. +// TODO: turn me private and have the actual commands be exportable. func ExecuteQMP(commands []string, last string) { c, err := net.Dial("tcp", "localhost:"+last) if err != nil {