cmd: add alvis

Signed-off-by: Xe Iaso <me@xeiaso.net>

6 files changed, 543 insertions, 0 deletions

diff --git a/cmd/alvis/fly.go b/cmd/alvis/fly.go
new file mode 100644
index 0000000..b079bbc
--- /dev/null
+++ b/cmd/alvis/fly.go
@@ -0,0 +1,16 @@
+package main
+
+import (
+	"fmt"
+	"log/slog"
+)
+
+type flySlogger struct{}
+
+func (flySlogger) Debug(v ...any) {
+	slog.Debug("fly logs", "vals", fmt.Sprint(v...))
+}
+
+func (flySlogger) Debugf(format string, v ...any) {
+	slog.Debug("fly logs", "vals", fmt.Sprintf(format, v...))
+}
diff --git a/cmd/alvis/incidents/.gitignore b/cmd/alvis/incidents/.gitignore
new file mode 100644
index 0000000..94a2dd1
--- /dev/null
+++ b/cmd/alvis/incidents/.gitignore
@@ -0,0 +1 @@
+*.json
\ No newline at end of file
diff --git a/cmd/alvis/main.go b/cmd/alvis/main.go
new file mode 100644
index 0000000..ac6268c
--- /dev/null
+++ b/cmd/alvis/main.go
@@ -0,0 +1,422 @@
+package main
+
+import (
+	"context"
+	"embed"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"log"
+	"log/slog"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	flyapi "github.com/superfly/flyctl/api"
+	"within.website/x/internal"
+	"within.website/x/internal/yeet"
+	"within.website/x/web/openai/chatgpt"
+)
+
+var (
+	flyAPIBaseURL = flag.String("fly-base-url", "https://api.fly.io", "Fly API base URL")
+	flyToken      = flag.String("fly-token", "", "Fly API token")
+	openAIToken   = flag.String("openai-token", "", "OpenAI API token")
+	openAIModel   = flag.String("openai-model", "gpt-3.5-turbo-16k", "OpenAI model to use")
+
+	//go:embed playbooks/*
+	playbooks embed.FS
+)
+
+func main() {
+	internal.HandleStartup()
+
+	if *openAIToken == "" {
+		log.Fatal("must provide --openai-token")
+	}
+
+	if *flyToken == "" {
+		log.Fatal("must provide --fly-token")
+	}
+
+	cli := chatgpt.NewClient(*openAIToken)
+
+	flyapi.SetBaseURL(*flyAPIBaseURL)
+	fly := flyapi.NewClient(*flyToken, "alvis", "devel", flySlogger{})
+
+	var playbook Playbook
+	data, err := playbooks.ReadFile("playbooks/xe-pronouns-healthcheck.json")
+	if err != nil {
+		log.Fatal(err)
+	}
+	if err := json.Unmarshal(data, &playbook); err != nil {
+		log.Fatal(err)
+	}
+
+	page := Page{
+		Service: "xe-pronouns",
+		Details: "health check failed for fly app xe-pronouns",
+		ID:      "P0001",
+	}
+
+	incident := NewIncident(cli, fly, playbook, page)
+
+	for {
+		if err := incident.Step(); err != nil {
+			log.Fatal(err)
+		}
+
+		if !incident.FinishedAt.IsZero() {
+			break
+		}
+
+		if !incident.EscalatedAt.IsZero() {
+			break
+		}
+	}
+
+	if err := incident.Summarize(); err != nil {
+		log.Fatal(err)
+	}
+
+	fout, err := os.Create(fmt.Sprintf("incidents/incident-%s.json", incident.Page.ID))
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer fout.Close()
+
+	enc := json.NewEncoder(fout)
+	enc.SetIndent("", "  ")
+	if err := enc.Encode(incident); err != nil {
+		log.Fatal(err)
+	}
+}
+
+type restartAppArgs struct {
+	App    string `json:"app"`
+	Reason string `json:"reason"`
+}
+
+func (r restartAppArgs) LogValue() slog.Value {
+	return slog.GroupValue(
+		slog.String("app", r.App),
+		slog.String("reason", r.Reason),
+	)
+}
+
+type waitArgs struct {
+	DurationMinutes int `json:"duration"`
+}
+
+func (w waitArgs) LogValue() slog.Value {
+	return slog.GroupValue(
+		slog.Int("duration", w.DurationMinutes),
+	)
+}
+
+type closeArgs struct {
+	Reason string `json:"reason"`
+}
+
+func (c closeArgs) LogValue() slog.Value {
+	return slog.GroupValue(
+		slog.String("reason", c.Reason),
+	)
+}
+
+func runcmd(cmdName string, args ...string) (string, error) {
+	ctx := context.Background()
+
+	slog.Info("running command", "cmd", cmdName, "args", args)
+
+	result, err := yeet.Output(ctx, cmdName, args...)
+	if err != nil {
+		return "", err
+	}
+
+	return result, nil
+}
+
+type Playbook struct {
+	Meta struct {
+		Service   string `json:"service"`
+		Condition string `json:"condition"`
+	} `json:"meta"`
+	Details        string `json:"details"`
+	HealthCheckURL string `json:"health_check_url"`
+}
+
+func (p Playbook) LogValue() slog.Value {
+	return slog.GroupValue(
+		slog.String("service", p.Meta.Service),
+		slog.String("condition", p.Meta.Condition),
+	)
+}
+
+type Page struct {
+	Service string `json:"service"`
+	Details string `json:"details"`
+	ID      string `json:"id"`
+}
+
+func (p Page) LogValue() slog.Value {
+	return slog.GroupValue(
+		slog.String("service", p.Service),
+		slog.String("details", p.Details),
+		slog.String("id", p.ID),
+	)
+}
+
+type Annotation struct {
+	Time time.Time `json:"time"`
+	Text string    `json:"text"`
+}
+
+func (a Annotation) LogValue() slog.Value {
+	return slog.GroupValue(
+		slog.Time("time", a.Time),
+		slog.String("text", a.Text),
+	)
+}
+
+type Incident struct {
+	Playbook    Playbook          `json:"playbook"`
+	Page        Page              `json:"page"`
+	StartedAt   time.Time         `json:"started_at"`
+	FinishedAt  time.Time         `json:"finished_at,omitempty"`
+	EscalatedAt time.Time         `json:"escalated_at,omitempty"`
+	Messages    []chatgpt.Message `json:"messages,omitempty"`
+	Annotations []Annotation      `json:"annotations,omitempty"`
+	CloseReason string            `json:"close_reason,omitempty"`
+	Summary     string            `json:"summary,omitempty"`
+
+	cli chatgpt.Client `json:"-"`
+	fly *flyapi.Client `json:"-"`
+}
+
+func NewIncident(cli chatgpt.Client, fly *flyapi.Client, playbook Playbook, page Page) *Incident {
+	messages := []chatgpt.Message{
+		{
+			Role:    "system",
+			Content: basePrompt + "\n\n" + playbook.Details,
+		},
+		{
+			Role:    "user",
+			Content: fmt.Sprintf("Error: %s", page.Details),
+		},
+	}
+
+	return &Incident{
+		Playbook:  playbook,
+		Page:      page,
+		StartedAt: time.Now(),
+		Messages:  messages,
+		cli:       cli,
+		fly:       fly,
+	}
+}
+
+func (i *Incident) LogValue() slog.Value {
+	return slog.GroupValue(
+		slog.String("service", i.Page.Service),
+		slog.String("details", i.Page.Details),
+		slog.String("id", i.Page.ID),
+		slog.Time("started_at", i.StartedAt),
+		slog.Int("message_count", len(i.Messages)),
+	)
+}
+
+func (i *Incident) Step() error {
+	ctx := context.Background()
+
+	resp, err := i.cli.Complete(ctx, chatgpt.Request{
+		Model:     *openAIModel,
+		Messages:  i.Messages,
+		Functions: functions,
+	})
+	if err != nil {
+		return err
+	}
+
+	msg := resp.Choices[0].Message
+
+	i.Messages = append(i.Messages, msg)
+
+	slog.Info("got response", "message", msg.Content, "function", msg.FunctionCall)
+
+	if resp.Choices[0].Message.FunctionCall != nil {
+		if err := i.ExecFunction(ctx, msg); err != nil {
+			return err
+		}
+	} else {
+		slog.Info("incident note", "incident", i, "note", msg.Content)
+	}
+
+	return nil
+}
+
+func (i *Incident) Annotate(text string) {
+	i.Annotations = append(i.Annotations, Annotation{
+		Time: time.Now(),
+		Text: text,
+	})
+}
+
+func (i *Incident) Annotatef(format string, args ...interface{}) {
+	i.Annotate(fmt.Sprintf(format, args...))
+}
+
+func (i *Incident) Reply(reason string) {
+	i.Messages = append(i.Messages, chatgpt.Message{
+		Role:    "user",
+		Content: reason,
+	})
+
+	slog.Info("reply to incident", "incident", i, "reason", reason)
+}
+
+func (i *Incident) Replyf(format string, args ...interface{}) {
+	i.Reply(fmt.Sprintf(format, args...))
+}
+
+func (i *Incident) Close(reason string) {
+	i.FinishedAt = time.Now()
+	slog.Info("closing incident", "incident", i, "reason", reason)
+	i.Annotatef("closed incident: %s", reason)
+	i.CloseReason = reason
+}
+
+func (i *Incident) Escalate() {
+	slog.Error("escalating incident", "incident", i)
+}
+
+func (i *Incident) ExecFunction(ctx context.Context, msg chatgpt.Message) error {
+	switch msg.FunctionCall.Name {
+	case "restart_fly_app":
+		var args restartAppArgs
+		if err := json.Unmarshal([]byte(msg.FunctionCall.Arguments), &args); err != nil {
+			return err
+		}
+
+		i.Annotatef("restarting app %s: %s", args.App, args.Reason)
+
+		slog.Info("got restart_fly_app", "args", args)
+
+		if _, err := i.fly.RestartApp(ctx, args.App); err != nil {
+			i.Annotatef("error restarting app: %s", err)
+			i.Replyf("error restarting app: %s", err)
+			return nil
+		}
+
+		i.Replyf("restarted app %s successfully", args.App)
+		i.Annotate("restarted app successfully")
+
+		time.Sleep(5 * time.Second)
+
+	case "perform_health_check":
+		log.Println("performing health check")
+		ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
+		defer cancel()
+
+		if err := performFunctionFiveTimesWithDelay(ctx, i.Playbook.HealthCheckURL); err != nil {
+			i.Replyf("health check failed:\n\n%s", err)
+		} else {
+			i.Reply("health check passed")
+		}
+
+	case "wait":
+		var args waitArgs
+		if err := json.Unmarshal([]byte(msg.FunctionCall.Arguments), &args); err != nil {
+			return err
+		}
+		slog.Info("got wait", "args", args)
+
+		i.Annotatef("waiting %d minutes", args.DurationMinutes)
+		time.Sleep(time.Duration(args.DurationMinutes) * time.Minute)
+		i.Annotate("done!")
+
+	case "close":
+		var args closeArgs
+		if err := json.Unmarshal([]byte(msg.FunctionCall.Arguments), &args); err != nil {
+			return err
+		}
+		slog.Info("got close", "args", args)
+		i.Close(args.Reason)
+
+	case "escalate":
+		i.Escalate()
+	}
+
+	return nil
+}
+
+func (i *Incident) Summarize() error {
+	var sb strings.Builder
+
+	fmt.Fprintf(&sb, "Incident %s\n", i.Page.ID)
+	fmt.Fprintf(&sb, "Service: %s\n", i.Page.Service)
+	fmt.Fprintf(&sb, "Details: %s\n", i.Page.Details)
+	fmt.Fprintf(&sb, "Started at: %s\n", i.StartedAt)
+	fmt.Fprintf(&sb, "Finished at: %s\n", i.FinishedAt)
+	fmt.Fprintf(&sb, "Close reason: %s\n", i.CloseReason)
+
+	for _, ann := range i.Annotations {
+		fmt.Fprintf(&sb, "%s: %s\n", ann.Time.Format(time.Kitchen), ann.Text)
+	}
+
+	resp, err := i.cli.Complete(context.Background(), chatgpt.Request{
+		Model: *openAIModel,
+		Messages: []chatgpt.Message{
+			{
+				Role:    "system",
+				Content: summaryPrompt,
+			},
+			{
+				Role:    "user",
+				Content: sb.String(),
+			},
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	i.Summary = resp.Choices[0].Message.Content
+
+	return nil
+}
+
+func performFunctionFiveTimesWithDelay(ctx context.Context, url string) error {
+	passCount := 0
+	var errs []error
+	for i := 0; i < 5; i++ {
+		if err := healthCheck(ctx, url); err == nil {
+			passCount++
+		} else {
+			slog.Error("error performing function", "url", url, "err", err)
+			errs = append(errs, err)
+		}
+		time.Sleep(time.Second)
+	}
+
+	if passCount >= 3 {
+		return errors.Join(errs...)
+	} else {
+		return nil
+	}
+}
+
+func healthCheck(ctx context.Context, url string) error {
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("got status %s", resp.Status)
+	}
+
+	return nil
+}
diff --git a/cmd/alvis/playbooks/README.md b/cmd/alvis/playbooks/README.md
new file mode 100644
index 0000000..e778c37
--- /dev/null
+++ b/cmd/alvis/playbooks/README.md
@@ -0,0 +1,18 @@
+# Playbooks
+
+A playbook is a series of instructions that Alvis should perform on your behalf when running into issues. Playbooks are written in JSON and are executed by Alvis when the stated problem occurs.
+
+For example, consider what you would do to restart a Fly app after the health check fails. restart the app.
+
+```json
+{
+    "meta": {
+        "service": "xe-pronouns",
+        "condition": "health check failed"
+    },
+
+    "health_check_url": "https://pronouns.within.lgbt/.within/health",
+    "details": "Run your own copy of health checks.\n\nIf your health check fails, restart the app.\nIf it succeeds, close the incident.\n\nWait for one minute afte restarting the app.\nRun the health check again after restarting the app.\n\nIf it fails again, escalate to the on-call engineer."
+}
+```
+
diff --git a/cmd/alvis/playbooks/xe-pronouns-healthcheck.json b/cmd/alvis/playbooks/xe-pronouns-healthcheck.json
new file mode 100644
index 0000000..1253427
--- /dev/null
+++ b/cmd/alvis/playbooks/xe-pronouns-healthcheck.json
@@ -0,0 +1,9 @@
+{
+    "meta": {
+        "service": "xe-pronouns",
+        "condition": "health check failed"
+    },
+
+    "health_check_url": "https://pronouns.within.lgbt/.within/health",
+    "details": "Run your own copy of health checks.\n\nIf your health check fails, restart the app.\nIf it succeeds, close the incident.\n\nWait for one minute afte restarting the app.\nRun the health check again after restarting the app.\n\nIf it fails again, escalate to the on-call engineer."
+}
\ No newline at end of file
diff --git a/cmd/alvis/prompts.go b/cmd/alvis/prompts.go
new file mode 100644
index 0000000..c00d51f
--- /dev/null
+++ b/cmd/alvis/prompts.go
@@ -0,0 +1,77 @@
+package main
+
+import "within.website/x/web/openai/chatgpt"
+
+const basePrompt = `You are Alvis, a large language model systems administrator.
+
+You are tasked with responding to pagerduty pages and taking action to fix the services in question. You will call the functions given in order to fix issues.
+
+If you get a fork/exec error, escalate.
+
+Any other text you type will be sent to the pagerduty incident as a comment.`
+
+const summaryPrompt = `You are Alvis, a large language model systems administrator.
+
+You will be given the events of the incident. Use them to write your summary and postmortem.`
+
+var functions = []chatgpt.Function{
+	{
+		Name:        "restart_fly_app",
+		Description: "restarts an app running on fly.io, after running this you should run a health check",
+		Parameters: chatgpt.Param{
+			Type: "object",
+			Properties: chatgpt.Properties{
+				"app": {
+					Type:        "string",
+					Description: "the name of the app to restart",
+				},
+				"reason": {
+					Type:        "string",
+					Description: "the reason for restarting the app",
+				},
+			},
+		},
+	},
+	{
+		Name:        "perform_health_check",
+		Description: "performs a health check on an app running on fly.io",
+		Parameters: chatgpt.Param{
+			Type:       "object",
+			Properties: chatgpt.Properties{},
+		},
+	},
+	{
+		Name:        "wait",
+		Description: "waits for a given amount of time before continuing",
+		Parameters: chatgpt.Param{
+			Type: "object",
+			Properties: chatgpt.Properties{
+				"duration": {
+					Type:        "integer",
+					Description: "the duration to wait for in minutes",
+				},
+			},
+		},
+	},
+	{
+		Name:        "close",
+		Description: "closes the incident, you should only do this if you are sure the issue is resolved.",
+		Parameters: chatgpt.Param{
+			Type: "object",
+			Properties: chatgpt.Properties{
+				"reason": {
+					Type:        "string",
+					Description: "the reason for closing the incident",
+				},
+			},
+		},
+	},
+	{
+		Name:        "escalate",
+		Description: "escalates the incident to the next level of support",
+		Parameters: chatgpt.Param{
+			Type:       "object",
+			Properties: chatgpt.Properties{},
+		},
+	},
+}