From 24f5fb57f3e36ff2f6150468c4cb10c4f5bb282c Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Sat, 17 Jun 2023 13:38:38 -0400 Subject: cmd/marabot: move emoji copying to another binary Signed-off-by: Xe Iaso --- cmd/marabot/copyemoji/main.go | 110 ++++++++++++++++++++++++++++ cmd/marabot/discord.go | 6 +- cmd/marabot/main.go | 17 +++-- cmd/marabot/schema.sql | 13 ++++ web/openai/chatgpt/chatgpt.go | 30 +++++++- web/revolt/client.go | 167 ++++++++++++++++++++++++++++++++++++------ web/revolt/emoji.go | 70 ++++++++++++++++++ web/revolt/http.go | 49 ++++++++++++- web/revolt/websocket.go | 4 + 9 files changed, 427 insertions(+), 39 deletions(-) create mode 100644 cmd/marabot/copyemoji/main.go diff --git a/cmd/marabot/copyemoji/main.go b/cmd/marabot/copyemoji/main.go new file mode 100644 index 0000000..13d240d --- /dev/null +++ b/cmd/marabot/copyemoji/main.go @@ -0,0 +1,110 @@ +package main + +import ( + "context" + "database/sql" + "flag" + "io" + "net/http" + + _ "modernc.org/sqlite" + "within.website/ln" + "within.website/ln/opname" + "within.website/x/internal" + "within.website/x/web/revolt" +) + +var ( + dbFile = flag.String("db-file", "../marabot.db", "Path to the database file") + furryholeDiscord = flag.String("furryhole-discord", "192289762302754817", "Discord channel ID for furryhole") + furryholeRevolt = flag.String("furryhole-revolt", "01FEXZ1XPWMEJXMF836FP16HB8", "Revolt channel ID for furryhole") + revoltEmail = flag.String("revolt-email", "", "Email for Revolt") + revoltPassword = flag.String("revolt-password", "", "Password for Revolt") + revoltAPIServer = flag.String("revolt-api-server", "https://api.revolt.chat", "API server for Revolt") + revoltWebsocketServer = flag.String("revolt-ws-server", "wss://ws.revolt.chat", "Websocket server for Revolt") +) + +func main() { + internal.HandleStartup() + + ctx, cancel := context.WithCancel(opname.With(context.Background(), "marabot.copyemoji")) + defer cancel() + + ln.Log(ctx, ln.Action("starting up")) + + db, err := sql.Open("sqlite", *dbFile) + if err != nil { + ln.FatalErr(ctx, err, ln.Action("opening sqlite database")) + } + defer db.Close() + + cli, err := revolt.NewWithEndpoint("", *revoltAPIServer, *revoltWebsocketServer) + if err != nil { + ln.FatalErr(ctx, err, ln.Action("creating revolt client")) + } + cli.SelfBot = &revolt.SelfBot{ + Email: *revoltEmail, + Password: *revoltPassword, + } + + if err := cli.Auth(ctx, "marabot-copyemoji"); err != nil { + ln.FatalErr(ctx, err, ln.Action("authing to revolt")) + } + + rows, err := db.QueryContext(ctx, "SELECT id, name, url FROM discord_emoji WHERE guild_id = ? AND id NOT IN ( SELECT discord_id FROM revolt_discord_emoji )", furryholeDiscord) + if err != nil { + ln.FatalErr(ctx, err, ln.Action("querying discord_emoji")) + } + defer rows.Close() + + for rows.Next() { + var id, name, url string + if err := rows.Scan(&id, &name, &url); err != nil { + ln.FatalErr(ctx, err, ln.Action("scanning discord_emoji")) + } + + resp, err := http.Get(url) + if err != nil { + ln.Error(ctx, err, ln.F{"url": url}, ln.Action("downloading emoji")) + continue + } + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) + if err != nil { + ln.Error(ctx, err, ln.F{"url": url}, ln.Action("reading emoji body")) + continue + } + + uploadID, err := cli.Upload(ctx, "emojis", name+".webp", data) + if err != nil { + ln.Error(ctx, err, ln.F{"url": url}, ln.Action("uploading emoji")) + continue + } + + emoji, err := cli.CreateEmoji(ctx, uploadID, revolt.CreateEmoji{ + Name: name, + NSFW: false, + Parent: revolt.Parent{ + ID: *furryholeRevolt, + Type: "Server", + }, + }) + if err != nil { + ln.Error(ctx, err, ln.F{"url": url}, ln.Action("creating emoji on revolt")) + continue + } + + if _, err := db.ExecContext(ctx, "INSERT INTO revolt_emoji (id, server_id, name, url) VALUES (?, ?, ?, ?)", emoji.ID, furryholeRevolt, emoji.Name, emoji.URL); err != nil { + ln.Error(ctx, err, ln.F{"id": emoji.ID}, ln.Action("inserting emoji")) + continue + } + + if _, err := db.ExecContext(ctx, "INSERT INTO revolt_discord_emoji(revolt_id, discord_id, name) VALUES (?, ?, ?)", emoji.ID, id, name); err != nil { + ln.Error(ctx, err, ln.F{"id": emoji.ID}, ln.Action("inserting emoji join record")) + continue + } + + ln.Log(ctx, ln.Info("created emoji"), ln.F{"id": emoji.ID, "name": emoji.Name}) + } +} diff --git a/cmd/marabot/discord.go b/cmd/marabot/discord.go index 77a9331..d9495df 100644 --- a/cmd/marabot/discord.go +++ b/cmd/marabot/discord.go @@ -12,7 +12,7 @@ import ( func importDiscordData(ctx context.Context, db *sql.DB, dg *discordgo.Session) error { ctx = opname.With(ctx, "import-discord-data") - channels, err := dg.GuildChannels(furryholeDiscord) + channels, err := dg.GuildChannels(*furryholeDiscord) if err != nil { return err } @@ -24,7 +24,7 @@ func importDiscordData(ctx context.Context, db *sql.DB, dg *discordgo.Session) e } } - roles, err := dg.GuildRoles(furryholeDiscord) + roles, err := dg.GuildRoles(*furryholeDiscord) if err != nil { return err } @@ -37,7 +37,7 @@ func importDiscordData(ctx context.Context, db *sql.DB, dg *discordgo.Session) e } // https://cdn.discordapp.com/emojis/664686615616290816.webp?size=240&quality=lossless - emoji, err := dg.GuildEmojis(furryholeDiscord) + emoji, err := dg.GuildEmojis(*furryholeDiscord) if err != nil { return err } diff --git a/cmd/marabot/main.go b/cmd/marabot/main.go index 6f2f072..3c5511a 100644 --- a/cmd/marabot/main.go +++ b/cmd/marabot/main.go @@ -29,15 +29,16 @@ var ( tsAuthkey = flag.String("ts-authkey", "", "Tailscale authkey") tsHostname = flag.String("ts-hostname", "", "Tailscale hostname") + adminDiscordUser = flag.String("admin-discord-user", "", "Discord user ID of the admin") + adminRevoltUser = flag.String("admin-revolt-user", "", "Revolt user ID of the admin") + + furryholeDiscord = flag.String("furryhole-discord", "192289762302754817", "Discord channel ID for furryhole") + furryholeRevolt = flag.String("furryhole-revolt", "01FEXZ1XPWMEJXMF836FP16HB8", "Revolt channel ID for furryhole") + //go:embed schema.sql dbSchema string ) -const ( - furryholeDiscord = "192289762302754817" - furryholeRevolt = "01H2VRKJFPYPEAE438B6JRFSCP" -) - func main() { internal.HandleStartup() @@ -63,7 +64,10 @@ func main() { go NewIRCBot(ctx, db, ircmsgs) // Init a new client. - client := revolt.NewWithEndpoint(*revoltToken, *revoltAPIServer, *revoltWebsocketServer) + client, err := revolt.NewWithEndpoint(*revoltToken, *revoltAPIServer, *revoltWebsocketServer) + if err != nil { + ln.FatalErr(ctx, err, ln.Action("creating revolt client")) + } mr := &MaraRevolt{ cli: client, @@ -185,5 +189,6 @@ func (mr *MaraRevolt) MessageCreate(ctx context.Context, msg *revolt.Message) er return err } } + return nil } diff --git a/cmd/marabot/schema.sql b/cmd/marabot/schema.sql index 15d884d..05a7f56 100644 --- a/cmd/marabot/schema.sql +++ b/cmd/marabot/schema.sql @@ -77,3 +77,16 @@ CREATE TABLE IF NOT EXISTS irc_messages ( content TEXT NOT NULL, tags TEXT NOT NULL ); + +CREATE TABLE IF NOT EXISTS revolt_emoji ( + id TEXT PRIMARY KEY, + server_id TEXT NOT NULL, + name TEXT NOT NULL, + url TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS revolt_discord_emoji ( + revolt_id TEXT NOT NULL, + discord_id TEXT NOT NULL, + name TEXT NOT NULL +); \ No newline at end of file diff --git a/web/openai/chatgpt/chatgpt.go b/web/openai/chatgpt/chatgpt.go index d0b70dc..aab4ce7 100644 --- a/web/openai/chatgpt/chatgpt.go +++ b/web/openai/chatgpt/chatgpt.go @@ -13,13 +13,35 @@ import ( ) type Request struct { - Model string `json:"model"` - Messages []Message `json:"messages"` + Model string `json:"model"` + Messages []Message `json:"messages"` + Functions []Function `json:"functions,omitempty"` + FunctionCall string `json:"function_call"` +} + +type Function struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + Parameters []Param `json:"parameters"` +} + +type Param struct { + Type string `json:"type"` + Description string `json:"description,omitempty"` + Enum []string `json:"enum,omitempty"` + Properties map[string]Param `json:"properties,omitempty"` + Required []string `json:"required,omitempty"` } type Message struct { - Role string `json:"role"` - Content string `json:"content"` + Role string `json:"role"` + Content string `json:"content"` + FunctionCall *Funcall `json:"function_call"` +} + +type Funcall struct { + Name string `json:"name"` + Arguments json.RawMessage `json:"arguments"` } func (m Message) ProxyFormat() string { diff --git a/web/revolt/client.go b/web/revolt/client.go index 60cc001..f83f8ea 100644 --- a/web/revolt/client.go +++ b/web/revolt/client.go @@ -1,51 +1,129 @@ package revolt import ( + "bytes" "context" "encoding/json" "fmt" + "io" + "mime/multipart" "net/http" + "net/textproto" "time" "github.com/sacOO7/gowebsocket" ) +type RevoltSettings struct { + Revolt string `json:"revolt"` + Features RevoltFeatures `json:"features"` + Ws string `json:"ws"` + App string `json:"app"` + Vapid string `json:"vapid"` + Build RevoltBuild `json:"build"` +} + +type RevoltCaptcha struct { + Enabled bool `json:"enabled"` + Key string `json:"key"` +} + +type RevoltAutumn struct { + Enabled bool `json:"enabled"` + URL string `json:"url"` +} + +type RevoltJanuary struct { + Enabled bool `json:"enabled"` + URL string `json:"url"` +} + +type RevoltVoso struct { + Enabled bool `json:"enabled"` + URL string `json:"url"` + Ws string `json:"ws"` +} + +type RevoltFeatures struct { + Captcha RevoltCaptcha `json:"captcha"` + Email bool `json:"email"` + InviteOnly bool `json:"invite_only"` + Autumn RevoltAutumn `json:"autumn"` + January RevoltJanuary `json:"january"` + Voso RevoltVoso `json:"voso"` +} + +type RevoltBuild struct { + CommitSha string `json:"commit_sha"` + CommitTimestamp string `json:"commit_timestamp"` + Semver string `json:"semver"` + OriginURL string `json:"origin_url"` + Timestamp string `json:"timestamp"` +} + // New creates a new client with the default Revolt server details. // // Use NewWithEndpoint to create a client with a custom endpoint. -func New(token string) *Client { - return &Client{ - HTTP: &http.Client{}, - Token: token, - BaseURL: "https://api.revolt.chat", - WSURL: "wss://ws.revolt.chat", - Ticker: time.NewTicker(3 * time.Second), +func New(token string) (*Client, error) { + var settings RevoltSettings + + resp, err := http.Get("https://api.revolt.chat/") + if err != nil { + return nil, fmt.Errorf("failed to get Revolt settings: %w", err) + } + defer resp.Body.Close() + + if err := json.NewDecoder(resp.Body).Decode(&settings); err != nil { + return nil, fmt.Errorf("failed to decode Revolt settings: %w", err) } + + return &Client{ + HTTP: &http.Client{}, + Token: token, + BaseURL: "https://api.revolt.chat", + WSURL: "wss://ws.revolt.chat", + Ticker: time.NewTicker(3 * time.Second), + Settings: settings, + }, nil } // NewWithEndpoint creates a new client with a custom Revolt endpoint. // // You can use this to test the library against an arbirtary Revolt server. -func NewWithEndpoint(token, baseURL, wsURL string) *Client { - return &Client{ - HTTP: &http.Client{}, - Token: token, - BaseURL: baseURL, - WSURL: wsURL, - Ticker: time.NewTicker(3 * time.Second), +func NewWithEndpoint(token, baseURL, wsURL string) (*Client, error) { + var settings RevoltSettings + + resp, err := http.Get(baseURL) + if err != nil { + return nil, fmt.Errorf("failed to get Revolt settings: %w", err) } + defer resp.Body.Close() + + if err := json.NewDecoder(resp.Body).Decode(&settings); err != nil { + return nil, fmt.Errorf("failed to decode Revolt settings: %w", err) + } + + return &Client{ + HTTP: &http.Client{}, + Token: token, + BaseURL: baseURL, + WSURL: wsURL, + Ticker: time.NewTicker(3 * time.Second), + Settings: settings, + }, nil } // Client struct. type Client struct { - SelfBot *SelfBot - Token string - Socket gowebsocket.Socket - HTTP *http.Client - Cache *Cache - BaseURL string - WSURL string - Ticker *time.Ticker + SelfBot *SelfBot + Token string + Socket gowebsocket.Socket + HTTP *http.Client + Cache *Cache + BaseURL string + WSURL string + Ticker *time.Ticker + Settings RevoltSettings } // Self bot struct. @@ -288,3 +366,48 @@ func (c *Client) FetchBot(ctx context.Context, id string) (*Bot, error) { err = json.Unmarshal(resp, bot) return bot.Bot, err } + +// Upload a file to Revolt using a multi-part form. +func (c *Client) Upload(ctx context.Context, tag, fname string, data []byte) (string, error) { + type response struct { + ID string `json:"id"` + } + + // Create a new multi-part form. + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + + header := textproto.MIMEHeader{} + header.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, fname)) + header.Set("Content-Type", http.DetectContentType(data)) + + // Add the file to the form. + part, err := writer.CreatePart(header) + if err != nil { + return "", err + } + _, err = io.Copy(part, bytes.NewReader(data)) + if err != nil { + return "", err + } + + // Close the form and set the content type. + err = writer.Close() + if err != nil { + return "", err + } + + // Send the request to the server. + resp, err := c.RequestWithPathAndContentType(ctx, "POST", c.Settings.Features.Autumn.URL+"/"+tag, writer.FormDataContentType(), body.Bytes()) + if err != nil { + return "", err + } + + // Parse the response and return the ID of the uploaded file. + var res response + err = json.Unmarshal(resp, &res) + if err != nil { + return "", err + } + return res.ID, nil +} diff --git a/web/revolt/emoji.go b/web/revolt/emoji.go index 57f37ac..57e5324 100644 --- a/web/revolt/emoji.go +++ b/web/revolt/emoji.go @@ -1,6 +1,76 @@ package revolt +import ( + "context" + "encoding/json" +) + // Emoji struct. type Emoji struct { + ID string `json:"_id"` + Name string `json:"name"` + Animated bool `json:"animated"` + NSFW bool `json:"nsfw"` + CreatorID string `json:"creator_id"` + URL string `json:"url"` +} + +// Emoji grabs a single emoji by URL. +func (c *Client) Emoji(ctx context.Context, id string) (*Emoji, error) { + emoji := &Emoji{} + + resp, err := c.Request(ctx, "GET", "/custom/emoji/"+id, []byte{}) + + if err != nil { + return nil, err + } + + err = json.Unmarshal(resp, emoji) + if err != nil { + return nil, err + } + + emoji.URL = c.Settings.Features.Autumn.URL + "/emojis/" + emoji.ID + + return emoji, err +} + +type CreateEmoji struct { + Name string `json:"name"` + NSFW bool `json:"nsfw"` + Parent Parent `json:"parent"` +} + +type Parent struct { + Type string `json:"type"` + ID string `json:"id"` +} + +// CreateEmoji creates a new emoji. +func (c *Client) CreateEmoji(ctx context.Context, uploadID string, emoji CreateEmoji) (*Emoji, error) { + data, err := json.Marshal(emoji) + if err != nil { + return nil, err + } + + var emojiData Emoji + + resp, err := c.Request(ctx, "PUT", "/custom/emoji/"+uploadID, data) + if err != nil { + return nil, err + } + + if err := json.Unmarshal(resp, &emojiData); err != nil { + return nil, err + } + + emojiData.URL = c.Settings.Features.Autumn.URL + "/emojis/" + emojiData.ID + + return &emojiData, nil +} +// DeleteEmoji deletes an emoji. +func (c *Client) DeleteEmoji(ctx context.Context, id string) error { + _, err := c.Request(ctx, "DELETE", "/custom/emoji/"+id, []byte{}) + return err } diff --git a/web/revolt/http.go b/web/revolt/http.go index b7293d2..377eb62 100644 --- a/web/revolt/http.go +++ b/web/revolt/http.go @@ -9,19 +9,18 @@ import ( "within.website/x/web" ) -// Send http request -func (c Client) Request(ctx context.Context, method, path string, data []byte) ([]byte, error) { +func (c *Client) RequestWithPathAndContentType(ctx context.Context, method, path, contentType string, data []byte) ([]byte, error) { reqBody := bytes.NewBuffer(data) <-c.Ticker.C // Prepare request - req, err := http.NewRequestWithContext(ctx, method, c.BaseURL+path, reqBody) + req, err := http.NewRequestWithContext(ctx, method, path, reqBody) if err != nil { return []byte{}, err } - req.Header.Set("content-type", "application/json") + req.Header.Set("content-type", contentType) // Set auth headers if c.SelfBot == nil { @@ -38,15 +37,57 @@ func (c Client) Request(ctx context.Context, method, path string, data []byte) ( } defer resp.Body.Close() + + if !(resp.StatusCode >= 200 && resp.StatusCode < 300) { + return []byte{}, web.NewError(200, resp) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return []byte{}, err + } + + return body, nil +} + +// Send http request +func (c Client) Request(ctx context.Context, method, path string, data []byte) ([]byte, error) { + reqBody := bytes.NewBuffer(data) + + <-c.Ticker.C + + // Prepare request + req, err := http.NewRequestWithContext(ctx, method, c.BaseURL+path, reqBody) + if err != nil { + return []byte{}, err + } + + req.Header.Set("content-type", "application/json") + + // Set auth headers + if c.SelfBot == nil { + req.Header.Set("x-bot-token", c.Token) + } else if c.SelfBot.SessionToken != "" { + req.Header.Set("x-session-token", c.SelfBot.SessionToken) + } + + // Send request + resp, err := c.HTTP.Do(req) if err != nil { return []byte{}, err } + defer resp.Body.Close() + if !(resp.StatusCode >= 200 && resp.StatusCode < 300) { return []byte{}, web.NewError(200, resp) } + body, err := io.ReadAll(resp.Body) + if err != nil { + return []byte{}, err + } + return body, nil } diff --git a/web/revolt/websocket.go b/web/revolt/websocket.go index 84fbbcd..2e2ad74 100644 --- a/web/revolt/websocket.go +++ b/web/revolt/websocket.go @@ -17,6 +17,10 @@ func (c *Client) Connect(ctx context.Context, handler Handler) { defer t.Stop() go func(ctx context.Context) { + if err := c.doWebsocket(ctx, c.Token, c.WSURL, handler); err != nil { + ln.Error(ctx, err, ln.Info("websocket error, retrying")) + } + for { select { case <-ctx.Done(): -- cgit v1.2.3