diff options
| author | Xe Iaso <me@xeiaso.net> | 2023-08-26 14:14:51 -0400 |
|---|---|---|
| committer | Xe Iaso <me@xeiaso.net> | 2023-08-26 14:14:51 -0400 |
| commit | 81fe4e8a12b362f7de9a97210f950c388d047664 (patch) | |
| tree | d71d879f62d74e528a1338470df268669e2565be /cmd | |
| parent | 924a12ab6915b7dad147ed57c5a384c142f82c1e (diff) | |
| download | x-81fe4e8a12b362f7de9a97210f950c388d047664.tar.xz x-81fe4e8a12b362f7de9a97210f950c388d047664.zip | |
Switch from ln to slog
ln had a good run, but it's not going to last for the long term. I'm
going to standardize everything on log/slog and deprecate ln.
Closes #385
Signed-off-by: Xe Iaso <me@xeiaso.net>
Diffstat (limited to 'cmd')
27 files changed, 249 insertions, 1443 deletions
diff --git a/cmd/apeirophobia/main.go b/cmd/apeirophobia/main.go index 5ad1792..2ba6dc2 100644 --- a/cmd/apeirophobia/main.go +++ b/cmd/apeirophobia/main.go @@ -7,11 +7,11 @@ import ( "flag" "fmt" "html/template" + "log/slog" "net/http" "os" "strings" - "golang.org/x/exp/slog" _ "modernc.org/sqlite" "tailscale.com/hostinfo" "tailscale.com/tsnet" diff --git a/cmd/marabot/.gitignore b/cmd/marabot/.gitignore deleted file mode 100644 index 657f4e1..0000000 --- a/cmd/marabot/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -*.db -*.db-shm -*.db-wal -.marabot.db-litestream -*.csv diff --git a/cmd/marabot/copyemoji/main.go b/cmd/marabot/copyemoji/main.go deleted file mode 100644 index 13d240d..0000000 --- a/cmd/marabot/copyemoji/main.go +++ /dev/null @@ -1,110 +0,0 @@ -package main - -import ( - "context" - "database/sql" - "flag" - "io" - "net/http" - - _ "modernc.org/sqlite" - "within.website/ln" - "within.website/ln/opname" - "within.website/x/internal" - "within.website/x/web/revolt" -) - -var ( - dbFile = flag.String("db-file", "../marabot.db", "Path to the database file") - furryholeDiscord = flag.String("furryhole-discord", "192289762302754817", "Discord channel ID for furryhole") - furryholeRevolt = flag.String("furryhole-revolt", "01FEXZ1XPWMEJXMF836FP16HB8", "Revolt channel ID for furryhole") - revoltEmail = flag.String("revolt-email", "", "Email for Revolt") - revoltPassword = flag.String("revolt-password", "", "Password for Revolt") - revoltAPIServer = flag.String("revolt-api-server", "https://api.revolt.chat", "API server for Revolt") - revoltWebsocketServer = flag.String("revolt-ws-server", "wss://ws.revolt.chat", "Websocket server for Revolt") -) - -func main() { - internal.HandleStartup() - - ctx, cancel := context.WithCancel(opname.With(context.Background(), "marabot.copyemoji")) - defer cancel() - - ln.Log(ctx, ln.Action("starting up")) - - db, err := sql.Open("sqlite", *dbFile) - if err != nil { - ln.FatalErr(ctx, err, ln.Action("opening sqlite database")) - } - defer db.Close() - - cli, err := revolt.NewWithEndpoint("", *revoltAPIServer, *revoltWebsocketServer) - if err != nil { - ln.FatalErr(ctx, err, ln.Action("creating revolt client")) - } - cli.SelfBot = &revolt.SelfBot{ - Email: *revoltEmail, - Password: *revoltPassword, - } - - if err := cli.Auth(ctx, "marabot-copyemoji"); err != nil { - ln.FatalErr(ctx, err, ln.Action("authing to revolt")) - } - - rows, err := db.QueryContext(ctx, "SELECT id, name, url FROM discord_emoji WHERE guild_id = ? AND id NOT IN ( SELECT discord_id FROM revolt_discord_emoji )", furryholeDiscord) - if err != nil { - ln.FatalErr(ctx, err, ln.Action("querying discord_emoji")) - } - defer rows.Close() - - for rows.Next() { - var id, name, url string - if err := rows.Scan(&id, &name, &url); err != nil { - ln.FatalErr(ctx, err, ln.Action("scanning discord_emoji")) - } - - resp, err := http.Get(url) - if err != nil { - ln.Error(ctx, err, ln.F{"url": url}, ln.Action("downloading emoji")) - continue - } - defer resp.Body.Close() - - data, err := io.ReadAll(resp.Body) - if err != nil { - ln.Error(ctx, err, ln.F{"url": url}, ln.Action("reading emoji body")) - continue - } - - uploadID, err := cli.Upload(ctx, "emojis", name+".webp", data) - if err != nil { - ln.Error(ctx, err, ln.F{"url": url}, ln.Action("uploading emoji")) - continue - } - - emoji, err := cli.CreateEmoji(ctx, uploadID, revolt.CreateEmoji{ - Name: name, - NSFW: false, - Parent: revolt.Parent{ - ID: *furryholeRevolt, - Type: "Server", - }, - }) - if err != nil { - ln.Error(ctx, err, ln.F{"url": url}, ln.Action("creating emoji on revolt")) - continue - } - - if _, err := db.ExecContext(ctx, "INSERT INTO revolt_emoji (id, server_id, name, url) VALUES (?, ?, ?, ?)", emoji.ID, furryholeRevolt, emoji.Name, emoji.URL); err != nil { - ln.Error(ctx, err, ln.F{"id": emoji.ID}, ln.Action("inserting emoji")) - continue - } - - if _, err := db.ExecContext(ctx, "INSERT INTO revolt_discord_emoji(revolt_id, discord_id, name) VALUES (?, ?, ?)", emoji.ID, id, name); err != nil { - ln.Error(ctx, err, ln.F{"id": emoji.ID}, ln.Action("inserting emoji join record")) - continue - } - - ln.Log(ctx, ln.Info("created emoji"), ln.F{"id": emoji.ID, "name": emoji.Name}) - } -} diff --git a/cmd/marabot/discord.go b/cmd/marabot/discord.go deleted file mode 100644 index 74db134..0000000 --- a/cmd/marabot/discord.go +++ /dev/null @@ -1,367 +0,0 @@ -package main - -import ( - "context" - "fmt" - "path/filepath" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/bwmarrin/discordgo" - "github.com/google/uuid" - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgxpool" - "within.website/ln" - "within.website/ln/opname" -) - -func (mr *MaraRevolt) importDiscordData(ctx context.Context, db *pgxpool.Pool, dg *discordgo.Session) error { - ctx = opname.With(ctx, "import-discord-data") - - tx, err := mr.pg.Begin(ctx) - if err != nil { - return err - } - defer tx.Rollback(ctx) - - channels, err := dg.GuildChannels(*furryholeDiscord) - if err != nil { - return err - } - - for _, ch := range channels { - if _, err := tx.Exec(ctx, "INSERT INTO discord_channels (id, guild_id, name, topic, nsfw) VALUES ($1, $2, $3, $4, $5) ON CONFLICT(id) DO UPDATE SET name = EXCLUDED.name, topic = EXCLUDED.topic, nsfw = EXCLUDED.nsfw", ch.ID, ch.GuildID, ch.Name, ch.Topic, ch.NSFW); err != nil { - ln.Error(ctx, err, ln.F{"channel_name": ch.Name, "channel_id": ch.ID}) - continue - } - } - - roles, err := dg.GuildRoles(*furryholeDiscord) - if err != nil { - return err - } - - for _, role := range roles { - if _, err := tx.Exec(ctx, "INSERT INTO discord_roles (guild_id, id, name, color, hoist, position) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT(id) DO UPDATE SET name = EXCLUDED.name, color = EXCLUDED.color, position = EXCLUDED.position", furryholeDiscord, role.ID, role.Name, fmt.Sprintf("#%06x", role.Color), role.Hoist, role.Position); err != nil { - ln.Error(ctx, err, ln.Action("inserting role")) - continue - } - } - - // https://cdn.discordapp.com/emojis/664686615616290816.webp?size=240&quality=lossless - emoji, err := dg.GuildEmojis(*furryholeDiscord) - if err != nil { - return err - } - for _, emoji := range emoji { - eURL := fmt.Sprintf("https://cdn.discordapp.com/emojis/%s?size=240&quality=lossless", emoji.ID) - if _, err := tx.Exec(ctx, "INSERT INTO discord_emoji (id, guild_id, name, url) VALUES ($1, $2, $3, $4) ON CONFLICT(id) DO UPDATE SET name = EXCLUDED.name, url = EXCLUDED.url", emoji.ID, furryholeDiscord, emoji.Name, eURL); err != nil { - ln.Error(ctx, err, ln.Action("inserting emoji")) - continue - } - - if err := mr.archiveAttachment(ctx, tx, eURL, "emoji", ""); err != nil { - return err - } - } - - rows, err := tx.Query(ctx, "SELECT url, message_id FROM discord_attachments WHERE url NOT IN ( SELECT url FROM s3_uploads )") - if err == nil { - defer rows.Close() - for rows.Next() { - var url, messageID string - if err := rows.Scan(&url, &messageID); err != nil { - continue - } - - if err := mr.archiveAttachment(ctx, tx, url, "attachments", messageID); err != nil { - return err - } - } - } - - if err := tx.Commit(ctx); err != nil { - return err - } - - return nil -} - -func (mr *MaraRevolt) DiscordMessageDelete(s *discordgo.Session, m *discordgo.MessageDelete) { - ctx := opname.With(context.Background(), "marabot.discord-message-delete") - - tx, err := mr.pg.Begin(ctx) - if err != nil { - ln.Error(ctx, err) - return - } - defer tx.Rollback(ctx) - - if _, err := tx.Exec(ctx, "DELETE FROM discord_messages WHERE id = $1", m.ID); err != nil { - ln.Error(ctx, err, ln.Action("nuking deleted messages")) - } - - rows, err := tx.Query(ctx, "SELECT id FROM s3_uploads WHERE message_id = $1", m.ID) - if err != nil { - ln.Error(ctx, err) - return - } - defer rows.Close() - - for rows.Next() { - var id string - if err := rows.Scan(&id); err != nil { - ln.Error(ctx, err) - return - } - - if _, err := tx.Exec(ctx, "DELETE FROM s3_uploads WHERE id = ?", id); err != nil { - ln.Error(ctx, err) - } - - if _, err := mr.s3.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{ - Key: aws.String(m.ID), - Bucket: awsS3Bucket, - }); err != nil { - ln.Error(ctx, err) - } - } - - if err := tx.Commit(ctx); err != nil { - ln.Error(ctx, err) - return - } -} - -func (mr *MaraRevolt) DiscordMessageEdit(s *discordgo.Session, m *discordgo.MessageUpdate) { - if _, err := mr.pg.Exec(context.Background(), "UPDATE discord_messages SET content = $1, edited_at = $2 WHERE id = $3", m.Content, time.Now().Format(time.RFC3339), m.ID); err != nil { - ln.Error(context.Background(), err) - } -} - -func (mr *MaraRevolt) DiscordMessageCreate(s *discordgo.Session, m *discordgo.MessageCreate) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - ctx = opname.With(ctx, "marabot.discordMessageCreate") - - tx, err := mr.pg.Begin(ctx) - if err != nil { - ln.Error(ctx, err) - return - } - defer tx.Rollback(ctx) - - if err := mr.discordMessageCreate(ctx, tx, s, m.Message); err != nil { - ln.Error(ctx, err, ln.F{ - "channel_id": m.ChannelID, - "message_id": m.ID, - }) - s.MessageReactionAdd(m.ChannelID, m.ID, "🔥") - } - - if err := tx.Commit(ctx); err != nil { - ln.Error(ctx, err, ln.F{ - "channel_id": m.ChannelID, - "message_id": m.ID, - }) - } -} - -func (mr *MaraRevolt) DiscordReactionAdd(s *discordgo.Session, mra *discordgo.MessageReactionAdd) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - ctx = opname.With(ctx, "marabot.discord-reaction-add") - - if mra.Emoji.Name == "💾" { - go mr.backfillDiscordChannel(s, mra.ChannelID, mra.MessageID) - return - } - - m, err := s.ChannelMessage(mra.ChannelID, mra.MessageID) - if err != nil { - ln.Error(ctx, err) - return - } - defer s.MessageReactionRemove(m.ChannelID, m.ID, "🔥", "@me") - - tx, err := mr.pg.Begin(ctx) - if err != nil { - ln.Error(ctx, err) - return - } - defer tx.Rollback(ctx) - - if err := mr.discordMessageCreate(ctx, tx, s, m); err != nil { - ln.Error(context.Background(), err, ln.F{ - "channel_id": m.ChannelID, - "message_id": m.ID, - }) - s.MessageReactionAdd(m.ChannelID, m.ID, "ðŸ˜") - } - - if err := tx.Commit(ctx); err != nil { - ln.Error(ctx, err, ln.F{ - "channel_id": m.ChannelID, - "message_id": m.ID, - }) - } -} - -func (mr *MaraRevolt) doesDiscordMessageExist(ctx context.Context, tx pgx.Tx, messageID string) (bool, error) { - var count int - if err := tx.QueryRow(ctx, "SELECT COUNT(*) FROM discord_messages WHERE id = $1", messageID).Scan(&count); err != nil { - return false, err - } - - if count > 0 { - return true, nil - } - - return false, nil -} - -func (mr *MaraRevolt) backfillDiscordChannel(s *discordgo.Session, channelID, messageID string) { - curr := messageID - ctx := opname.With(context.Background(), "marabot.backfillDiscordChannel") - - ln.Log(ctx, ln.Action("archiving channel from message"), ln.F{"channel_id": channelID, "message_id": messageID}) - - t := time.NewTicker(5 * time.Second) - defer t.Stop() - - done := false - -outer: - for range t.C { - tx, err := mr.pg.Begin(ctx) - if err != nil { - ln.Error(ctx, err) - return - } - - ln.Log(ctx, ln.Action("fetching batch of messages"), ln.F{"curr": curr}) - msgs, err := s.ChannelMessages(channelID, 100, "", curr, "") - if err != nil { - ln.Error(ctx, err) - s.ChannelMessageSend(channelID, fmt.Sprintf("error getting messages past %s: %v", curr, err)) - break - } - ln.Log(ctx, ln.Action("fetching batch of messages"), ln.F{"num": len(msgs)}) - if len(msgs) == 0 { - break - } - - for _, msg := range msgs { - // found, err := mr.doesDiscordMessageExist(ctx, tx, msg.ID) - // if err != nil { - // ln.Error(ctx, err, ln.F{"message_id": msg.ID}) - // continue - // } - - // if found { - // if !done { - // ln.Log(ctx, ln.Action("stopping archival")) - // } - // done = true - // } - - before := time.Now() - if err := mr.discordMessageCreate(ctx, tx, s, msg); err != nil { - ln.Error(ctx, err, ln.F{"message_id": msg.ID}) - tx.Rollback(ctx) - continue outer - } - dur := time.Since(before) - ln.Log(ctx, ln.F{"message_id": msg.ID, "created_at": msg.Timestamp, "archival_time": dur}) - // found, err := mr.doesDiscordMessageExist(ctx, tx, msg.ID) - // if err != nil { - // ln.Error(ctx, err, ln.F{"message_id": msg.ID}) - // continue - // } - - curr = msg.ID - } - - if err := tx.Commit(ctx); err != nil { - ln.Error(ctx, err, ln.F{ - "channel_id": channelID, - "message_id": messageID, - }) - } - - if done { - break - } - - } - -} - -func (mr *MaraRevolt) discordMessageCreate(ctx context.Context, tx pgx.Tx, s *discordgo.Session, m *discordgo.Message) error { - if _, err := tx.Exec(ctx, `INSERT INTO discord_users (id, username, avatar_url, accent_color) -VALUES ($1, $2, $3, $4) -ON CONFLICT(id) -DO UPDATE SET username = EXCLUDED.username, avatar_url = EXCLUDED.avatar_url, accent_color = EXCLUDED.accent_color`, m.Author.ID, m.Author.Username, m.Author.AvatarURL(""), m.Author.AccentColor); err != nil { - return err - } - - if err := mr.archiveAttachment(ctx, tx, m.Author.AvatarURL(""), "avatars", ""); err != nil { - return err - } - - if _, err := tx.Exec(ctx, `INSERT INTO discord_messages (id, guild_id, channel_id, author_id, content, created_at, edited_at, webhook_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT(id) DO UPDATE SET content = EXCLUDED.content`, m.ID, m.GuildID, m.ChannelID, m.Author.ID, m.Content, m.Timestamp.Format(time.RFC3339), m.EditedTimestamp, m.WebhookID); err != nil { - return err - } - - if m.WebhookID != "" { - if _, err := tx.Exec(ctx, "INSERT INTO discord_webhook_message_info (id, name, avatar_url) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", m.ID, m.Author.Username, m.Author.AvatarURL("")); err != nil { - return err - } - } - - for _, att := range m.Attachments { - if _, err := tx.Exec(ctx, `INSERT INTO discord_attachments (id, message_id, url, proxy_url, filename, content_type, width, height, size) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT DO NOTHING`, att.ID, m.ID, att.URL, att.ProxyURL, att.Filename, att.ContentType, att.Width, att.Height, att.Size); err != nil { - return err - } - - if err := mr.archiveAttachment(ctx, tx, att.URL, "attachments", m.ID); err != nil { - return err - } - - for _, emb := range m.Embeds { - if emb.Image == nil { - continue - } - if _, err := tx.Exec(ctx, `INSERT INTO discord_attachments (id, message_id, url, proxy_url, filename, content_type, width, height, size) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT DO NOTHING`, uuid.NewString(), m.ID, emb.Image.URL, emb.Image.ProxyURL, filepath.Base(emb.Image.URL), "", emb.Image.Width, emb.Image.Height, 0); err != nil { - return err - } - - if err := mr.archiveAttachment(ctx, tx, emb.Image.URL, "attachments", m.ID); err != nil { - return err - } - } - - ch, err := s.Channel(m.ChannelID) - if err != nil { - return err - } - - if _, err := tx.Exec(ctx, "INSERT INTO discord_channels (id, guild_id, name, topic, nsfw) VALUES ($1, $2, $3, $4, $5) ON CONFLICT(id) DO UPDATE SET name = EXCLUDED.name, topic = EXCLUDED.topic, nsfw = EXCLUDED.nsfw", ch.ID, ch.GuildID, ch.Name, ch.Topic, ch.NSFW); err != nil { - return err - } - - for _, emoji := range m.GetCustomEmojis() { - eURL := fmt.Sprintf("https://cdn.discordapp.com/emojis/%s?size=240&quality=lossless", emoji.ID) - if _, err := tx.Exec(ctx, "INSERT INTO discord_emoji (id, guild_id, name, url) VALUES ($1, $2, $3, $4) ON CONFLICT(id) DO UPDATE SET name = EXCLUDED.name, url = EXCLUDED.url", emoji.ID, furryholeDiscord, emoji.Name, eURL); err != nil { - return err - } - mr.attachmentPreprocess.Add([3]string{eURL, "emoji", ""}, len(eURL)) - if err := mr.archiveAttachment(ctx, tx, eURL, "emoji", m.ID); err != nil { - return err - } - } - } - - return nil -} diff --git a/cmd/marabot/irc.go b/cmd/marabot/irc.go deleted file mode 100644 index 9fbbcd7..0000000 --- a/cmd/marabot/irc.go +++ /dev/null @@ -1,86 +0,0 @@ -package main - -import ( - "context" - "flag" - "time" - - irc "github.com/thoj/go-ircevent" - "within.website/ln" - "within.website/ln/opname" - "within.website/x/internal" - "within.website/x/web/revolt" -) - -var ( - ircNick = flag.String("irc-nick", "[Mara]", "IRC nickname") - ircUser = flag.String("irc-user", "sh0rk", "IRC username") - ircReal = flag.String("irc-real", "Friendly sh0rk Mara", "IRC realname") - ircServer = flag.String("irc-server", "chrysalis:6667", "IRC server to connect to") - ircSASLUsername = flag.String("irc-sasl-username", "", "SASL username") - ircSASLPassword = flag.String("irc-sasl-password", "", "SASL password") - ircRevoltChannel = flag.String("irc-revolt-channel", "", "channel to copy #xeserv messages to") -) - -func (mr *MaraRevolt) IRCBot(ctx context.Context) { - ctx = opname.With(ctx, "ircbot") - ctx = ln.WithF(ctx, ln.F{ - "irc_server": *ircServer, - }) - for { - irccon := irc.IRC(*ircNick, *ircUser) - go func() { - <-ctx.Done() - irccon.Disconnect() - }() - - go func() { - t := time.NewTicker(250 * time.Millisecond) - defer t.Stop() - for { - select { - case <-ctx.Done(): - return - case msg := <-mr.ircmsgs: - <-t.C - irccon.Privmsg("#xeserv", msg) - } - } - }() - - if *ircSASLUsername != "" && *ircSASLPassword != "" { - irccon.UseSASL = true - irccon.SASLLogin = *ircSASLUsername - irccon.SASLPassword = *ircSASLPassword - irccon.SASLMech = "plain" - } - - irccon.AddCallback("001", func(e *irc.Event) { irccon.Join("#xeserv") }) - irccon.AddCallback("PRIVMSG", func(e *irc.Event) { - if _, err := mr.pg.Exec(ctx, `INSERT INTO irc_messages(nick, user, host, channel, content, tags) VALUES ($1, $2, $3, $4, $5, $6)`, e.Nick, e.User, e.Host, e.Arguments[0], e.Message(), ""); err != nil { - ln.Error(ctx, err) - } - - if e.Arguments[0] == "#xeserv" { - sendMsg := &revolt.SendMessage{ - Masquerade: &revolt.Masquerade{ - Name: e.Nick, - AvatarURL: "https://cdn.xeiaso.net/avatar/" + internal.Hash(e.User, e.Host), - }, - Content: e.Message(), - } - - if _, err := mr.cli.ChannelSendMessage(ctx, *ircRevoltChannel, sendMsg); err != nil { - ln.Error(ctx, err) - return - } - } - }) - err := irccon.Connect(*ircServer) - if err != nil { - ln.Error(ctx, err) - return - } - irccon.Loop() - } -} diff --git a/cmd/marabot/litestream.yaml b/cmd/marabot/litestream.yaml deleted file mode 100644 index 16a26b0..0000000 --- a/cmd/marabot/litestream.yaml +++ /dev/null @@ -1,8 +0,0 @@ -dbs: - - path: /home/cadey/code/Xe/x/cmd/marabot/marabot.db - replicas: - - path: /mnt/arsene/backup/marabot - - type: s3 - bucket: xeserv-marabot-backups - path: db - region: ca-central-1 diff --git a/cmd/marabot/main.go b/cmd/marabot/main.go deleted file mode 100644 index ae5b99a..0000000 --- a/cmd/marabot/main.go +++ /dev/null @@ -1,377 +0,0 @@ -package main - -import ( - "bytes" - "context" - "crypto/sha512" - _ "embed" - "flag" - "fmt" - "io" - "net/http" - "os" - "os/signal" - "path/filepath" - "syscall" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" - "github.com/bwmarrin/discordgo" - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgxpool" - "tailscale.com/hostinfo" - "within.website/ln" - "within.website/ln/opname" - "within.website/x/bundler" - "within.website/x/internal" - "within.website/x/web" - "within.website/x/web/revolt" -) - -var ( - pgURL = flag.String("database-url", "", "URL for the database (postgres)") - - discordToken = flag.String("discord-token", "", "Discord bot token") - revoltToken = flag.String("revolt-token", "", "Revolt bot token") - revoltAPIServer = flag.String("revolt-api-server", "https://api.revolt.chat", "API server for Revolt") - revoltWebsocketServer = flag.String("revolt-ws-server", "wss://ws.revolt.chat", "Websocket server for Revolt") - revoltBotID = flag.String("revolt-bot-id", "", "bot ID for revolt") - tsAuthkey = flag.String("ts-authkey", "", "Tailscale authkey") - tsHostname = flag.String("ts-hostname", "", "Tailscale hostname") - - adminDiscordUser = flag.String("admin-discord-user", "", "Discord user ID of the admin") - adminRevoltUser = flag.String("admin-revolt-user", "", "Revolt user ID of the admin") - - furryholeDiscord = flag.String("furryhole-discord", "192289762302754817", "Discord channel ID for furryhole") - furryholeRevolt = flag.String("furryhole-revolt", "01FEXZ1XPWMEJXMF836FP16HB8", "Revolt channel ID for furryhole") - awsS3Bucket = flag.String("aws-s3-bucket", "", "S3 bucket name") - awsS3Region = flag.String("aws-s3-region", "ca-central-1", "S3 bucket region") - - //go:embed schema.sql - dbSchema string -) - -func main() { - internal.HandleStartup() - - hostinfo.SetApp("within.website/x/cmd/marabot") - - ctx, cancel := context.WithCancel(opname.With(context.Background(), "marabot")) - defer cancel() - - ln.Log(ctx, ln.Action("starting up")) - - pgcfg, err := pgxpool.ParseConfig(*pgURL) - if err != nil { - ln.FatalErr(ctx, err, ln.Action("parsing postgres config")) - } - - pgcfg.MinConns = 5 - pgcfg.MaxConns = 20 - - pg, err := pgxpool.New(ctx, *pgURL) - if err != nil { - ln.FatalErr(ctx, err, ln.Action("opening postgres database")) - } - defer pg.Close() - - if err := pg.Ping(ctx); err != nil { - ln.FatalErr(ctx, err, ln.Action("testing postgres connection")) - } - - ircmsgs := make(chan string, 10) - - // Init a new client. - client, err := revolt.NewWithEndpoint(*revoltToken, *revoltAPIServer, *revoltWebsocketServer) - if err != nil { - ln.FatalErr(ctx, err, ln.Action("creating revolt client")) - } - - sess, err := session.NewSession(&aws.Config{ - Region: aws.String(*awsS3Region)}, - ) - - uploader := s3manager.NewUploader(sess) - - mr := &MaraRevolt{ - cli: client, - pg: pg, - ircmsgs: ircmsgs, - uploader: uploader, - s3: s3.New(sess), - } - - mr.attachmentUpload = bundler.New(mr.S3Upload) - mr.attachmentUpload.BundleCountThreshold = 5 - mr.attachmentUpload.DelayThreshold = time.Minute - - mr.attachmentPreprocess = bundler.New(mr.PreprocessLinks) - mr.attachmentPreprocess.BundleCountThreshold = 10 - mr.attachmentPreprocess.DelayThreshold = 30 * time.Second - - go mr.IRCBot(ctx) - - client.Connect(ctx, mr) - - dg, err := discordgo.New("Bot " + *discordToken) - if err != nil { - ln.FatalErr(ctx, err, ln.Action("creating discord client")) - } - - dg.AddHandler(mr.DiscordMessageCreate) - dg.AddHandler(mr.DiscordMessageDelete) - dg.AddHandler(mr.DiscordMessageEdit) - dg.AddHandler(mr.DiscordReactionAdd) - - if err := dg.Open(); err != nil { - ln.FatalErr(ctx, err, ln.Action("opening discord client")) - } - defer dg.Close() - - if err := mr.importDiscordData(ctx, pg, dg); err != nil { - ln.Error(ctx, err) - } - - // Wait for close. - sc := make(chan os.Signal, 1) - - signal.Notify( - sc, - syscall.SIGINT, - syscall.SIGTERM, - os.Interrupt, - ) - for { - select { - case <-ctx.Done(): - return - case <-sc: - ln.Log(ctx, ln.Info("shutting down")) - cancel() - mr.attachmentPreprocess.Flush() - mr.attachmentUpload.Flush() - time.Sleep(150 * time.Millisecond) - return - } - } -} - -type Attachment struct { - ID string `json:"id"` - URL string `json:"url"` - Kind string `json:"kind"` - ContentType string `json:"content_type"` - CreatedAt string `json:"created_at"` - MessageID *string `json:"message_id"` - Data []byte `json:"-"` -} - -func (mr *MaraRevolt) PreprocessLinks(data [][3]string) { - ctx := opname.With(context.Background(), "marabot.link-preprocessor") - ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) - defer cancel() - - mr.preprocessLinks(ctx, data) -} - -func (mr *MaraRevolt) preprocessLinks(ctx context.Context, data [][3]string) { - tx, err := mr.pg.Begin(ctx) - if err != nil { - ln.Error(ctx, err) - return - } - defer tx.Rollback(ctx) - - for _, linkkind := range data { - kind := linkkind[1] - link := linkkind[0] - msgID := linkkind[2] - - var count int - if err := tx.QueryRow(ctx, "SELECT COUNT(*) FROM s3_uploads WHERE url = ?", link).Scan(&count); err != nil { - ln.Error(ctx, err) - continue - } - if count != 0 { - continue - } - - att, err := hashURL(link, kind) - if err != nil { - ln.Error(ctx, err, ln.F{"link": link, "kind": kind}) - - if werr, ok := err.(*web.Error); ok { - if werr.GotStatus == http.StatusNotFound { - tx.Exec(ctx, "DELETE FROM discord_users WHERE avatar_url = ?", link) - tx.Exec(ctx, "DELETE FROM discord_attachments WHERE url = ?", link) - tx.Exec(ctx, "DELETE FROM discord_emoji WHERE url = ?", link) - tx.Exec(ctx, "DELETE FROM revolt_attachments WHERE url = ?", link) - tx.Exec(ctx, "DELETE FROM revolt_users WHERE avatar_url = ?", link) - tx.Exec(ctx, "DELETE FROM revolt_emoji WHERE url = ?", link) - } - } - - continue - } - - att.MessageID = aws.String(msgID) - - mr.attachmentUpload.Add(att, len(att.Data)) - } - if err := tx.Commit(ctx); err != nil { - ln.Error(ctx, err) - } -} - -func (mr *MaraRevolt) archiveAttachment(ctx context.Context, tx pgx.Tx, link, kind, messageID string) error { - att, err := hashURL(link, kind) - if err != nil { - ln.Error(ctx, err, ln.F{"link": link, "kind": kind}) - - if werr, ok := err.(*web.Error); ok { - if werr.GotStatus == http.StatusNotFound { - tx.Exec(ctx, "DELETE FROM discord_users WHERE avatar_url = $1", link) - tx.Exec(ctx, "DELETE FROM discord_attachments WHERE url = $1", link) - tx.Exec(ctx, "DELETE FROM discord_emoji WHERE url = $1", link) - tx.Exec(ctx, "DELETE FROM revolt_attachments WHERE url = $1", link) - tx.Exec(ctx, "DELETE FROM revolt_users WHERE avatar_url = $1", link) - tx.Exec(ctx, "DELETE FROM revolt_emoji WHERE url = $1", link) - return werr - } else { - return err - } - } - } - - if att == nil { - return nil - } - - att.MessageID = aws.String(messageID) - - key := filepath.Join(att.Kind, att.ID) - - f := ln.F{"kind": att.Kind, "id": att.ID, "url": att.URL, "content_type": att.ContentType} - - var count int - if err := tx.QueryRow(ctx, "SELECT COUNT(*) FROM s3_uploads WHERE id = $1", att.ID).Scan(&count); err != nil { - ln.Error(ctx, err, f) - return err - } - - f["count"] = count - - if count != 0 { - return nil - } - - if _, err := mr.uploader.UploadWithContext(ctx, &s3manager.UploadInput{ - Bucket: aws.String(*awsS3Bucket), - Key: aws.String(key), - ContentType: aws.String(att.ContentType), - Body: bytes.NewBuffer(att.Data), - Metadata: map[string]*string{ - "Original-URL": aws.String(att.URL), - "Message-ID": att.MessageID, - }, - }); err != nil { - return err - } - - if _, err := tx.Exec(ctx, "INSERT INTO s3_uploads(id, url, kind, content_type, created_at, message_id) VALUES ($1, $2, $3, $4, $5, $6)", att.ID, att.URL, att.Kind, att.ContentType, att.CreatedAt, att.MessageID); err != nil { - ln.Error(ctx, err, ln.Action("saving upload information to DB"), f) - } - - return nil -} - -func hashURL(itemURL, kind string) (*Attachment, error) { - resp, err := http.Get(itemURL) |
