Xesite V4 (#723)

* scripts/ditherify: fix quoting

Signed-off-by: Xe Iaso <me@xeiaso.net>

* clean up some old files

Signed-off-by: Xe Iaso <me@xeiaso.net>

* import site into lume

Signed-off-by: Xe Iaso <me@xeiaso.net>

* initial go code

Signed-off-by: Xe Iaso <me@xeiaso.net>

* move vods index to top level

Signed-off-by: Xe Iaso <me@xeiaso.net>

* remove the ads

Signed-off-by: Xe Iaso <me@xeiaso.net>

* internal/lume: metrics

Signed-off-by: Xe Iaso <me@xeiaso.net>

* delete old code

Signed-off-by: Xe Iaso <me@xeiaso.net>

* load config into memory

Signed-off-by: Xe Iaso <me@xeiaso.net>

* autogenerate data from dhall config

Signed-off-by: Xe Iaso <me@xeiaso.net>

* various cleanups, import clackset logic

Signed-off-by: Xe Iaso <me@xeiaso.net>

* Update signalboost.dhall (#722)

Added myself, and also fixed someone’s typo

* Add Connor Edwards to signal boost (#721)

* add cache headers

Signed-off-by: Xe Iaso <me@xeiaso.net>

* move command to xesite folder

Signed-off-by: Xe Iaso <me@xeiaso.net>

* xesite: listen for GitHub webhook push events

Signed-off-by: Xe Iaso <me@xeiaso.net>

* xesite: 5 minute timeout for rebuilding the site

Signed-off-by: Xe Iaso <me@xeiaso.net>

* xesite: add rebuild metrics

Signed-off-by: Xe Iaso <me@xeiaso.net>

* xesite: update default variables

Signed-off-by: Xe Iaso <me@xeiaso.net>

* don't commit binaries oops lol

Signed-off-by: Xe Iaso <me@xeiaso.net>

* lume: make search have a light background

Signed-off-by: Xe Iaso <me@xeiaso.net>

* add a notfound page

Signed-off-by: Xe Iaso <me@xeiaso.net>

* fetch info from patreon API

Signed-off-by: Xe Iaso <me@xeiaso.net>

* create contact page

Signed-off-by: Xe Iaso <me@xeiaso.net>

* Toot embedding

Signed-off-by: Xe Iaso <me@xeiaso.net>

* attempt a docker image

Signed-off-by: Xe Iaso <me@xeiaso.net>

* lume: fix deno lock

Signed-off-by: Xe Iaso <me@xeiaso.net>

* add gokrazy post

Signed-off-by: Xe Iaso <me@xeiaso.net>

* cmd/xesite: go up before trying to connect to the saas proxy

Signed-off-by: Xe Iaso <me@xeiaso.net>

* blog: add Sine post/demo

Signed-off-by: Xe Iaso <me@xeiaso.net>

---------

Signed-off-by: Xe Iaso <me@xeiaso.net>
Co-authored-by: bri <284789+b-@users.noreply.github.com>
Co-authored-by: Connor Edwards <38229097+cedws@users.noreply.github.com>

5 files changed, 447 insertions, 0 deletions

diff --git a/cmd/patreon-saasproxy/main.go b/cmd/patreon-saasproxy/main.go
new file mode 100644
index 0000000..cfe9717
--- /dev/null
+++ b/cmd/patreon-saasproxy/main.go
@@ -0,0 +1,146 @@
+package main
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"expvar"
+	"flag"
+	"log"
+	"log/slog"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/facebookgo/flagenv"
+	_ "github.com/joho/godotenv/autoload"
+	"golang.org/x/oauth2"
+	"gopkg.in/mxpv/patreon-go.v1"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/hostinfo"
+	"tailscale.com/metrics"
+	"tailscale.com/tsnet"
+	"tailscale.com/tsweb"
+	"xeiaso.net/v4/internal"
+)
+
+var (
+	clientID          = flag.String("client-id", "", "Patreon client ID")
+	clientSecret      = flag.String("client-secret", "", "Patreon client secret")
+	dataDir           = flag.String("data-dir", "./var", "Directory to store data in")
+	tailscaleHostname = flag.String("tailscale-hostname", "patreon-saasproxy", "Tailscale hostname to use")
+
+	tokenFetches = metrics.LabelMap{Label: "host"}
+)
+
+func main() {
+	flagenv.Parse()
+	flag.Parse()
+	internal.Slog()
+
+	hostinfo.SetApp("xeiaso.net/v4/cmd/patreon-saasproxy")
+
+	expvar.Publish("gauge_xesite_patreon_token_fetch", &tokenFetches)
+
+	os.MkdirAll(*dataDir, 0700)
+	os.MkdirAll(filepath.Join(*dataDir, "tsnet"), 0700)
+
+	srv := &tsnet.Server{
+		Hostname: *tailscaleHostname,
+		Dir:      filepath.Join(*dataDir, "tsnet"),
+		Logf:     func(string, ...any) {},
+	}
+
+	defer srv.Close()
+
+	lc, err := srv.LocalClient()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	config := oauth2.Config{
+		ClientID:     *clientID,
+		ClientSecret: *clientSecret,
+		Endpoint: oauth2.Endpoint{
+			AuthURL:  patreon.AuthorizationURL,
+			TokenURL: patreon.AccessTokenURL,
+		},
+		Scopes: []string{"users", "pledges-to-me", "my-campaign"},
+	}
+
+	if !internal.FileExists(filepath.Join(*dataDir, "patreon-token.json")) {
+		val, ok := os.LookupEnv("PATREON_TOKEN_JSON_B64")
+		if !ok {
+			log.Fatal("PATREON_TOKEN_JSON_B64 not set")
+		}
+
+		fout, err := os.Create(filepath.Join(*dataDir, "patreon-token.json"))
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer fout.Close()
+
+		decoded, err := base64.StdEncoding.DecodeString(val)
+		if err != nil {
+			slog.Error("can't decode token", "err", err, "val", val)
+			log.Fatal(err)
+		}
+
+		if _, err := fout.Write(decoded); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	token, err := internal.ReadToken(filepath.Join(*dataDir, "patreon-token.json"))
+	if err != nil {
+		log.Fatalf("error reading token: %v", err)
+	}
+
+	cts := internal.CachingTokenSource(filepath.Join(*dataDir, "patreon-token.json"), &config, token)
+
+	s := &Server{
+		lc:  lc,
+		cts: cts,
+	}
+
+	http.HandleFunc("/give-token", s.GiveToken)
+	http.HandleFunc("/metrics", tsweb.VarzHandler)
+
+	ln, err := srv.Listen("tcp", ":80")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	slog.Info("listening over tailscale", "hostname", *tailscaleHostname)
+
+	log.Fatal(http.Serve(ln, nil))
+}
+
+type Server struct {
+	lc  *tailscale.LocalClient
+	cts oauth2.TokenSource
+}
+
+func (s *Server) GiveToken(w http.ResponseWriter, r *http.Request) {
+	whois, err := s.lc.WhoIs(r.Context(), r.RemoteAddr)
+	if err != nil {
+		slog.Error("whois failed", "err", err, "remoteAddr", r.RemoteAddr)
+		http.Error(w, "invalid remote address", http.StatusBadRequest)
+		return
+	}
+
+	tokenFetches.Add(whois.Node.Name, 1)
+
+	token, err := s.cts.Token()
+	if err != nil {
+		slog.Error("token fetch failed", "err", err)
+		http.Error(w, "token fetch failed", http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+
+	if err := json.NewEncoder(w).Encode(token); err != nil {
+		slog.Error("token encode failed", "err", err)
+		return
+	}
+}
diff --git a/cmd/patreon-saasproxy/var/.gitignore b/cmd/patreon-saasproxy/var/.gitignore
new file mode 100644
index 0000000..d6b7ef3
--- /dev/null
+++ b/cmd/patreon-saasproxy/var/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/cmd/xesite/github.go b/cmd/xesite/github.go
new file mode 100644
index 0000000..bde7c9d
--- /dev/null
+++ b/cmd/xesite/github.go
@@ -0,0 +1,67 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"expvar"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"time"
+
+	"github.com/go-git/go-git/v5"
+	"tailscale.com/metrics"
+	"xeiaso.net/v4/internal/github"
+	"xeiaso.net/v4/internal/lume"
+)
+
+var (
+	webhookCount = metrics.LabelMap{Label: "source"}
+	buildErrors  = metrics.LabelMap{Label: "err"}
+)
+
+func init() {
+	expvar.Publish("gauge_xesite_webhook_count", &webhookCount)
+	expvar.Publish("gauge_xesite_build_errors", &buildErrors)
+}
+
+type GitHubWebhook struct {
+	fs *lume.FS
+}
+
+func (gh *GitHubWebhook) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	webhookCount.Add("github", 1)
+
+	if r.Header.Get("X-GitHub-Event") != "push" {
+		slog.Info("not a push event", "event", r.Header.Get("X-GitHub-Event"))
+	}
+
+	var event github.PushEvent
+	if err := json.NewDecoder(r.Body).Decode(&event); err != nil {
+		slog.Error("error decoding event", "error", err)
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	slog.Info("push!", "ref", event.Ref, "author", event.Pusher.Login)
+
+	if event.Ref != "refs/heads/"+*gitBranch {
+		slog.Error("not the right branch", "branch", event.Ref)
+		fmt.Fprintln(w, "OK")
+		return
+	}
+
+	go func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+		defer cancel()
+		if err := gh.fs.Update(ctx); err != nil {
+			buildErrors.Add(err.Error(), 1)
+			if err == git.NoErrAlreadyUpToDate {
+				slog.Info("already up to date")
+				return
+			}
+
+			slog.Error("error updating", "error", err)
+		}
+	}()
+}
diff --git a/cmd/xesite/main.go b/cmd/xesite/main.go
new file mode 100644
index 0000000..566678b
--- /dev/null
+++ b/cmd/xesite/main.go
@@ -0,0 +1,134 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"log/slog"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/donatj/hmacsig"
+	"github.com/facebookgo/flagenv"
+	"github.com/go-git/go-git/v5"
+	_ "github.com/joho/godotenv/autoload"
+	"tailscale.com/hostinfo"
+	"tailscale.com/tsnet"
+	"tailscale.com/tsweb"
+	"xeiaso.net/v4/internal"
+	"xeiaso.net/v4/internal/lume"
+)
+
+var (
+	bind                = flag.String("bind", ":3000", "Port to listen on")
+	devel               = flag.Bool("devel", false, "Enable development mode")
+	dataDir             = flag.String("data-dir", "./var", "Directory to store data in")
+	gitBranch           = flag.String("git-branch", "main", "Git branch to clone")
+	gitRepo             = flag.String("git-repo", "https://github.com/Xe/site", "Git repository to clone")
+	githubSecret        = flag.String("github-secret", "", "GitHub secret to use for webhooks")
+	patreonSaasProxyURL = flag.String("patreon-saasproxy-url", "http://patreon-saasproxy/give-token", "URL to use for the patreon saasproxy")
+	siteURL             = flag.String("site-url", "https://xeiaso.net/", "URL to use for the site")
+	tsnetHostname       = flag.String("tailscale-hostname", "xesite", "Tailscale hostname to use")
+)
+
+func main() {
+	flagenv.Parse()
+	flag.Parse()
+	internal.Slog()
+
+	hostinfo.SetApp("xeiaso.net/v4/cmd/xesite")
+
+	ctx := context.Background()
+
+	ln, err := net.Listen("tcp", *bind)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	os.MkdirAll(*dataDir, 0700)
+	os.MkdirAll(filepath.Join(*dataDir, "tsnet"), 0700)
+
+	srv := &tsnet.Server{
+		Hostname: *tsnetHostname + "-" + os.Getenv("FLY_REGION"),
+		Logf:     func(string, ...any) {},
+		Dir:      filepath.Join(*dataDir, "tsnet"),
+	}
+
+	if err := srv.Start(); err != nil {
+		log.Fatal(err)
+	}
+
+	if _, err := srv.Up(context.Background()); err != nil {
+		log.Fatal(err)
+	}
+
+	hc := srv.HTTPClient()
+
+	pc, err := NewPatreonClient(hc)
+	if err != nil {
+		slog.Error("can't create patreon client", "err", err)
+	}
+
+	fs, err := lume.New(ctx, &lume.Options{
+		Branch:        *gitBranch,
+		Repo:          *gitRepo,
+		StaticSiteDir: "lume",
+		URL:           *siteURL,
+		Development:   *devel,
+		PatreonClient: pc,
+		DataDir:       *dataDir,
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	defer fs.Close()
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	mux := http.NewServeMux()
+	mux.Handle("/", http.FileServer(http.FS(fs)))
+	mux.HandleFunc("/metrics", tsweb.VarzHandler)
+
+	mux.HandleFunc("/blog.atom", func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, "/blog/index.rss", http.StatusMovedPermanently)
+	})
+
+	// NOTE(Xe): Had to rename this page because of a Lume/Go embed bug.
+	mux.HandleFunc(`/blog/%F0%9F%A5%BA`, func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, "/blog/xn--ts9h/", http.StatusMovedPermanently)
+	})
+
+	if *devel {
+		mux.HandleFunc("/.within/hook/github", func(w http.ResponseWriter, r *http.Request) {
+			if err := fs.Update(r.Context()); err != nil {
+				if err == git.NoErrAlreadyUpToDate {
+					w.WriteHeader(http.StatusOK)
+					fmt.Fprintln(w, "already up to date")
+					return
+				}
+				log.Println(err)
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+				return
+			}
+		})
+	} else {
+		gh := &GitHubWebhook{fs: fs}
+		s := hmacsig.Handler256(gh, *githubSecret)
+		mux.Handle("/.within/hook/github", s)
+	}
+
+	mux.Handle("/.within/hook/patreon", &PatreonWebhook{fs: fs})
+
+	var h http.Handler = mux
+	h = internal.ClackSet(fs.Clacks()).Middleware(h)
+	h = internal.CacheHeader(h)
+
+	slog.Info("starting server", "bind", *bind)
+	log.Fatal(http.Serve(ln, h))
+}
diff --git a/cmd/xesite/patreon.go b/cmd/xesite/patreon.go
new file mode 100644
index 0000000..94462cc
--- /dev/null
+++ b/cmd/xesite/patreon.go
@@ -0,0 +1,98 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/go-git/go-git/v5"
+	"golang.org/x/oauth2"
+	"gopkg.in/mxpv/patreon-go.v1"
+	"xeiaso.net/v4/internal/lume"
+	"xeiaso.net/v4/internal/saasproxytoken"
+)
+
+var (
+	patreonWebhookSecret = flag.String("patreon-webhook-secret", "", "Patreon webhook secret")
+)
+
+func NewPatreonClient(hc *http.Client) (*patreon.Client, error) {
+	ts := saasproxytoken.RemoteTokenSource(*patreonSaasProxyURL, hc)
+	tc := oauth2.NewClient(context.Background(), ts)
+
+	client := patreon.NewClient(tc)
+	if u, err := client.FetchUser(); err != nil {
+		return nil, err
+	} else {
+		slog.Info("logged in as", "user", u.Data.Attributes.FullName)
+	}
+
+	return client, nil
+}
+
+type PatreonWebhook struct {
+	fs *lume.FS
+}
+
+func (p *PatreonWebhook) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if !strings.HasPrefix(r.Header.Get(patreon.HeaderEventType), "pledges:") {
+		slog.Debug("not a pledge event", "event", r.Header.Get(patreon.HeaderEventType))
+		http.Error(w, "invalid request", http.StatusBadRequest)
+		return
+	}
+
+	defer r.Body.Close()
+
+	data, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 8*1024*1024)) // 8 kb limit
+	if err != nil {
+		slog.Error("error reading body", "error", err)
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	ok, err := patreon.VerifySignature(data, *patreonWebhookSecret, r.Header.Get(patreon.HeaderSignature))
+	if err != nil {
+		slog.Error("error verifying signature", "error", err)
+		http.Error(w, "error reading body", http.StatusBadRequest)
+		return
+	}
+
+	if !ok {
+		slog.Error("invalid signature")
+		http.Error(w, "error reading body", http.StatusBadRequest)
+		return
+	}
+
+	webhookCount.Add("patreon", 1)
+
+	var event patreon.WebhookPledge
+	if err := json.Unmarshal(data, &event); err != nil {
+		slog.Error("error decoding event", "error", err)
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	slog.Info("new pledge!", "patron", event.Data.Relationships.Patron.Data.ID, "pledge", event.Data.ID, "amount", event.Data.Attributes.AmountCents)
+
+	go func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+		defer cancel()
+		if err := p.fs.Update(ctx); err != nil {
+			buildErrors.Add(err.Error(), 1)
+			if err == git.NoErrAlreadyUpToDate {
+				slog.Info("already up to date")
+				return
+			}
+
+			slog.Error("error updating", "error", err)
+		}
+	}()
+
+	fmt.Fprintln(w, "Rebuild triggered")
+}