tokenize

Signed-off-by: Xe Iaso <me@xeiaso.net>

7 files changed, 82 insertions, 16 deletions

diff --git a/cmd/tokenize/main.go b/cmd/tokenize/main.go
new file mode 100644
index 0000000..4e0db40
--- /dev/null
+++ b/cmd/tokenize/main.go
@@ -0,0 +1,50 @@
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+
+	gpt3encoder "github.com/samber/go-gpt-3-encoder"
+	"within.website/x/internal"
+)
+
+var (
+	decode = flag.Bool("decode", false, "if true, decode instead of encode")
+)
+
+func main() {
+	internal.HandleStartup()
+
+	enc, err := gpt3encoder.NewEncoder()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if *decode {
+		var tokens []int
+		if err := json.NewDecoder(os.Stdin).Decode(&tokens); err != nil {
+			log.Fatal(err)
+		}
+
+		fmt.Fprintln(os.Stdout, enc.Decode(tokens))
+		return
+	}
+
+	data, err := io.ReadAll(os.Stdin)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	tokens, err := enc.Encode(string(data))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if err := json.NewEncoder(os.Stdout).Encode(&tokens); err != nil {
+		log.Fatal(err)
+	}
+}
diff --git a/flake.lock b/flake.lock
index d264149..18fa34d 100644
--- a/flake.lock
+++ b/flake.lock
@@ -160,16 +160,15 @@
     "wasigo": {
       "flake": false,
       "locked": {
-        "lastModified": 1680820691,
-        "narHash": "sha256-PBYwz5tCa54XOJBbfrY1zjCSTfDh/SOoH7D6InTMxcA=",
-        "owner": "Pryz",
+        "lastModified": 1681491667,
+        "narHash": "sha256-Bb9EydUXgoKnjbaMWxa48VYU3Q0+xisGs5xzqsEuPtc=",
+        "owner": "golang",
         "repo": "go",
-        "rev": "8b46a289d82f8a9313b1c4e07d11cb2373b0478c",
+        "rev": "ac9d777caf06364524ce9c67203a14a899f71a57",
         "type": "github"
       },
       "original": {
-        "owner": "Pryz",
-        "ref": "wasip1-wasm",
+        "owner": "golang",
         "repo": "go",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index 50d9b86..fee89af 100644
--- a/flake.nix
+++ b/flake.nix
@@ -34,7 +34,7 @@
     # go + wasip1
     wasigo = {
       # https://github.com/Pryz/go/archive/refs/heads/wasip1-wasm.zip
-      url = "github:Pryz/go/wasip1-wasm";
+      url = "github:golang/go";
       flake = false;
     };
   };
@@ -254,7 +254,6 @@
             libavif
             sqlite-interactive
 
-            cargo
             cargo-watch
             rustfmt
             rust-analyzer
@@ -263,6 +262,9 @@
             wabt
             bloaty
             rust
+
+            tinyemu
+            zig
           ];
 
           XEDN_STATIC = self.packages.${system}.xedn-static;
diff --git a/go.mod b/go.mod
index b717413..6c88967 100644
--- a/go.mod
+++ b/go.mod
@@ -60,6 +60,7 @@ require (
 	github.com/aws/smithy-go v1.13.5 // indirect
 	github.com/coreos/go-iptables v0.6.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dlclark/regexp2 v1.7.0 // indirect
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
 	github.com/fogleman/gg v1.3.0 // indirect
@@ -95,6 +96,8 @@ require (
 	github.com/mndrix/ps v0.0.0-20170330174427-18e65badd6ab // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/samber/go-gpt-3-encoder v0.3.1 // indirect
+	github.com/samber/lo v1.37.0 // indirect
 	github.com/sendgrid/rest v2.6.9+incompatible // indirect
 	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
 	github.com/tailscale/certstore v0.1.1-0.20220316223106-78d6e1c49d8d // indirect
diff --git a/go.sum b/go.sum
index c97e4f4..3f51347 100644
--- a/go.sum
+++ b/go.sum
@@ -135,6 +135,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/dblohm7/wingoes v0.0.0-20221124203957-6ac47ab19aa5 h1:84SSlQpWqllOmtng34NorWGJbzX00SI2J4MQjXNYUuU=
 github.com/disintegration/imaging v1.6.1 h1:JnBbK6ECIZb1NsWIikP9pd8gIlTIRx7fuDNpU9fsxOE=
 github.com/disintegration/imaging v1.6.1/go.mod h1:xuIt+sRxDFrHS0drzXUlCJthkJ8k7lkkUojDSR247MQ=
+github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo=
+github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/eaburns/peggy v0.0.0-20180405011029-d685ddd3cbcb/go.mod h1:5tfPwI6ukiK3W5vJzkj5MBQKHHY9Gcy2y6k1FC/23Xk=
 github.com/eaburns/peggy v1.0.2 h1:RJwNVF4cvzLGiKInHGBT8sVwP5HDuVP/PaBU9tHO9Rk=
 github.com/eaburns/peggy v1.0.2/go.mod h1:X2pbl0EV5erfnK8kSGwo0lBCgMGokvR1E6KerAoDKXg=
@@ -422,6 +424,10 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo=
 github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
+github.com/samber/go-gpt-3-encoder v0.3.1 h1:YWb9GsGYUgSX/wPtsEHjyNGRQXsQ9vDCg9SU2x9uMeU=
+github.com/samber/go-gpt-3-encoder v0.3.1/go.mod h1:27nvdvk9ZtALyNtgs9JsPCMYja0Eleow/XzgjqwRtLU=
+github.com/samber/lo v1.37.0 h1:XjVcB8g6tgUp8rsPsJ2CvhClfImrpL04YpQHXeHPhRw=
+github.com/samber/lo v1.37.0/go.mod h1:9vaz2O4o8oOnK23pd2TrXufcbdbJIa3b6cstBWKpopA=
 github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a h1:iLcLb5Fwwz7g/DLK89F+uQBDeAhHhwdzB5fSlVdhGcM=
 github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a/go.mod h1:wozgYq9WEBQBaIJe4YZ0qTSFAMxmcwBhQH0fO0R34Z0=
 github.com/sendgrid/rest v2.6.9+incompatible h1:1EyIcsNdn9KIisLW50MKwmSRSK+ekueiEMJ7NEoxJo0=
diff --git a/gomod2nix.toml b/gomod2nix.toml
index 502e4de..ef3c0ed 100644
--- a/gomod2nix.toml
+++ b/gomod2nix.toml
@@ -73,6 +73,9 @@ schema = 3
   [mod."github.com/disintegration/imaging"]
     version = "v1.6.1"
     hash = "sha256-3IWhzlxuMQIQzhTYUT6SiToBjOzR9kfyoDCh1EetqgQ="
+  [mod."github.com/dlclark/regexp2"]
+    version = "v1.7.0"
+    hash = "sha256-Z/M62esiZ0fVwvueVQhwz18z0eS22LZ3DJ4O8FKp3AY="
   [mod."github.com/eaburns/peggy"]
     version = "v1.0.2"
     hash = "sha256-toux6NpVog1DW+zEWX8dAj+hCZaeMG6z2fe4cAz2LDw="
@@ -229,6 +232,12 @@ schema = 3
   [mod."github.com/rs/cors"]
     version = "v1.8.3"
     hash = "sha256-VgVB4HKAhPSjNg96mIEUN1bt5ZQng8Fi3ZABy3CDWQE="
+  [mod."github.com/samber/go-gpt-3-encoder"]
+    version = "v0.3.1"
+    hash = "sha256-quLfn1bazwQer7OBLtzjfN40cE39Q+whsUG4lWa/0p0="
+  [mod."github.com/samber/lo"]
+    version = "v1.37.0"
+    hash = "sha256-5m0advnk/wy/bFygNyxaPVX2PgA5oF1lSHP71hAOhCc="
   [mod."github.com/sebest/xff"]
     version = "v0.0.0-20210106013422-671bd2870b3a"
     hash = "sha256-t7Gdp7S71Koej5Njg3jOo5vwyj02tJCbJjrq/vZGOho="
diff --git a/internal/internal.go b/internal/internal.go
index 0ac1981..8eedfab 100644
--- a/internal/internal.go
+++ b/internal/internal.go
@@ -10,7 +10,6 @@ import (
 
 	"github.com/posener/complete"
 	"go4.org/legal"
-	"within.website/ln"
 	"within.website/ln/opname"
 	"within.website/x/internal/confyg/flagconfyg"
 	"within.website/x/internal/flagenv"
@@ -33,7 +32,7 @@ var (
 func configFileLocation() string {
 	dir, err := os.UserConfigDir()
 	if err != nil {
-		ln.Error(context.Background(), err, ln.Debug("can't read config dir"))
+		//ln.Error(context.Background(), err, ln.Debug("can't read config dir"))
 		return ""
 	}
 
@@ -47,10 +46,10 @@ func configFileLocation() string {
 // This always loads from the following configuration sources in the following
 // order:
 //
-//     - command line flags (to get -config)
-//     - environment variables
-//     - configuration file (if -config is set)
-//     - command line flags
+//   - command line flags (to get -config)
+//   - environment variables
+//   - configuration file (if -config is set)
+//   - command line flags
 //
 // This is done this way to ensure that command line flags always are the deciding
 // factor as an escape hatch.
@@ -61,8 +60,6 @@ func HandleStartup() {
 	ctx := opname.With(context.Background(), "internal.HandleStartup")
 
 	if *config != "" {
-		ln.Log(ctx, ln.Info("loading config"), ln.F{"path": *config})
-
 		flagconfyg.CmdParse(ctx, *config)
 	}
 	flag.Parse()