From 97d1efae1e5acb3d39a6803eb3138812fe2ad91b Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Fri, 27 Oct 2023 09:01:37 -0400 Subject: cmd/tokenize: nuke failed experiment Signed-off-by: Xe Iaso --- cmd/tokenize/main.go | 50 -------------------------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 cmd/tokenize/main.go (limited to 'cmd/tokenize') diff --git a/cmd/tokenize/main.go b/cmd/tokenize/main.go deleted file mode 100644 index 4e0db40..0000000 --- a/cmd/tokenize/main.go +++ /dev/null @@ -1,50 +0,0 @@ -package main - -import ( - "encoding/json" - "flag" - "fmt" - "io" - "log" - "os" - - gpt3encoder "github.com/samber/go-gpt-3-encoder" - "within.website/x/internal" -) - -var ( - decode = flag.Bool("decode", false, "if true, decode instead of encode") -) - -func main() { - internal.HandleStartup() - - enc, err := gpt3encoder.NewEncoder() - if err != nil { - log.Fatal(err) - } - - if *decode { - var tokens []int - if err := json.NewDecoder(os.Stdin).Decode(&tokens); err != nil { - log.Fatal(err) - } - - fmt.Fprintln(os.Stdout, enc.Decode(tokens)) - return - } - - data, err := io.ReadAll(os.Stdin) - if err != nil { - log.Fatal(err) - } - - tokens, err := enc.Encode(string(data)) - if err != nil { - log.Fatal(err) - } - - if err := json.NewEncoder(os.Stdout).Encode(&tokens); err != nil { - log.Fatal(err) - } -} -- cgit v1.2.3