From c592b6d195aedcd6ec86e8c60d3ba91d524e293b Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Mon, 19 Jun 2023 12:56:31 -0400 Subject: second reshuffling Signed-off-by: Xe Iaso --- bundler/bundler.go | 401 +++++++++++++++++++++++++++++ bundler/bundler_test.go | 36 +++ cmd/marabot/main.go | 2 +- cmd/marabot/revolt.go | 2 +- cmd/within.website/main.go | 2 +- i18n/LICENSE.md | 21 -- i18n/README.md | 79 ------ i18n/doc.go | 2 - i18n/legal.go | 29 --- i18n/lingo.go | 132 ---------- i18n/lingo_test.go | 87 ------- i18n/locale.go | 101 -------- i18n/locale_test.go | 39 --- i18n/translations/de_DE.json | 32 --- i18n/translations/en_US.json | 32 --- i18n/translations/sr_RS.json | 32 --- internal/bundler/bundler.go | 401 ----------------------------- internal/bundler/bundler_test.go | 36 --- localca/LICENSE | 21 -- localca/doc.go | 10 - localca/legal.go | 27 -- localca/localca.go | 121 --------- localca/localca_test.go | 83 ------ localca/minica.go | 234 ----------------- localca/utils.go | 83 ------ misc/i18n/LICENSE.md | 21 ++ misc/i18n/README.md | 79 ++++++ misc/i18n/doc.go | 2 + misc/i18n/legal.go | 29 +++ misc/i18n/lingo.go | 132 ++++++++++ misc/i18n/lingo_test.go | 87 +++++++ misc/i18n/locale.go | 101 ++++++++ misc/i18n/locale_test.go | 39 +++ misc/i18n/translations/de_DE.json | 32 +++ misc/i18n/translations/en_US.json | 32 +++ misc/i18n/translations/sr_RS.json | 32 +++ misc/localca/LICENSE | 21 ++ misc/localca/doc.go | 10 + misc/localca/legal.go | 27 ++ misc/localca/localca.go | 121 +++++++++ misc/localca/localca_test.go | 83 ++++++ misc/localca/minica.go | 234 +++++++++++++++++ misc/localca/utils.go | 83 ++++++ misc/namegen/elfs/elfs.go | 511 +++++++++++++++++++++++++++++++++++++ misc/namegen/elfs/elfs_test.go | 11 + misc/namegen/namegen.go | 36 +++ misc/namegen/namegen_test.go | 11 + misc/namegen/tarot/doc.go | 3 + misc/namegen/tarot/namegen.go | 40 +++ misc/namegen/tarot/namegen_test.go | 10 + namegen/elfs/elfs.go | 511 ------------------------------------- namegen/elfs/elfs_test.go | 11 - namegen/namegen.go | 36 --- namegen/namegen_test.go | 11 - namegen/tarot/doc.go | 3 - namegen/tarot/namegen.go | 40 --- namegen/tarot/namegen_test.go | 10 - vanity/LICENSE | 41 --- vanity/legal.go | 47 ---- vanity/vanity.go | 188 -------------- web/vanity/LICENSE | 41 +++ web/vanity/legal.go | 47 ++++ web/vanity/vanity.go | 188 ++++++++++++++ 63 files changed, 2503 insertions(+), 2503 deletions(-) create mode 100644 bundler/bundler.go create mode 100644 bundler/bundler_test.go delete mode 100644 i18n/LICENSE.md delete mode 100644 i18n/README.md delete mode 100644 i18n/doc.go delete mode 100644 i18n/legal.go delete mode 100644 i18n/lingo.go delete mode 100644 i18n/lingo_test.go delete mode 100644 i18n/locale.go delete mode 100644 i18n/locale_test.go delete mode 100644 i18n/translations/de_DE.json delete mode 100644 i18n/translations/en_US.json delete mode 100644 i18n/translations/sr_RS.json delete mode 100644 internal/bundler/bundler.go delete mode 100644 internal/bundler/bundler_test.go delete mode 100644 localca/LICENSE delete mode 100644 localca/doc.go delete mode 100644 localca/legal.go delete mode 100644 localca/localca.go delete mode 100644 localca/localca_test.go delete mode 100644 localca/minica.go delete mode 100644 localca/utils.go create mode 100644 misc/i18n/LICENSE.md create mode 100644 misc/i18n/README.md create mode 100644 misc/i18n/doc.go create mode 100644 misc/i18n/legal.go create mode 100644 misc/i18n/lingo.go create mode 100644 misc/i18n/lingo_test.go create mode 100644 misc/i18n/locale.go create mode 100644 misc/i18n/locale_test.go create mode 100644 misc/i18n/translations/de_DE.json create mode 100644 misc/i18n/translations/en_US.json create mode 100644 misc/i18n/translations/sr_RS.json create mode 100644 misc/localca/LICENSE create mode 100644 misc/localca/doc.go create mode 100644 misc/localca/legal.go create mode 100644 misc/localca/localca.go create mode 100644 misc/localca/localca_test.go create mode 100644 misc/localca/minica.go create mode 100644 misc/localca/utils.go create mode 100644 misc/namegen/elfs/elfs.go create mode 100644 misc/namegen/elfs/elfs_test.go create mode 100644 misc/namegen/namegen.go create mode 100644 misc/namegen/namegen_test.go create mode 100644 misc/namegen/tarot/doc.go create mode 100644 misc/namegen/tarot/namegen.go create mode 100644 misc/namegen/tarot/namegen_test.go delete mode 100644 namegen/elfs/elfs.go delete mode 100644 namegen/elfs/elfs_test.go delete mode 100644 namegen/namegen.go delete mode 100644 namegen/namegen_test.go delete mode 100644 namegen/tarot/doc.go delete mode 100644 namegen/tarot/namegen.go delete mode 100644 namegen/tarot/namegen_test.go delete mode 100644 vanity/LICENSE delete mode 100644 vanity/legal.go delete mode 100644 vanity/vanity.go create mode 100644 web/vanity/LICENSE create mode 100644 web/vanity/legal.go create mode 100644 web/vanity/vanity.go diff --git a/bundler/bundler.go b/bundler/bundler.go new file mode 100644 index 0000000..7c623b5 --- /dev/null +++ b/bundler/bundler.go @@ -0,0 +1,401 @@ +// Copyright 2016 Google LLC. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bundler supports bundling (batching) of items. Bundling amortizes an +// action with fixed costs over multiple items. For example, if an API provides +// an RPC that accepts a list of items as input, but clients would prefer +// adding items one at a time, then a Bundler can accept individual items from +// the client and bundle many of them into a single RPC. +// +// This package is experimental and subject to change without notice. +package bundler + +import ( + "context" + "errors" + "sync" + "time" + + "golang.org/x/sync/semaphore" +) + +type mode int + +const ( + DefaultDelayThreshold = time.Second + DefaultBundleCountThreshold = 10 + DefaultBundleByteThreshold = 1e6 // 1M + DefaultBufferedByteLimit = 1e9 // 1G +) + +const ( + none mode = iota + add + addWait +) + +var ( + // ErrOverflow indicates that Bundler's stored bytes exceeds its BufferedByteLimit. + ErrOverflow = errors.New("bundler reached buffered byte limit") + + // ErrOversizedItem indicates that an item's size exceeds the maximum bundle size. + ErrOversizedItem = errors.New("item size exceeds bundle byte limit") + + // errMixedMethods indicates that mutually exclusive methods has been + // called subsequently. + errMixedMethods = errors.New("calls to Add and AddWait cannot be mixed") +) + +// A Bundler collects items added to it into a bundle until the bundle +// exceeds a given size, then calls a user-provided function to handle the +// bundle. +// +// The exported fields are only safe to modify prior to the first call to Add +// or AddWait. +type Bundler[T any] struct { + // Starting from the time that the first message is added to a bundle, once + // this delay has passed, handle the bundle. The default is DefaultDelayThreshold. + DelayThreshold time.Duration + + // Once a bundle has this many items, handle the bundle. Since only one + // item at a time is added to a bundle, no bundle will exceed this + // threshold, so it also serves as a limit. The default is + // DefaultBundleCountThreshold. + BundleCountThreshold int + + // Once the number of bytes in current bundle reaches this threshold, handle + // the bundle. The default is DefaultBundleByteThreshold. This triggers handling, + // but does not cap the total size of a bundle. + BundleByteThreshold int + + // The maximum size of a bundle, in bytes. Zero means unlimited. + BundleByteLimit int + + // The maximum number of bytes that the Bundler will keep in memory before + // returning ErrOverflow. The default is DefaultBufferedByteLimit. + BufferedByteLimit int + + // The maximum number of handler invocations that can be running at once. + // The default is 1. + HandlerLimit int + + handler func([]T) // called to handle a bundle + + mu sync.Mutex // guards access to fields below + flushTimer *time.Timer // implements DelayThreshold + handlerCount int // # of bundles currently being handled (i.e. handler is invoked on them) + sem *semaphore.Weighted // enforces BufferedByteLimit + semOnce sync.Once // guards semaphore initialization + // The current bundle we're adding items to. Not yet in the queue. + // Appended to the queue once the flushTimer fires or the bundle + // thresholds/limits are reached. If curBundle is nil and tail is + // not, we first try to add items to tail. Once tail is full or handled, + // we create a new curBundle for the incoming item. + curBundle *bundle[T] + // The next bundle in the queue to be handled. Nil if the queue is + // empty. + head *bundle[T] + // The last bundle in the queue to be handled. Nil if the queue is + // empty. If curBundle is nil and tail isn't, we attempt to add new + // items to the tail until if becomes full or has been passed to the + // handler. + tail *bundle[T] + curFlush *sync.WaitGroup // counts outstanding bundles since last flush + prevFlush chan bool // signal used to wait for prior flush + + // The first call to Add or AddWait, mode will be add or addWait respectively. + // If there wasn't call yet then mode is none. + mode mode + // TODO: consider alternative queue implementation for head/tail bundle. see: + // https://code-review.googlesource.com/c/google-api-go-client/+/47991/4/support/bundler/bundler.go#74 +} + +// A bundle is a group of items that were added individually and will be passed +// to a handler as a slice. +type bundle[T any] struct { + items []T // slice of T + size int // size in bytes of all items + next *bundle[T] // bundles are handled in order as a linked list queue + flush *sync.WaitGroup // the counter that tracks flush completion +} + +// add appends item to this bundle and increments the total size. It requires +// that b.mu is locked. +func (bu *bundle[T]) add(item T, size int) { + bu.items = append(bu.items, item) + bu.size += size +} + +// New creates a new Bundler. +// +// handler is a function that will be called on each bundle. If itemExample is +// of type T, the argument to handler is of type []T. handler is always called +// sequentially for each bundle, and never in parallel. +// +// Configure the Bundler by setting its thresholds and limits before calling +// any of its methods. +func New[T any](handler func([]T)) *Bundler[T] { + b := &Bundler[T]{ + DelayThreshold: DefaultDelayThreshold, + BundleCountThreshold: DefaultBundleCountThreshold, + BundleByteThreshold: DefaultBundleByteThreshold, + BufferedByteLimit: DefaultBufferedByteLimit, + HandlerLimit: 1, + + handler: handler, + curFlush: &sync.WaitGroup{}, + } + return b +} + +func (b *Bundler[T]) initSemaphores() { + // Create the semaphores lazily, because the user may set limits + // after NewBundler. + b.semOnce.Do(func() { + b.sem = semaphore.NewWeighted(int64(b.BufferedByteLimit)) + }) +} + +// enqueueCurBundle moves curBundle to the end of the queue. The bundle may be +// handled immediately if we are below HandlerLimit. It requires that b.mu is +// locked. +func (b *Bundler[T]) enqueueCurBundle() { + // We don't require callers to check if there is a pending bundle. It + // may have already been appended to the queue. If so, return early. + if b.curBundle == nil { + return + } + // If we are below the HandlerLimit, the queue must be empty. Handle + // immediately with a new goroutine. + if b.handlerCount < b.HandlerLimit { + b.handlerCount++ + go b.handle(b.curBundle) + } else if b.tail != nil { + // There are bundles on the queue, so append to the end + b.tail.next = b.curBundle + b.tail = b.curBundle + } else { + // The queue is empty, so initialize the queue + b.head = b.curBundle + b.tail = b.curBundle + } + b.curBundle = nil + if b.flushTimer != nil { + b.flushTimer.Stop() + b.flushTimer = nil + } +} + +// setMode sets the state of Bundler's mode. If mode was defined before +// and passed state is different from it then return an error. +func (b *Bundler[T]) setMode(m mode) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.mode == m || b.mode == none { + b.mode = m + return nil + } + return errMixedMethods +} + +// canFit returns true if bu can fit an additional item of size bytes based +// on the limits of Bundler b. +func (b *Bundler[T]) canFit(bu *bundle[T], size int) bool { + return (b.BundleByteLimit <= 0 || bu.size+size <= b.BundleByteLimit) && + (b.BundleCountThreshold <= 0 || len(bu.items) < b.BundleCountThreshold) +} + +// Add adds item to the current bundle. It marks the bundle for handling and +// starts a new one if any of the thresholds or limits are exceeded. +// The type of item must be assignable to the itemExample parameter of the NewBundler +// method, otherwise there will be a panic. +// +// If the item's size exceeds the maximum bundle size (Bundler.BundleByteLimit), then +// the item can never be handled. Add returns ErrOversizedItem in this case. +// +// If adding the item would exceed the maximum memory allowed +// (Bundler.BufferedByteLimit) or an AddWait call is blocked waiting for +// memory, Add returns ErrOverflow. +// +// Add never blocks. +func (b *Bundler[T]) Add(item T, size int) error { + if err := b.setMode(add); err != nil { + return err + } + // If this item exceeds the maximum size of a bundle, + // we can never send it. + if b.BundleByteLimit > 0 && size > b.BundleByteLimit { + return ErrOversizedItem + } + + // If adding this item would exceed our allotted memory + // footprint, we can't accept it. + // (TryAcquire also returns false if anything is waiting on the semaphore, + // so calls to Add and AddWait shouldn't be mixed.) + b.initSemaphores() + if !b.sem.TryAcquire(int64(size)) { + return ErrOverflow + } + + b.mu.Lock() + defer b.mu.Unlock() + return b.add(item, size) +} + +// add adds item to the tail of the bundle queue or curBundle depending on space +// and nil-ness (see inline comments). It marks curBundle for handling (by +// appending it to the queue) if any of the thresholds or limits are exceeded. +// curBundle is lazily initialized. It requires that b.mu is locked. +func (b *Bundler[T]) add(item T, size int) error { + // If we don't have a curBundle, see if we can add to the queue tail. + if b.tail != nil && b.curBundle == nil && b.canFit(b.tail, size) { + b.tail.add(item, size) + return nil + } + + // If we can't fit in the existing curBundle, move it onto the queue. + if b.curBundle != nil && !b.canFit(b.curBundle, size) { + b.enqueueCurBundle() + } + + // Create a curBundle if we don't have one. + if b.curBundle == nil { + b.curFlush.Add(1) + b.curBundle = &bundle[T]{ + items: []T{}, + flush: b.curFlush, + } + } + + // Add the item. + b.curBundle.add(item, size) + + // If curBundle is ready for handling, move it to the queue. + if b.curBundle.size >= b.BundleByteThreshold || + len(b.curBundle.items) == b.BundleCountThreshold { + b.enqueueCurBundle() + } + + // If we created a new bundle and it wasn't immediately handled, set a timer + if b.curBundle != nil && b.flushTimer == nil { + b.flushTimer = time.AfterFunc(b.DelayThreshold, b.tryHandleBundles) + } + + return nil +} + +// tryHandleBundles is the timer callback that handles or queues any current +// bundle after DelayThreshold time, even if the bundle isn't completely full. +func (b *Bundler[T]) tryHandleBundles() { + b.mu.Lock() + b.enqueueCurBundle() + b.mu.Unlock() +} + +// next returns the next bundle that is ready for handling and removes it from +// the internal queue. It requires that b.mu is locked. +func (b *Bundler[T]) next() *bundle[T] { + if b.head == nil { + return nil + } + out := b.head + b.head = b.head.next + if b.head == nil { + b.tail = nil + } + out.next = nil + return out +} + +// handle calls the user-specified handler on the given bundle. handle is +// intended to be run as a goroutine. After the handler returns, we update the +// byte total. handle continues processing additional bundles that are ready. +// If no more bundles are ready, the handler count is decremented and the +// goroutine ends. +func (b *Bundler[T]) handle(bu *bundle[T]) { + for bu != nil { + b.handler(bu.items) + bu = b.postHandle(bu) + } +} + +func (b *Bundler[T]) postHandle(bu *bundle[T]) *bundle[T] { + b.mu.Lock() + defer b.mu.Unlock() + + b.sem.Release(int64(bu.size)) + bu.flush.Done() + + bu = b.next() + if bu == nil { + b.handlerCount-- + } + return bu +} + +// AddWait adds item to the current bundle. It marks the bundle for handling and +// starts a new one if any of the thresholds or limits are exceeded. +// +// If the item's size exceeds the maximum bundle size (Bundler.BundleByteLimit), then +// the item can never be handled. AddWait returns ErrOversizedItem in this case. +// +// If adding the item would exceed the maximum memory allowed (Bundler.BufferedByteLimit), +// AddWait blocks until space is available or ctx is done. +// +// Calls to Add and AddWait should not be mixed on the same Bundler. +func (b *Bundler[T]) AddWait(ctx context.Context, item T, size int) error { + if err := b.setMode(addWait); err != nil { + return err + } + // If this item exceeds the maximum size of a bundle, + // we can never send it. + if b.BundleByteLimit > 0 && size > b.BundleByteLimit { + return ErrOversizedItem + } + // If adding this item would exceed our allotted memory footprint, block + // until space is available. The semaphore is FIFO, so there will be no + // starvation. + b.initSemaphores() + if err := b.sem.Acquire(ctx, int64(size)); err != nil { + return err + } + + b.mu.Lock() + defer b.mu.Unlock() + return b.add(item, size) +} + +// Flush invokes the handler for all remaining items in the Bundler and waits +// for it to return. +func (b *Bundler[T]) Flush() { + b.mu.Lock() + + // If a curBundle is pending, move it to the queue. + b.enqueueCurBundle() + + // Store a pointer to the WaitGroup that counts outstanding bundles + // in the current flush and create a new one to track the next flush. + wg := b.curFlush + b.curFlush = &sync.WaitGroup{} + + // Flush must wait for all prior, outstanding flushes to complete. + // We use a channel to communicate completion between each flush in + // the sequence. + prev := b.prevFlush + next := make(chan bool) + b.prevFlush = next + + b.mu.Unlock() + + // Wait until the previous flush is finished. + if prev != nil { + <-prev + } + + // Wait until this flush is finished. + wg.Wait() + + // Allow the next flush to finish. + close(next) +} diff --git a/bundler/bundler_test.go b/bundler/bundler_test.go new file mode 100644 index 0000000..9a269da --- /dev/null +++ b/bundler/bundler_test.go @@ -0,0 +1,36 @@ +package bundler + +import ( + "testing" +) + +func TestBundler(t *testing.T) { + input := []int{1, 2, 3, 4} + done := false + b := New[int](func(data []int) { + if len(data) != 4 { + t.Errorf("Wanted len(data) == %d, got: %d", len(input), len(data)) + } + + sum := 0 + const wantSum = 10 + for _, i := range data { + sum += i + } + + if sum != wantSum { + t.Errorf("wanted sum of inputs to be %d, got: %d", wantSum, sum) + } + done = true + }) + + for _, i := range input { + b.Add(i, 1) + } + + b.Flush() + + if !done { + t.Fatal("function wasn't called") + } +} diff --git a/cmd/marabot/main.go b/cmd/marabot/main.go index bbf6e65..612a4bc 100644 --- a/cmd/marabot/main.go +++ b/cmd/marabot/main.go @@ -25,8 +25,8 @@ import ( "tailscale.com/hostinfo" "within.website/ln" "within.website/ln/opname" + "within.website/x/bundler" "within.website/x/internal" - "within.website/x/internal/bundler" "within.website/x/web" "within.website/x/web/revolt" ) diff --git a/cmd/marabot/revolt.go b/cmd/marabot/revolt.go index 4588713..33d1d06 100644 --- a/cmd/marabot/revolt.go +++ b/cmd/marabot/revolt.go @@ -10,8 +10,8 @@ import ( "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/aws/aws-sdk-go/service/s3/s3manager" "within.website/ln" + "within.website/x/bundler" "within.website/x/internal" - "within.website/x/internal/bundler" "within.website/x/web/revolt" ) diff --git a/cmd/within.website/main.go b/cmd/within.website/main.go index c95da9a..cea40d6 100644 --- a/cmd/within.website/main.go +++ b/cmd/within.website/main.go @@ -12,7 +12,7 @@ import ( "within.website/ln/ex" "within.website/ln/opname" "within.website/x/internal" - "within.website/x/vanity" + "within.website/x/web/vanity" ) //go:generate go-bindata -pkg main static diff --git a/i18n/LICENSE.md b/i18n/LICENSE.md deleted file mode 100644 index c72833f..0000000 --- a/i18n/LICENSE.md +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2014 Dusan Lilic - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/i18n/README.md b/i18n/README.md deleted file mode 100644 index c260ea2..0000000 --- a/i18n/README.md +++ /dev/null @@ -1,79 +0,0 @@ -lingo -===== - -Very basic Golang library for i18n. There are others that do the job, but this is my take on the problem. - -Features: ---------- -1. Storing messages in JSON files. -2. Support for nested declarations. -2. Detecting language based on Request headers. -3. Very simple to use. - -Usage: ------- - 1. Import Lingo into your project - - ```go - import "github.com/kortem/lingo" - ``` - 1. Create a dir to store translations, and write them in JSON files named [locale].json. For example: - - ``` - en_US.json - sr_RS.json - de.json - ... - ``` - You can write nested JSON too. - ```json - { - "main.title" : "CutleryPlus", - "main.subtitle" : "Knives that put cut in cutlery.", - "menu" : { - "home" : "Home", - "products": { - "self": "Products", - "forks" : "Forks", - "knives" : "Knives", - "spoons" : "Spoons" - }, - } - } - ``` - 2. Initialize a Lingo like this: - - ```go - l := lingo.New("default_locale", "path/to/translations/dir") - ``` - - 3. Get bundle for specific locale via either `string`: - - ```go - t1 := l.TranslationsForLocale("en_US") - t2 := l.TranslationsForLocale("de_DE") - ``` - This way Lingo will return the bundle for specific locale, or default if given is not found. - Alternatively (or primarily), you can get it with `*http.Request`: - - ```go - t := l.TranslationsForRequest(req) - ``` - This way Lingo finds best suited locale via `Accept-Language` header, or if there is no match, returns default. - `Accept-Language` header is set by the browser, so basically it will serve the language the user has set to his browser. - 4. Once you get T instance just fire away! - - ```go - r1 := t1.Value("main.subtitle") - // "Knives that put cut in cutlery." - r1 := t2.Value("main.subtitle") - // "Messer, die legte in Besteck geschnitten." - r3 := t1.Value("menu.products.self") - // "Products" - r5 := t1.Value("error.404", req.URL.Path) - // "Page index.html not found!" - ``` - -Contributions: ------ -I regard this little library as feature-complete, but if you have an idea on how to improve it, feel free to create issues. Also, pull requests are welcome. Enjoy! diff --git a/i18n/doc.go b/i18n/doc.go deleted file mode 100644 index 2aa7275..0000000 --- a/i18n/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package i18n handles internationalization and the like for Go programs. -package i18n diff --git a/i18n/legal.go b/i18n/legal.go deleted file mode 100644 index 7f68dff..0000000 --- a/i18n/legal.go +++ /dev/null @@ -1,29 +0,0 @@ -package i18n - -import "go4.org/legal" - -func init() { - legal.RegisterLicense(`i18n library of this software copyright: - -The MIT License (MIT) - -Copyright (c) 2014 Dusan Lilic - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE.`) -} diff --git a/i18n/lingo.go b/i18n/lingo.go deleted file mode 100644 index 4824c01..0000000 --- a/i18n/lingo.go +++ /dev/null @@ -1,132 +0,0 @@ -package i18n - -import ( - "encoding/json" - "io/ioutil" - "log" - "net/http" - "strconv" - "strings" -) - -// L represents Lingo bundle, containing map of all Ts by locale, -// as well as default locale and list of supported locales -type L struct { - bundle map[string]T - deflt string - supported []Locale -} - -func (l *L) exists(locale string) bool { - _, exists := l.bundle[locale] - return exists -} - -// TranslationsForRequest will get the best matched T for given -// Request. If no T is found, returns default T -func (l *L) TranslationsForRequest(r *http.Request) T { - locales := GetLocales(r) - for _, locale := range locales { - t, exists := l.bundle[locales[0].Name()] - if exists { - return t - } - for _, sup := range l.supported { - if locale.Lang == sup.Lang { - return l.bundle[sup.Name()] - } - } - } - return l.bundle[l.deflt] -} - -// TranslationsForLocale will get the T for specific locale. -// If no locale is found, returns default T -func (l *L) TranslationsForLocale(locale string) T { - t, exists := l.bundle[locale] - if exists { - return t - } - return l.bundle[l.deflt] -} - -// T represents translations map for specific locale -type T struct { - transl map[string]interface{} -} - -// Value traverses the translations map and finds translation for -// given key. If no translation is found, returns value of given key. -func (t T) Value(key string, args ...string) string { - if t.exists(key) { - res, ok := t.transl[key].(string) - if ok { - return t.parseArgs(res, args) - } - } - ksplt := strings.Split(key, ".") - for i := range ksplt { - k1 := strings.Join(ksplt[0:i], ".") - k2 := strings.Join(ksplt[i:len(ksplt)], ".") - if t.exists(k1) { - newt := &T{ - transl: t.transl[k1].(map[string]interface{}), - } - return newt.Value(k2, args...) - } - } - return key -} - -// parseArgs replaces the argument placeholders with given arguments -func (t T) parseArgs(value string, args []string) string { - res := value - for i := 0; i < len(args); i++ { - tok := "{" + strconv.Itoa(i) + "}" - res = strings.Replace(res, tok, args[i], -1) - } - return res -} - -// exists checks if value exists for given key -func (t T) exists(key string) bool { - _, ok := t.transl[key] - return ok -} - -// New creates the Lingo bundle. -// Params: -// Default locale, to be used when requested locale -// is not found. -// Path, absolute or relative path to a folder where -// translation .json files are kept -func New(deflt, path string) *L { - files, _ := ioutil.ReadDir(path) - l := &L{ - bundle: make(map[string]T), - deflt: deflt, - supported: make([]Locale, 0), - } - for _, f := range files { - fileName := f.Name() - dat, err := ioutil.ReadFile(path + "/" + fileName) - if err != nil { - log.Printf("Cannot read file %s, file corrupt.", fileName) - log.Printf("Error: %s", err) - continue - } - t := T{ - transl: make(map[string]interface{}), - } - err = json.Unmarshal(dat, &t.transl) - if err != nil { - log.Printf("Cannot read file %s, invalid JSON.", fileName) - log.Printf("Error: %s", err) - continue - } - locale := strings.Split(fileName, ".")[0] - l.supported = append(l.supported, ParseLocale(locale)) - l.bundle[locale] = t - } - return l -} diff --git a/i18n/lingo_test.go b/i18n/lingo_test.go deleted file mode 100644 index ca1f559..0000000 --- a/i18n/lingo_test.go +++ /dev/null @@ -1,87 +0,0 @@ -package i18n - -import ( - "net/http" - "net/http/httptest" - "net/url" - "testing" -) - -func TestLingo(t *testing.T) { - l := New("de_DE", "translations") - t1 := l.TranslationsForLocale("en_US") - r1 := t1.Value("main.subtitle") - r1Exp := "Knives that put cut in cutlery." - if r1 != r1Exp { - t.Errorf("Expected \""+r1Exp+"\", got %s", r1) - t.Fail() - } - r2 := t1.Value("home.title") - r2Exp := "Welcome to CutleryPlus!" - if r2 != r2Exp { - t.Errorf("Expected \""+r2Exp+"\", got %s", r2) - t.Fail() - } - r3 := t1.Value("menu.products.self") - r3Exp := "Products" - if r3 != r3Exp { - t.Errorf("Expected \""+r3Exp+"\", got %s", r3) - t.Fail() - } - r4 := t1.Value("menu.non.existant") - r4Exp := "non.existant" - if r4 != r4Exp { - t.Errorf("Expected \""+r4Exp+"\", got %s", r4) - t.Fail() - } - r5 := t1.Value("error.404", "idnex.html") - r5Exp := "Page idnex.html not found!" - if r5 != r5Exp { - t.Errorf("Expected \""+r5Exp+"\", got \"%s\"", r5) - t.Fail() - } -} - -func TestLingoHttp(t *testing.T) { - l := New("en_US", "translations") - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - expected := r.Header.Get("Expected-Results") - t1 := l.TranslationsForRequest(r) - r1 := t1.Value("error.500") - if r1 != expected { - t.Errorf("Expected \""+expected+"\", got %s", r1) - t.Fail() - } - })) - defer srv.Close() - url, _ := url.Parse(srv.URL) - - req1 := &http.Request{ - Method: "GET", - Header: map[string][]string{ - "Accept-Language": {"sr, en-gb;q=0.8, en;q=0.7"}, - "Expected-Results": {"Greska sa nase strane, pokusajte ponovo."}, - }, - URL: url, - } - req2 := &http.Request{ - Method: "GET", - Header: map[string][]string{ - "Accept-Language": {"en-US, en-gb;q=0.8, en;q=0.7"}, - "Expected-Results": {"Something is wrong on our side, please try again."}, - }, - URL: url, - } - req3 := &http.Request{ - Method: "GET", - Header: map[string][]string{ - "Accept-Language": {"de-at, en-gb;q=0.8, en;q=0.7"}, - "Expected-Results": {"Stimmt etwas nicht auf unserer Seite ist, versuchen Sie es erneut."}, - }, - URL: url, - } - - http.DefaultClient.Do(req1) - http.DefaultClient.Do(req2) - http.DefaultClient.Do(req3) -} diff --git a/i18n/locale.go b/i18n/locale.go deleted file mode 100644 index 6b4a793..0000000 --- a/i18n/locale.go +++ /dev/null @@ -1,101 +0,0 @@ -package i18n - -import ( - "errors" - "net/http" - "strconv" - "strings" -) - -// Locale is locale value from the Accept-Language header in request -type Locale struct { - Lang, Country string - Qual float64 -} - -// Name returns the locale value in 'lang' or 'lang_country' format -// eg: de_DE, en_US, gb -func (l *Locale) Name() string { - if len(l.Country) > 0 { - return l.Lang + "_" + l.Country - } - return l.Lang -} - -// ParseLocale creates a Locale from a locale string -func ParseLocale(locale string) Locale { - locsplt := strings.Split(locale, "_") - resp := Locale{} - resp.Lang = locsplt[0] - if len(locsplt) > 1 { - resp.Country = locsplt[1] - } - return resp -} - -const ( - acceptLanguage = "Accept-Language" -) - -func supportedLocales(alstr string) []Locale { - locales := make([]Locale, 0) - alstr = strings.Replace(alstr, " ", "", -1) - if alstr == "" { - return locales - } - al := strings.Split(alstr, ",") - for _, lstr := range al { - locales = append(locales, Locale{ - Lang: parseLang(lstr), - Country: parseCountry(lstr), - Qual: parseQual(lstr), - }) - } - return locales -} - -// GetLocales returns supported locales for the given requet -func GetLocales(r *http.Request) []Locale { - return supportedLocales(r.Header.Get(acceptLanguage)) -} - -// GetPreferredLocale return preferred locale for the given reuqest -// returns error if there is no preferred locale -func GetPreferredLocale(r *http.Request) (*Locale, error) { - locales := GetLocales(r) - if len(locales) == 0 { - return &Locale{}, errors.New("No locale found") - } - return &locales[0], nil -} - -func parseLang(val string) string { - locale := strings.Split(val, ";")[0] - lang := strings.Split(locale, "-")[0] - return lang -} - -func parseCountry(val string) string { - locale := strings.Split(val, ";")[0] - spl := strings.Split(locale, "-") - if len(spl) > 1 { - return spl[1] - } - return "" -} - -func parseQual(val string) float64 { - spl := strings.Split(val, ";") - if len(spl) > 1 { - qualSpl := strings.Split(spl[1], "=") - if len(qualSpl) > 1 { - qual, err := strconv.ParseFloat(qualSpl[1], 64) - if err != nil { - return 1 - } - return qual - } - } - return 1 -} - diff --git a/i18n/locale_test.go b/i18n/locale_test.go deleted file mode 100644 index 6887064..0000000 --- a/i18n/locale_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package i18n - -import ( - "testing" -) - -func TestLocale(t *testing.T) { - l0 := supportedLocales("ja-JP;q") - if len(l0) != 1 { - t.Errorf("Expected number of locales \"1\", got %d", len(l0)) - t.Fail() - } - l1 := supportedLocales("en,de-AT; q=0.8,de;q=0.6,bg; q=0.4,en-US;q=0.2,sr;q=0.2") - if len(l1) != 6 { - t.Errorf("Expected number of locales \"6\", got %d", len(l1)) - t.Fail() - } - l2 := supportedLocales("en") - if len(l2) != 1 { - t.Errorf("Expected number of locales \"1\", got %d", len(l2)) - t.Fail() - } - l3 := supportedLocales("") - if len(l3) != 0 { - t.Errorf("Expected number of locales \"0\", got %d", len(l3)) - t.Fail() - } - l4 := ParseLocale("en_US") - if l4.Lang != "en" || l4.Country != "US" { - t.Errorf("Expected \"en\" and \"US\", got %s and %s", l4.Lang, l4.Country) - t.Fail() - } - l5 := ParseLocale("en") - if l5.Lang != "en" || l5.Country != "" { - t.Errorf("Expected \"en\" and \"\", got %s and %s", l5.Lang, l5.Country) - t.Fail() - } - -} diff --git a/i18n/translations/de_DE.json b/i18n/translations/de_DE.json deleted file mode 100644 index c12186d..0000000 --- a/i18n/translations/de_DE.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "main.title" : "CutleryPlus", - "main.subtitle" : "Messer, die legte in Besteck geschnitten.", - "menu" : { - "home" : "Home", - "products": { - "self": "Produkte", - "forks" : "Gabeln", - "knives" : "Messer", - "spoons" : "Löffel" - }, - "gallery" : "Galerie", - "about" : "Über uns", - "contact" : "Kontakt" - }, - "home" : { - "title": "Willkommen in CutleryPlus!", - "text" : { - "p1": "Lorem ipsum...", - "p2": "Ein weiterer ipsum lorem." - } - }, - "error" : { - "404" : "Seite {0} wurde nicht gefunden.", - "500" : "Stimmt etwas nicht auf unserer Seite ist, versuchen Sie es erneut.", - "contact" : { - "name" : "Sie müssen Ihren Namen eingeben.", - "email" : "Sie müssen Ihre E-Mail ein.", - "text" : "Sie können eine leere Nachricht nicht zu senden." - } - } -} \ No newline at end of file diff --git a/i18n/translations/en_US.json b/i18n/translations/en_US.json deleted file mode 100644 index 6fa0b8f..0000000 --- a/i18n/translations/en_US.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "main.title" : "CutleryPlus", - "main.subtitle" : "Knives that put cut in cutlery.", - "menu" : { - "home" : "Home", - "products": { - "self": "Products", - "forks" : "Forks", - "knives" : "Knives", - "spoons" : "Spoons" - }, - "gallery" : "Gallery", - "about" : "About us", - "contact" : "Contact" - }, - "home" : { - "title": "Welcome to CutleryPlus!", - "text" : { - "p1": "Lorem ipsum...", - "p2": "Another ipsum lorem." - } - }, - "error" : { - "404" : "Page {0} not found!", - "500" : "Something is wrong on our side, please try again.", - "contact" : { - "name" : "You must enter your name.", - "email" : "You must enter your email.", - "text" : "You cannot send an empty message." - } - } -} \ No newline at end of file diff --git a/i18n/translations/sr_RS.json b/i18n/translations/sr_RS.json deleted file mode 100644 index fe2bea5..0000000 --- a/i18n/translations/sr_RS.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "main.title" : "CutleryPlus", - "main.subtitle" : "Escajg za svakoga", - "menu" : { - "home" : "Pocetna", - "products": { - "self": "Proizvodi", - "forks" : "Viljuske", - "knives" : "Nozevi", - "spoons" : "Kasike" - }, - "gallery" : "Galerija", - "about" : "O nama", - "contact" : "Kontakt" - }, - "home" : { - "title": "Dobrodosli u CutleryPlus!", - "text" : { - "p1": "Lorem ipsum...", - "p2": "Jos jedan ipsum lorem." - } - }, - "error" : { - "404" : "Stranica {0} ne postoji.", - "500" : "Greska sa nase strane, pokusajte ponovo.", - "contact" : { - "name" : "Ime ne sme biti prazno.", - "email" : "Email ne sme biti prazan.", - "text" : "Ne mozete poslati praznu poruku." - } - } -} \ No newline at end of file diff --git a/internal/bundler/bundler.go b/internal/bundler/bundler.go deleted file mode 100644 index 7c623b5..0000000 --- a/internal/bundler/bundler.go +++ /dev/null @@ -1,401 +0,0 @@ -// Copyright 2016 Google LLC. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package bundler supports bundling (batching) of items. Bundling amortizes an -// action with fixed costs over multiple items. For example, if an API provides -// an RPC that accepts a list of items as input, but clients would prefer -// adding items one at a time, then a Bundler can accept individual items from -// the client and bundle many of them into a single RPC. -// -// This package is experimental and subject to change without notice. -package bundler - -import ( - "context" - "errors" - "sync" - "time" - - "golang.org/x/sync/semaphore" -) - -type mode int - -const ( - DefaultDelayThreshold = time.Second - DefaultBundleCountThreshold = 10 - DefaultBundleByteThreshold = 1e6 // 1M - DefaultBufferedByteLimit = 1e9 // 1G -) - -const ( - none mode = iota - add - addWait -) - -var ( - // ErrOverflow indicates that Bundler's stored bytes exceeds its BufferedByteLimit. - ErrOverflow = errors.New("bundler reached buffered byte limit") - - // ErrOversizedItem indicates that an item's size exceeds the maximum bundle size. - ErrOversizedItem = errors.New("item size exceeds bundle byte limit") - - // errMixedMethods indicates that mutually exclusive methods has been - // called subsequently. - errMixedMethods = errors.New("calls to Add and AddWait cannot be mixed") -) - -// A Bundler collects items added to it into a bundle until the bundle -// exceeds a given size, then calls a user-provided function to handle the -// bundle. -// -// The exported fields are only safe to modify prior to the first call to Add -// or AddWait. -type Bundler[T any] struct { - // Starting from the time that the first message is added to a bundle, once - // this delay has passed, handle the bundle. The default is DefaultDelayThreshold. - DelayThreshold time.Duration - - // Once a bundle has this many items, handle the bundle. Since only one - // item at a time is added to a bundle, no bundle will exceed this - // threshold, so it also serves as a limit. The default is - // DefaultBundleCountThreshold. - BundleCountThreshold int - - // Once the number of bytes in current bundle reaches this threshold, handle - // the bundle. The default is DefaultBundleByteThreshold. This triggers handling, - // but does not cap the total size of a bundle. - BundleByteThreshold int - - // The maximum size of a bundle, in bytes. Zero means unlimited. - BundleByteLimit int - - // The maximum number of bytes that the Bundler will keep in memory before - // returning ErrOverflow. The default is DefaultBufferedByteLimit. - BufferedByteLimit int - - // The maximum number of handler invocations that can be running at once. - // The default is 1. - HandlerLimit int - - handler func([]T) // called to handle a bundle - - mu sync.Mutex // guards access to fields below - flushTimer *time.Timer // implements DelayThreshold - handlerCount int // # of bundles currently being handled (i.e. handler is invoked on them) - sem *semaphore.Weighted // enforces BufferedByteLimit - semOnce sync.Once // guards semaphore initialization - // The current bundle we're adding items to. Not yet in the queue. - // Appended to the queue once the flushTimer fires or the bundle - // thresholds/limits are reached. If curBundle is nil and tail is - // not, we first try to add items to tail. Once tail is full or handled, - // we create a new curBundle for the incoming item. - curBundle *bundle[T] - // The next bundle in the queue to be handled. Nil if the queue is - // empty. - head *bundle[T] - // The last bundle in the queue to be handled. Nil if the queue is - // empty. If curBundle is nil and tail isn't, we attempt to add new - // items to the tail until if becomes full or has been passed to the - // handler. - tail *bundle[T] - curFlush *sync.WaitGroup // counts outstanding bundles since last flush - prevFlush chan bool // signal used to wait for prior flush - - // The first call to Add or AddWait, mode will be add or addWait respectively. - // If there wasn't call yet then mode is none. - mode mode - // TODO: consider alternative queue implementation for head/tail bundle. see: - // https://code-review.googlesource.com/c/google-api-go-client/+/47991/4/support/bundler/bundler.go#74 -} - -// A bundle is a group of items that were added individually and will be passed -// to a handler as a slice. -type bundle[T any] struct { - items []T // slice of T - size int // size in bytes of all items - next *bundle[T] // bundles are handled in order as a linked list queue - flush *sync.WaitGroup // the counter that tracks flush completion -} - -// add appends item to this bundle and increments the total size. It requires -// that b.mu is locked. -func (bu *bundle[T]) add(item T, size int) { - bu.items = append(bu.items, item) - bu.size += size -} - -// New creates a new Bundler. -// -// handler is a function that will be called on each bundle. If itemExample is -// of type T, the argument to handler is of type []T. handler is always called -// sequentially for each bundle, and never in parallel. -// -// Configure the Bundler by setting its thresholds and limits before calling -// any of its methods. -func New[T any](handler func([]T)) *Bundler[T] { - b := &Bundler[T]{ - DelayThreshold: DefaultDelayThreshold, - BundleCountThreshold: DefaultBundleCountThreshold, - BundleByteThreshold: DefaultBundleByteThreshold, - BufferedByteLimit: DefaultBufferedByteLimit, - HandlerLimit: 1, - - handler: handler, - curFlush: &sync.WaitGroup{}, - } - return b -} - -func (b *Bundler[T]) initSemaphores() { - // Create the semaphores lazily, because the user may set limits - // after NewBundler. - b.semOnce.Do(func() { - b.sem = semaphore.NewWeighted(int64(b.BufferedByteLimit)) - }) -} - -// enqueueCurBundle moves curBundle to the end of the queue. The bundle may be -// handled immediately if we are below HandlerLimit. It requires that b.mu is -// locked. -func (b *Bundler[T]) enqueueCurBundle() { - // We don't require callers to check if there is a pending bundle. It - // may have already been appended to the queue. If so, return early. - if b.curBundle == nil { - return - } - // If we are below the HandlerLimit, the queue must be empty. Handle - // immediately with a new goroutine. - if b.handlerCount < b.HandlerLimit { - b.handlerCount++ - go b.handle(b.curBundle) - } else if b.tail != nil { - // There are bundles on the queue, so append to the end - b.tail.next = b.curBundle - b.tail = b.curBundle - } else { - // The queue is empty, so initialize the queue - b.head = b.curBundle - b.tail = b.curBundle - } - b.curBundle = nil - if b.flushTimer != nil { - b.flushTimer.Stop() - b.flushTimer = nil - } -} - -// setMode sets the state of Bundler's mode. If mode was defined before -// and passed state is different from it then return an error. -func (b *Bundler[T]) setMode(m mode) error { - b.mu.Lock() - defer b.mu.Unlock() - if b.mode == m || b.mode == none { - b.mode = m - return nil - } - return errMixedMethods -} - -// canFit returns true if bu can fit an additional item of size bytes based -// on the limits of Bundler b. -func (b *Bundler[T]) canFit(bu *bundle[T], size int) bool { - return (b.BundleByteLimit <= 0 || bu.size+size <= b.BundleByteLimit) && - (b.BundleCountThreshold <= 0 || len(bu.items) < b.BundleCountThreshold) -} - -// Add adds item to the current bundle. It marks the bundle for handling and -// starts a new one if any of the thresholds or limits are exceeded. -// The type of item must be assignable to the itemExample parameter of the NewBundler -// method, otherwise there will be a panic. -// -// If the item's size exceeds the maximum bundle size (Bundler.BundleByteLimit), then -// the item can never be handled. Add returns ErrOversizedItem in this case. -// -// If adding the item would exceed the maximum memory allowed -// (Bundler.BufferedByteLimit) or an AddWait call is blocked waiting for -// memory, Add returns ErrOverflow. -// -// Add never blocks. -func (b *Bundler[T]) Add(item T, size int) error { - if err := b.setMode(add); err != nil { - return err - } - // If this item exceeds the maximum size of a bundle, - // we can never send it. - if b.BundleByteLimit > 0 && size > b.BundleByteLimit { - return ErrOversizedItem - } - - // If adding this item would exceed our allotted memory - // footprint, we can't accept it. - // (TryAcquire also returns false if anything is waiting on the semaphore, - // so calls to Add and AddWait shouldn't be mixed.) - b.initSemaphores() - if !b.sem.TryAcquire(int64(size)) { - return ErrOverflow - } - - b.mu.Lock() - defer b.mu.Unlock() - return b.add(item, size) -} - -// add adds item to the tail of the bundle queue or curBundle depending on space -// and nil-ness (see inline comments). It marks curBundle for handling (by -// appending it to the queue) if any of the thresholds or limits are exceeded. -// curBundle is lazily initialized. It requires that b.mu is locked. -func (b *Bundler[T]) add(item T, size int) error { - // If we don't have a curBundle, see if we can add to the queue tail. - if b.tail != nil && b.curBundle == nil && b.canFit(b.tail, size) { - b.tail.add(item, size) - return nil - } - - // If we can't fit in the existing curBundle, move it onto the queue. - if b.curBundle != nil && !b.canFit(b.curBundle, size) { - b.enqueueCurBundle() - } - - // Create a curBundle if we don't have one. - if b.curBundle == nil { - b.curFlush.Add(1) - b.curBundle = &bundle[T]{ - items: []T{}, - flush: b.curFlush, - } - } - - // Add the item. - b.curBundle.add(item, size) - - // If curBundle is ready for handling, move it to the queue. - if b.curBundle.size >= b.BundleByteThreshold || - len(b.curBundle.items) == b.BundleCountThreshold { - b.enqueueCurBundle() - } - - // If we created a new bundle and it wasn't immediately handled, set a timer - if b.curBundle != nil && b.flushTimer == nil { - b.flushTimer = time.AfterFunc(b.DelayThreshold, b.tryHandleBundles) - } - - return nil -} - -// tryHandleBundles is the timer callback that handles or queues any current -// bundle after DelayThreshold time, even if the bundle isn't completely full. -func (b *Bundler[T]) tryHandleBundles() { - b.mu.Lock() - b.enqueueCurBundle() - b.mu.Unlock() -} - -// next returns the next bundle that is ready for handling and removes it from -// the internal queue. It requires that b.mu is locked. -func (b *Bundler[T]) next() *bundle[T] { - if b.head == nil { - return nil - } - out := b.head - b.head = b.head.next - if b.head == nil { - b.tail = nil - } - out.next = nil - return out -} - -// handle calls the user-specified handler on the given bundle. handle is -// intended to be run as a goroutine. After the handler returns, we update the -// byte total. handle continues processing additional bundles that are ready. -// If no more bundles are ready, the handler count is decremented and the -// goroutine ends. -func (b *Bundler[T]) handle(bu *bundle[T]) { - for bu != nil { - b.handler(bu.items) - bu = b.postHandle(bu) - } -} - -func (b *Bundler[T]) postHandle(bu *bundle[T]) *bundle[T] { - b.mu.Lock() - defer b.mu.Unlock() - - b.sem.Release(int64(bu.size)) - bu.flush.Done() - - bu = b.next() - if bu == nil { - b.handlerCount-- - } - return bu -} - -// AddWait adds item to the current bundle. It marks the bundle for handling and -// starts a new one if any of the thresholds or limits are exceeded. -// -// If the item's size exceeds the maximum bundle size (Bundler.BundleByteLimit), then -// the item can never be handled. AddWait returns ErrOversizedItem in this case. -// -// If adding the item would exceed the maximum memory allowed (Bundler.BufferedByteLimit), -// AddWait blocks until space is available or ctx is done. -// -// Calls to Add and AddWait should not be mixed on the same Bundler. -func (b *Bundler[T]) AddWait(ctx context.Context, item T, size int) error { - if err := b.setMode(addWait); err != nil { - return err - } - // If this item exceeds the maximum size of a bundle, - // we can never send it. - if b.BundleByteLimit > 0 && size > b.BundleByteLimit { - return ErrOversizedItem - } - // If adding this item would exceed our allotted memory footprint, block - // until space is available. The semaphore is FIFO, so there will be no - // starvation. - b.initSemaphores() - if err := b.sem.Acquire(ctx, int64(size)); err != nil { - return err - } - - b.mu.Lock() - defer b.mu.Unlock() - return b.add(item, size) -} - -// Flush invokes the handler for all remaining items in the Bundler and waits -// for it to return. -func (b *Bundler[T]) Flush() { - b.mu.Lock() - - // If a curBundle is pending, move it to the queue. - b.enqueueCurBundle() - - // Store a pointer to the WaitGroup that counts outstanding bundles - // in the current flush and create a new one to track the next flush. - wg := b.curFlush - b.curFlush = &sync.WaitGroup{} - - // Flush must wait for all prior, outstanding flushes to complete. - // We use a channel to communicate completion between each flush in - // the sequence. - prev := b.prevFlush - next := make(chan bool) - b.prevFlush = next - - b.mu.Unlock() - - // Wait until the previous flush is finished. - if prev != nil { - <-prev - } - - // Wait until this flush is finished. - wg.Wait() - - // Allow the next flush to finish. - close(next) -} diff --git a/internal/bundler/bundler_test.go b/internal/bundler/bundler_test.go deleted file mode 100644 index 9a269da..0000000 --- a/internal/bundler/bundler_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package bundler - -import ( - "testing" -) - -func TestBundler(t *testing.T) { - input := []int{1, 2, 3, 4} - done := false - b := New[int](func(data []int) { - if len(data) != 4 { - t.Errorf("Wanted len(data) == %d, got: %d", len(input), len(data)) - } - - sum := 0 - const wantSum = 10 - for _, i := range data { - sum += i - } - - if sum != wantSum { - t.Errorf("wanted sum of inputs to be %d, got: %d", wantSum, sum) - } - done = true - }) - - for _, i := range input { - b.Add(i, 1) - } - - b.Flush() - - if !done { - t.Fatal("function wasn't called") - } -} diff --git a/localca/LICENSE b/localca/LICENSE deleted file mode 100644 index 6aa3d70..0000000 --- a/localca/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2016 Jacob Hoffman-Andrews - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/localca/doc.go b/localca/doc.go deleted file mode 100644 index fb4e829..0000000 --- a/localca/doc.go +++ /dev/null @@ -1,10 +0,0 @@ -// Package localca uses an autocert.Cache to store and generate TLS certificates -// for domains on demand. -// -// This is kind of powerful, and as such it is limited to only generate -// certificates as subdomains of a given domain. -// -// The design and implementation of this is kinda stolen from minica[1]. -// -// [1]: https://github.com/jsha/minica -package localca diff --git a/localca/legal.go b/localca/legal.go deleted file mode 100644 index 9d35318..0000000 --- a/localca/legal.go +++ /dev/null @@ -1,27 +0,0 @@ -package localca - -import "go4.org/legal" - -func init() { - legal.RegisterLicense(`MIT License - -Copyright (c) 2016 Jacob Hoffman-Andrews - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE.`) -} diff --git a/localca/localca.go b/localca/localca.go deleted file mode 100644 index 360b53f..0000000 --- a/localca/localca.go +++ /dev/null @@ -1,121 +0,0 @@ -package localca - -import ( - "context" - "crypto/tls" - "encoding/pem" - "errors" - "strings" - "time" - - "golang.org/x/crypto/acme/autocert" -) - -var ( - ErrBadData = errors.New("localca: certificate data is bad") - ErrDomainDoesntHaveSuffix = errors.New("localca: domain doesn't have the given suffix") -) - -// Manager automatically provisions and caches TLS certificates in a given -// autocert Cache. If it cannot fetch a certificate on demand, the certificate -// is dynamically generated with a lifetime of 100 years, which should be good -// enough. -type Manager struct { - Cache autocert.Cache - DomainSuffix string - - *issuer -} - -// New creates a new Manager with the given key filename, certificate filename, -// allowed domain suffix and autocert cache. All given certificates will be -// created if they don't already exist. -func New(keyFile, certFile, suffix string, cache autocert.Cache) (Manager, error) { - iss, err := getIssuer(keyFile, certFile, true) - - if err != nil { - return Manager{}, err - } - - result := Manager{ - DomainSuffix: suffix, - Cache: cache, - issuer: iss, - } - - return result, nil -} - -func (m Manager) GetCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { - name := hello.ServerName - if !strings.Contains(strings.Trim(name, "."), ".") { - return nil, errors.New("localca: server name component count invalid") - } - - if !strings.HasSuffix(name, m.DomainSuffix) { - return nil, ErrDomainDoesntHaveSuffix - } - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - - data, err := m.Cache.Get(ctx, name) - if err != nil && err != autocert.ErrCacheMiss { - return nil, err - } - - if err == autocert.ErrCacheMiss { - data, _, err = m.issuer.sign([]string{name}, nil) - if err != nil { - return nil, err - } - err = m.Cache.Put(ctx, name, data) - if err != nil { - return nil, err - } - } - - cert, err := loadCertificate(name, data) - if err != nil { - return nil, err - } - - return cert, nil -} - -func loadCertificate(name string, data []byte) (*tls.Certificate, error) { - priv, pub := pem.Decode(data) - if priv == nil || !strings.Contains(priv.Type, "PRIVATE") { - return nil, ErrBadData - } - privKey, err := parsePrivateKey(priv.Bytes) - if err != nil { - return nil, err - } - - // public - var pubDER [][]byte - for len(pub) > 0 { - var b *pem.Block - b, pub = pem.Decode(pub) - if b == nil { - break - } - pubDER = append(pubDER, b.Bytes) - } - if len(pub) > 0 { - return nil, ErrBadData - } - - // verify and create TLS cert - leaf, err := validCert(name, pubDER, privKey, time.Now()) - if err != nil { - return nil, err - } - tlscert := &tls.Certificate{ - Certificate: pubDER, - PrivateKey: privKey, - Leaf: leaf, - } - return tlscert, nil -} diff --git a/localca/localca_test.go b/localca/localca_test.go deleted file mode 100644 index 0c85fba..0000000 --- a/localca/localca_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package localca - -import ( - "context" - "crypto/tls" - "io" - "io/ioutil" - "os" - "path" - "testing" - "time" - - "golang.org/x/crypto/acme/autocert" -) - -func TestLocalCA(t *testing.T) { - dir, err := ioutil.TempDir("", "localca-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - cache := autocert.DirCache(dir) - - keyFile := path.Join(dir, "key.pem") - certFile := path.Join(dir, "cert.pem") - const suffix = "club" - - m, err := New(keyFile, certFile, suffix, cache) - if err != nil { - t.Fatal(err) - } - - t.Run("local", func(t *testing.T) { - _, err = m.GetCertificate(&tls.ClientHelloInfo{ - ServerName: "foo.local.cetacean.club", - }) - if err != nil { - t.Fatal(err) - } - }) - - t.Run("network", func(t *testing.T) { - t.Skip("no") - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - tc := &tls.Config{ - GetCertificate: m.GetCertificate, - } - - go func() { - lis, err := tls.Listen("tcp", ":9293", tc) - if err != nil { - t.Fatal(err) - } - defer lis.Close() - - for { - select { - case <-ctx.Done(): - return - default: - } - - cli, err := lis.Accept() - if err != nil { - t.Fatal(err) - } - defer cli.Close() - - go io.Copy(cli, cli) - } - }() - - time.Sleep(130 * time.Millisecond) - cli, err := tls.Dial("tcp", "localhost:9293", &tls.Config{InsecureSkipVerify: true}) - if err != nil { - t.Fatal(err) - } - defer cli.Close() - - cli.Write([]byte("butts")) - }) -} diff --git a/localca/minica.go b/localca/minica.go deleted file mode 100644 index d47330b..0000000 --- a/localca/minica.go +++ /dev/null @@ -1,234 +0,0 @@ -package localca - -import ( - "bytes" - "crypto" - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "crypto/x509/pkix" - "encoding/hex" - "encoding/pem" - "fmt" - "io/ioutil" - "math" - "math/big" - "net" - "os" - "strings" - "time" -) - -type issuer struct { - key crypto.Signer - cert *x509.Certificate -} - -func getIssuer(keyFile, certFile string, autoCreate bool) (*issuer, error) { - keyContents, keyErr := ioutil.ReadFile(keyFile) - certContents, certErr := ioutil.ReadFile(certFile) - if os.IsNotExist(keyErr) && os.IsNotExist(certErr) { - err := makeIssuer(keyFile, certFile) - if err != nil { - return nil, err - } - return getIssuer(keyFile, certFile, false) - } else if keyErr != nil { - return nil, fmt.Errorf("%s (but %s exists)", keyErr, certFile) - } else if certErr != nil { - return nil, fmt.Errorf("%s (but %s exists)", certErr, keyFile) - } - key, err := readPrivateKey(keyContents) - if err != nil { - return nil, fmt.Errorf("reading private key from %s: %s", keyFile, err) - } - - cert, err := readCert(certContents) - if err != nil { - return nil, fmt.Errorf("reading CA certificate from %s: %s", certFile, err) - } - - equal, err := publicKeysEqual(key.Public(), cert.PublicKey) - if err != nil { - return nil, fmt.Errorf("comparing public keys: %s", err) - } else if !equal { - return nil, fmt.Errorf("public key in CA certificate %s doesn't match private key in %s", - certFile, keyFile) - } - return &issuer{key, cert}, nil -} - -func readPrivateKey(keyContents []byte) (crypto.Signer, error) { - block, _ := pem.Decode(keyContents) - if block == nil { - return nil, fmt.Errorf("no PEM found") - } else if block.Type != "RSA PRIVATE KEY" && block.Type != "ECDSA PRIVATE KEY" { - return nil, fmt.Errorf("incorrect PEM type %s", block.Type) - } - return x509.ParsePKCS1PrivateKey(block.Bytes) -} - -func readCert(certContents []byte) (*x509.Certificate, error) { - block, _ := pem.Decode(certContents) - if block == nil { - return nil, fmt.Errorf("no PEM found") - } else if block.Type != "CERTIFICATE" { - return nil, fmt.Errorf("incorrect PEM type %s", block.Type) - } - return x509.ParseCertificate(block.Bytes) -} - -func makeIssuer(keyFile, certFile string) error { - keyData, key, err := makeKey() - if err != nil { - return err - } - ioutil.WriteFile(keyFile, keyData, 0600) - certData, _, err := makeRootCert(key, certFile) - if err != nil { - return err - } - ioutil.WriteFile(certFile, certData, 0600) - return nil -} - -func makeKey() ([]byte, *rsa.PrivateKey, error) { - key, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - return nil, nil, err - } - der := x509.MarshalPKCS1PrivateKey(key) - if err != nil { - return nil, nil, err - } - buf := bytes.NewBuffer([]byte{}) - err = pem.Encode(buf, &pem.Block{ - Type: "RSA PRIVATE KEY", - Bytes: der, - }) - if err != nil { - return nil, nil, err - } - return buf.Bytes(), key, nil -} - -func makeRootCert(key crypto.Signer, filename string) ([]byte, *x509.Certificate, error) { - serial, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64)) - if err != nil { - return nil, nil, err - } - template := &x509.Certificate{ - Subject: pkix.Name{ - CommonName: "localca root ca " + hex.EncodeToString(serial.Bytes()[:3]), - },