1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
|
package lume
import (
"archive/zip"
"compress/gzip"
"fmt"
"io"
"log/slog"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"xeiaso.net/v4/internal"
)
const (
compressionGZIP = 0x69
)
func init() {
zip.RegisterCompressor(compressionGZIP, func(w io.Writer) (io.WriteCloser, error) {
return gzip.NewWriterLevel(w, gzip.BestCompression)
})
zip.RegisterDecompressor(compressionGZIP, func(r io.Reader) io.ReadCloser {
rdr, err := gzip.NewReader(r)
if err != nil {
slog.Error("can't read from gzip stream", "err", err)
panic(err)
}
return rdr
})
}
// ZipFolder takes a source folder and a target zip file name
// and compresses the folder contents into the zip file
func ZipFolder(source, target string) error {
// Create a zip file
fout, err := os.Create(target)
if err != nil {
return err
}
defer fout.Close()
// Create a zip writer
w := zip.NewWriter(fout)
defer w.Close()
// Walk through the source folder
return filepath.Walk(source, func(path string, info os.FileInfo, err error) error {
// Handle errors
if err != nil {
return err
}
// Skip directories
if info.IsDir() {
return nil
}
// Create a header from the file info
header, err := zip.FileInfoHeader(info)
if err != nil {
return err
}
compressible, err := isCompressible(path)
if err != nil {
return err
}
if compressible {
header.Method = compressionGZIP
}
// Open the file
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
// Set the header name to the relative path of the file
header.Name, err = filepath.Rel(source, path)
if err != nil {
return err
}
// Create a fout for the file header
fout, err := w.CreateHeader(header)
if err != nil {
return err
}
// Copy the file contents to the writer
_, err = io.Copy(fout, file)
return err
})
}
// isCompressible checks if a file has a compressible mime type by its header and name.
// It returns true if the file is compressible, false otherwise.
func isCompressible(fname string) (bool, error) {
// Check if the file has a known non-compressible extension
// Source: [1]
nonCompressibleExt := []string{".7z", ".bz2", ".gif", ".gz", ".jpeg", ".jpg", ".mp3", ".mp4", ".png", ".rar", ".zip", ".pf_fragment", ".pf_index", ".pf_meta", ".ico"}
// Get the file extension from the name
ext := filepath.Ext(fname)
// Loop through the non-compressible extensions and compare with the file extension
for _, n := range nonCompressibleExt {
if ext == n {
// The file is not compressible by its name
return false, nil
}
}
compressibleExt := []string{".js", ".json", ".txt", ".dot", ".css", ".pdf", ".svg"}
// Loop through the compressible extensions and compare with the file extension
for _, n := range compressibleExt {
if ext == n {
// The file is compressible by its name
return true, nil
}
}
// A list of common mime types that are not compressible
// Source: [1]
nonCompressible := map[string]bool{
"application": true,
"image": true,
"audio": true,
"video": true,
}
fin, err := os.Open(fname)
if err != nil {
return false, fmt.Errorf("can't read file %s: %w", fname, err)
}
defer fin.Close()
// Read the first 512 bytes of the file
buffer := make([]byte, 512)
if _, err = fin.Read(buffer); err != nil {
return false, fmt.Errorf("can't read from file %s: %w", fname, err)
}
// Detect the mime type from the buffer
mimeType := http.DetectContentType(buffer)
// Split the mime type by "/" and get the first part
parts := strings.Split(mimeType, "/")
if len(parts) < 2 {
slog.Debug("can't detect mime type of file, it's probably not compressible", "fname", fname, "mimeType", mimeType)
return false, nil
}
mainType := parts[0]
// Check if the main type is in the non-compressible map
if nonCompressible[mainType] {
// The file is not compressible by its header
return false, nil
}
// The file is compressible by both its header and name
return true, nil
}
type ZipServer struct {
lock sync.RWMutex
zip *zip.ReadCloser
}
func NewZipServer(zipPath string) (*ZipServer, error) {
file, err := zip.OpenReader(zipPath)
if err != nil {
return nil, err
}
result := &ZipServer{
zip: file,
}
return result, nil
}
func (zs *ZipServer) Update(fname string) error {
zs.lock.Lock()
defer zs.lock.Unlock()
old := zs.zip
file, err := zip.OpenReader(fname)
if err != nil {
return err
}
zs.zip = file
old.Close()
return nil
}
func (zs *ZipServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
zs.lock.RLock()
defer zs.lock.RUnlock()
vals := internal.ParseValueAndParams(r.Header.Get("Accept-Encoding"))
slog.Info("accept-encoding", "vals", vals)
http.FileServer(http.FS(zs.zip)).ServeHTTP(w, r)
}
|
