diff options
| -rw-r--r-- | cmd/orodyagzou/diagram.jpg | bin | 0 -> 95741 bytes | |||
| -rw-r--r-- | cmd/patchouli/diagrams/architecture.png | bin | 0 -> 61811 bytes | |||
| -rw-r--r-- | cmd/patchouli/main.go | 134 | ||||
| -rw-r--r-- | cmd/patchouli/var/.gitignore | 2 | ||||
| -rw-r--r-- | cmd/patchouli/ytdlp/ytdlp.go | 113 | ||||
| -rw-r--r-- | kube/alrest/gitea/deployment.yaml | 25 | ||||
| -rw-r--r-- | kube/alrest/gitea/pvc.yaml | 15 | ||||
| -rw-r--r-- | kube/alrest/gitea/runner.yaml | 8 | ||||
| -rw-r--r-- | kube/alrest/minio/deployment.yaml | 2 | ||||
| -rw-r--r-- | kube/alrest/minio/pvc.yaml | 2 | ||||
| -rw-r--r-- | kube/alrest/registry/certificate.yaml | 2 | ||||
| -rw-r--r-- | kube/alrest/registry/external-dns.yaml | 11 | ||||
| -rw-r--r-- | kube/alrest/registry/pull-through/dockerhub/certificate.yaml | 16 | ||||
| -rw-r--r-- | kube/alrest/staticsites/caddy1/caddy.yaml | 2 | ||||
| -rw-r--r-- | kube/alrest/staticsites/caddy1/tulpanomicon.yaml | 6 | ||||
| -rw-r--r-- | kube/alrest/vms/arona/unfuck-dhcpcd.service | 11 | ||||
| -rw-r--r-- | kube/alrest/vms/feixiao/feixiao.yaml | 32 | ||||
| -rw-r--r-- | kube/alrest/vms/kustomization.yaml | 5 | ||||
| -rw-r--r-- | kube/alrest/vms/xesite-ci/vm.yaml | 77 |
19 files changed, 323 insertions, 140 deletions
diff --git a/cmd/orodyagzou/diagram.jpg b/cmd/orodyagzou/diagram.jpg Binary files differnew file mode 100644 index 0000000..e557a73 --- /dev/null +++ b/cmd/orodyagzou/diagram.jpg diff --git a/cmd/patchouli/diagrams/architecture.png b/cmd/patchouli/diagrams/architecture.png Binary files differnew file mode 100644 index 0000000..f53c074 --- /dev/null +++ b/cmd/patchouli/diagrams/architecture.png diff --git a/cmd/patchouli/main.go b/cmd/patchouli/main.go new file mode 100644 index 0000000..a316a0c --- /dev/null +++ b/cmd/patchouli/main.go @@ -0,0 +1,134 @@ +package main + +import ( + "context" + "database/sql" + "flag" + "fmt" + "log" + "log/slog" + "net/http" + "path/filepath" + + "connectrpc.com/connect" + _ "github.com/ncruces/go-sqlite3/embed" + "github.com/ncruces/go-sqlite3/gormlite" + slogGorm "github.com/orandin/slog-gorm" + "google.golang.org/protobuf/types/known/timestamppb" + "gorm.io/gorm" + gormPrometheus "gorm.io/plugin/prometheus" + "within.website/x/buf/patchouli" + "within.website/x/buf/patchouli/patchouliconnect" + "within.website/x/cmd/patchouli/ytdlp" + "within.website/x/internal" +) + +var ( + bind = flag.String("bind", ":2934", "HTTP bind address") + dataDir = flag.String("data-dir", "./var", "location to store data persistently") +) + +func main() { + internal.HandleStartup() + + slog.Info("starting up", "bind", *bind, "data-dir", *dataDir) + + db, err := connectDB() + if err != nil { + log.Fatalf("can't connect to DB: %w", err) + } + _ = db + + s := &Server{db: db} + + compress1KB := connect.WithCompressMinBytes(1024) + + mux := http.NewServeMux() + mux.Handle(patchouliconnect.NewSyndicateHandler(s, compress1KB)) + + slog.Info("listening", "url", "http://localhost"+*bind) + log.Fatal(http.ListenAndServe(*bind, mux)) +} + +func connectDB() (*gorm.DB, error) { + db, err := gorm.Open(gormlite.Open(filepath.Join(*dataDir, "data.db")), &gorm.Config{ + Logger: slogGorm.New( + slogGorm.WithErrorField("err"), + slogGorm.WithRecordNotFoundError(), + ), + }) + if err != nil { + return nil, fmt.Errorf("failed to connect to database: %w", err) + } + + if err := db.AutoMigrate( + &Video{}, + ); err != nil { + return nil, fmt.Errorf("failed to migrate schema: %w", err) + } + + db.Use(gormPrometheus.New(gormPrometheus.Config{ + DBName: "mi", + })) + + return db, nil +} + +type Server struct { + db *gorm.DB +} + +func (s *Server) Info( + ctx context.Context, + req *connect.Request[patchouli.TwitchInfoReq], +) ( + *connect.Response[patchouli.TwitchInfoResp], + error, +) { + metadata, err := ytdlp.Metadata(ctx, req.Msg.GetUrl()) + if err != nil { + slog.Error("can't fetch metadata for video", "url", req.Msg.GetUrl(), "err", err) + return nil, connect.NewError(connect.CodeInternal, err) + } + + result := &patchouli.TwitchInfoResp{ + Id: metadata.ID, + Title: metadata.Title, + ThumbnailUrl: metadata.Thumbnail, + Duration: metadata.DurationString, + UploadDate: timestamppb.New(metadata.UploadDate.Time), + Url: req.Msg.GetUrl(), + } + + return connect.NewResponse(result), nil +} + +func (s *Server) Download( + ctx context.Context, + req *connect.Request[patchouli.TwitchDownloadReq], +) ( + *connect.Response[patchouli.TwitchDownloadResp], + error, +) { + dir := filepath.Join(*dataDir, "video") + + if err := ytdlp.Download(ctx, req.Msg.GetUrl(), dir); err != nil { + return nil, err + } + + result := &patchouli.TwitchDownloadResp{ + Url: req.Msg.Url, + Location: dir, + } + + return connect.NewResponse(result), nil +} + +type Video struct { + gorm.Model + TwitchID string `gorm:"uniqueIndex"` + TwitchURL string `gorm:"uniqueIndex"` + Title string + State string + BlogPath sql.NullString +} diff --git a/cmd/patchouli/var/.gitignore b/cmd/patchouli/var/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/cmd/patchouli/var/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore
\ No newline at end of file diff --git a/cmd/patchouli/ytdlp/ytdlp.go b/cmd/patchouli/ytdlp/ytdlp.go new file mode 100644 index 0000000..096ea2b --- /dev/null +++ b/cmd/patchouli/ytdlp/ytdlp.go @@ -0,0 +1,113 @@ +package ytdlp + +import ( + "bytes" + "context" + "database/sql/driver" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "time" + + "within.website/x/internal" +) + +const dateFormat = "20060102" + +type VideoMetadata struct { + ID string `json:"id"` + Title string `json:"title"` + Thumbnail string `json:"thumbnail"` + DurationString string `json:"duration_string"` + UploadDate Date `json:"upload_date"` + URL string `json:"url" gorm:"uniqueIndex"` +} + +type Date struct { + time.Time +} + +func (d *Date) MarshalJSON() ([]byte, error) { + result := d.Format(dateFormat) + + return []byte(fmt.Sprintf("%q", result)), nil +} + +func (d *Date) UnmarshalJSON(data []byte) error { + str := string(data) + str = str[1 : len(str)-1] + + parsedTime, err := time.Parse(dateFormat, str) + if err != nil { + return err + } + + d.Time = parsedTime + return nil +} + +// Value implements the driver.Valuer interface +func (d Date) Value() (driver.Value, error) { + return d.Format(dateFormat), nil +} + +// Scan implements the sql.Scanner interface +func (d *Date) Scan(value interface{}) error { + if value == nil { + *d = Date{Time: time.Time{}} + return nil + } + + switch v := value.(type) { + case time.Time: + *d = Date{Time: v} + case []byte: + parsedTime, err := time.Parse(dateFormat, string(v)) + if err != nil { + return err + } + *d = Date{Time: parsedTime} + case string: + parsedTime, err := time.Parse(dateFormat, v) + if err != nil { + return err + } + *d = Date{Time: parsedTime} + default: + return fmt.Errorf("cannot scan type %T into Date", value) + } + + return nil +} + +func Metadata(ctx context.Context, url string) (*VideoMetadata, error) { + result, err := internal.RunJSON[VideoMetadata](ctx, "yt-dlp", "--dump-json", url) + if err != nil { + return nil, err + } + + return &result, nil +} + +func Download(ctx context.Context, url, to string) error { + exePath, err := exec.LookPath("yt-dlp") + if err != nil { + return fmt.Errorf("can't find yt-dlp: %w", err) + } + + cmd := exec.CommandContext(ctx, exePath, "-o", filepath.Join(to, "%(id)s.%(ext)s"), "--write-info-json", url) + + var stdout, stderr bytes.Buffer + + cmd.Stdout = io.MultiWriter(&stdout, os.Stdout) + cmd.Stderr = io.MultiWriter(&stderr, os.Stderr) + + if err := cmd.Run(); err != nil { + // TODO(Xe): return stdout/err? + return fmt.Errorf("can't download %s: %w", url, err) + } + + return nil +} diff --git a/kube/alrest/gitea/deployment.yaml b/kube/alrest/gitea/deployment.yaml index 0aee0d0..f4d1080 100644 --- a/kube/alrest/gitea/deployment.yaml +++ b/kube/alrest/gitea/deployment.yaml @@ -7,13 +7,15 @@ data: USER_UID: "1000" USER_GID: "1000" - #GITEA__storage__STORAGE_TYPE: minio - GITEA__storage__STORAGE_TYPE: local + GITEA__storage__STORAGE_TYPE: minio + #GITEA__storage__STORAGE_TYPE: local GITEA__storage__SERVE_DIRECT: "true" - GITEA__storage__MINIO_ENDPOINT: https://fly.storage.tigris.dev - GITEA__storage__MINIO_BUCKET: alrest-gitea - GITEA__storage__MINIO_LOCATION: auto - GITEA__storage__MINIO_USE_SSL: "true" + GITEA__storage__MINIO_ENDPOINT: http://minio.default.svc.alrest.xeserv.us + GITEA__storage__MINIO_BUCKET: gitea + GITEA__storage__MINIO_LOCATION: alrest + GITEA__storage__MINIO_USE_SSL: "false" + GITEA__storage__MINIO_ACCESS_KEY_ID: "nkb2jIEnRmoiu56R5vsH" + GITEA__stroage__MINIO_SECRET_ACCESS_KEY: "pZFQDZFFMzqqvQdAjwpdEQrRIbMlu0H9cd25MmkQ" --- apiVersion: apps/v1 kind: Deployment @@ -41,9 +43,6 @@ spec: - name: data persistentVolumeClaim: claimName: gitea - - name: tigris - persistentVolumeClaim: - claimName: gitea-tigris containers: - name: main image: gitea/gitea:1-rootless @@ -61,8 +60,6 @@ spec: envFrom: - configMapRef: name: gitea - - secretRef: - name: tigris-creds ports: - name: http containerPort: 3000 @@ -77,12 +74,6 @@ spec: - name: data mountPath: /etc/gitea subPath: conf - - name: tigris - mountPath: /var/lib/gitea/data/attachments - subPath: attachments - - name: tigris - mountPath: /var/lib/gitea/git/lfs - subPath: lfs livenessProbe: httpGet: path: /api/healthz diff --git a/kube/alrest/gitea/pvc.yaml b/kube/alrest/gitea/pvc.yaml index e000fd8..7c7515e 100644 --- a/kube/alrest/gitea/pvc.yaml +++ b/kube/alrest/gitea/pvc.yaml @@ -9,17 +9,4 @@ spec: storageClassName: longhorn resources: requests: - storage: 64Gi ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: gitea-tigris -spec: - accessModes: - - ReadWriteOnce - - ReadWriteMany - storageClassName: tigris - resources: - requests: - storage: 64Ti
\ No newline at end of file + storage: 64Gi
\ No newline at end of file diff --git a/kube/alrest/gitea/runner.yaml b/kube/alrest/gitea/runner.yaml index 98f6976..1ea60fd 100644 --- a/kube/alrest/gitea/runner.yaml +++ b/kube/alrest/gitea/runner.yaml @@ -3,7 +3,7 @@ kind: Secret metadata: name: runner-secret stringData: - token: "CY2fOnh2aP4scVZc9w4dEDkyxBGnsezNZqRWcKyi" + token: "6ALUVtYp8UAl5FHFUwGZ1lG03lMR2xta2cV9m2mW" type: Opaque --- apiVersion: v1 @@ -17,7 +17,7 @@ data: } --- apiVersion: apps/v1 -kind: Deployment +kind: DaemonSet metadata: labels: app.kubernetes.io/name: act-runner @@ -27,11 +27,9 @@ metadata: keel.sh/trigger: poll keel.sh/pollSchedule: "@hourly" spec: - replicas: 3 selector: matchLabels: app.kubernetes.io/name: act-runner - strategy: {} template: metadata: labels: @@ -48,7 +46,7 @@ spec: name: docker-daemon-config containers: - name: runner - image: gitea/act_runner:nightly-dind-rootless + image: gitea/act_runner:nightly-dind imagePullPolicy: Always env: - name: GITEA_INSTANCE_URL diff --git a/kube/alrest/minio/deployment.yaml b/kube/alrest/minio/deployment.yaml index a67d508..aeda39e 100644 --- a/kube/alrest/minio/deployment.yaml +++ b/kube/alrest/minio/deployment.yaml @@ -33,8 +33,6 @@ spec: value: hunter22 ports: - containerPort: 9000 - hostPort: 9000 name: http - containerPort: 9001 - hostPort: 9001 name: webui
\ No newline at end of file diff --git a/kube/alrest/minio/pvc.yaml b/kube/alrest/minio/pvc.yaml index 249e993..0a4ab19 100644 --- a/kube/alrest/minio/pvc.yaml +++ b/kube/alrest/minio/pvc.yaml @@ -5,7 +5,7 @@ metadata: spec: accessModes: - ReadWriteMany - storageClassName: itsuki + storageClassName: rotational resources: requests: storage: 100Gi
\ No newline at end of file diff --git a/kube/alrest/registry/certificate.yaml b/kube/alrest/registry/certificate.yaml index f2b6d37..6103204 100644 --- a/kube/alrest/registry/certificate.yaml +++ b/kube/alrest/registry/certificate.yaml @@ -13,7 +13,7 @@ spec: - digital signature - key encipherment dnsNames: - - registry.default.svc.alrest.xeserv.us + - registry.int.xeserv.us issuerRef: name: "letsencrypt-prod" kind: ClusterIssuer diff --git a/kube/alrest/registry/external-dns.yaml b/kube/alrest/registry/external-dns.yaml index 3f39a8c..72b2be0 100644 --- a/kube/alrest/registry/external-dns.yaml +++ b/kube/alrest/registry/external-dns.yaml @@ -2,11 +2,10 @@ apiVersion: externaldns.k8s.io/v1alpha1 kind: DNSEndpoint metadata: name: registry-internal - namespace: default spec: endpoints: - - dnsName: registry.default.svc.alrest.xeserv.us - recordTTL: 3600 - recordType: A - targets: - - 10.223.208.190
\ No newline at end of file + - dnsName: registry.int.xeserv.us + recordTTL: 3600 + recordType: A + targets: + - 10.223.208.190
\ No newline at end of file diff --git a/kube/alrest/registry/pull-through/dockerhub/certificate.yaml b/kube/alrest/registry/pull-through/dockerhub/certificate.yaml index bce9b22..f568c0f 100644 --- a/kube/alrest/registry/pull-through/dockerhub/certificate.yaml +++ b/kube/alrest/registry/pull-through/dockerhub/certificate.yaml @@ -13,8 +13,20 @@ spec: - digital signature - key encipherment dnsNames: - - pt-dh.default.svc.alrest.xeserv.us + - pt-dh.int.xeserv.us issuerRef: name: "letsencrypt-prod" kind: ClusterIssuer - group: cert-manager.io
\ No newline at end of file + group: cert-manager.io +--- +apiVersion: externaldns.k8s.io/v1alpha1 +kind: DNSEndpoint +metadata: + name: pt-dh +spec: + endpoints: + - dnsName: pt-dh.int.xeserv.us + recordTTL: 3600 + recordType: A + targets: + - 10.211.44.197
\ No newline at end of file diff --git a/kube/alrest/staticsites/caddy1/caddy.yaml b/kube/alrest/staticsites/caddy1/caddy.yaml index 7c7f3e6..e10b315 100644 --- a/kube/alrest/staticsites/caddy1/caddy.yaml +++ b/kube/alrest/staticsites/caddy1/caddy.yaml @@ -107,7 +107,7 @@ spec: claimName: xena-greedo - name: tulpanomicon persistentVolumeClaim: - claimName: tulpanomicon + claimName: tulpanomicon2 - name: xn--u7hz981o persistentVolumeClaim: claimName: xn--u7hz981o--2 diff --git a/kube/alrest/staticsites/caddy1/tulpanomicon.yaml b/kube/alrest/staticsites/caddy1/tulpanomicon.yaml index add8c57..49657fd 100644 --- a/kube/alrest/staticsites/caddy1/tulpanomicon.yaml +++ b/kube/alrest/staticsites/caddy1/tulpanomicon.yaml @@ -1,15 +1,15 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: tulpanomicon + name: tulpanomicon2 spec: - storageClassName: "longhorn" + storageClassName: "rotational" resources: requests: storage: 512Mi volumeMode: Filesystem accessModes: - - ReadWriteMany + - ReadWriteOnce --- apiVersion: networking.k8s.io/v1 kind: Ingress diff --git a/kube/alrest/vms/arona/unfuck-dhcpcd.service b/kube/alrest/vms/arona/unfuck-dhcpcd.service new file mode 100644 index 0000000..456750f --- /dev/null +++ b/kube/alrest/vms/arona/unfuck-dhcpcd.service @@ -0,0 +1,11 @@ +[Unit] +Description=A hack to start dhcpcd at the right time +After=network.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/sbin/dhcpcd enp1s0 + +[Install] +WantedBy=default.target
\ No newline at end of file diff --git a/kube/alrest/vms/feixiao/feixiao.yaml b/kube/alrest/vms/feixiao/feixiao.yaml index 1657ef8..98cc668 100644 --- a/kube/alrest/vms/feixiao/feixiao.yaml +++ b/kube/alrest/vms/feixiao/feixiao.yaml @@ -7,13 +7,13 @@ metadata: kubevirt.io/domain: feixiao spec: storage: - storageClassName: longhorn + storageClassName: rotational volumeMode: Block accessModes: - ReadWriteOnce resources: requests: - storage: 64Gi + storage: 128Gi source: http: url: "https://cloud-images.ubuntu.com/daily/server/noble/current/noble-server-cloudimg-amd64.img" @@ -21,7 +21,22 @@ spec: apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: feixiao-storage + name: feixiao-longhorn + namespace: waifud +spec: + accessModes: + - ReadWriteOnce + - ReadWriteMany + volumeMode: Block + storageClassName: rotational + resources: + requests: + storage: 128Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: feixiao-itsuki namespace: waifud spec: accessModes: @@ -91,22 +106,25 @@ spec: bus: sata readonly: true name: cloudinitdisk - filesystems: - - name: storage - virtiofs: {} + filesystems: [] + # - name: storage + # virtiofs: {} machine: type: q35 resources: requests: memory: 8192M volumes: + - name: storage + persistentVolumeClaim: + claimName: "feixiao" - name: disk0 persistentVolumeClaim: claimName: "feixiao" - cloudInitNoCloud: userData: | #cloud-config - hostname: feixiao + hostname: feixiao: ssh_pwauth: True disable_root: false diff --git a/kube/alrest/vms/kustomization.yaml b/kube/alrest/vms/kustomization.yaml index 207cafa..4eb8c94 100644 --- a/kube/alrest/vms/kustomization.yaml +++ b/kube/alrest/vms/kustomization.yaml @@ -1,6 +1,3 @@ resources: - ./namespace.yaml - - arona/arona.yaml - - feixiao/feixiao.yaml - - sena/sena.yaml - - xesite-ci/vm.yaml
\ No newline at end of file + - arona/arona.yaml
\ No newline at end of file diff --git a/kube/alrest/vms/xesite-ci/vm.yaml b/kube/alrest/vms/xesite-ci/vm.yaml deleted file mode 100644 index cf25acd..0000000 --- a/kube/alrest/vms/xesite-ci/vm.yaml +++ /dev/null @@ -1,77 +0,0 @@ -apiVersion: cdi.kubevirt.io/v1beta1 -kind: DataVolume -metadata: - name: "ci-xe-site" - namespace: ci -spec: - storage: - storageClassName: longhorn - volumeMode: Block # or filesystem - accessModes: - - ReadWriteOnce # Necessary as it's no longer has a default - resources: - requests: - storage: 64Gi - source: - http: - url: "https://cloud-images.ubuntu.com/daily/server/noble/current/noble-server-cloudimg-amd64.img" ---- -apiVersion: kubevirt.io/v1 -kind: VirtualMachine -metadata: - labels: - kubevirt.io/os: linux - name: "ci-xe-site" - namespace: ci -spec: - running: true - template: - metadata: - creationTimestamp: null - labels: - kubevirt.io/domain: "ubuntu-22.04" - spec: - domain: - cpu: - cores: 4 - devices: - disks: - - disk: - bus: virtio - name: disk0 - - cdrom: - bus: sata - readonly: true - name: cloudinitdisk - machine: - type: q35 - resources: - requests: - memory: 4096M - volumes: - - name: disk0 - persistentVolumeClaim: - claimName: "ci-xe-site" - - cloudInitNoCloud: - userData: | - #cloud-config - hostname: ci-xe-site - ssh_pwauth: True - disable_root: false - - write_files: - - encoding: b64 - content: bmV0d29yazoKICB2ZXJzaW9uOiAyCiAgZXRoZXJuZXRzOgogICAgaWQwOgogICAgICBkaGNwNDogdHJ1ZQogICAgICBtYXRjaDoKICAgICAgICBuYW1lOiBlbnAqCg== - owner: root:root - path: /etc/netplan/99-net-fix.yaml - permissions: '0644' - - users: - - name: xe - groups: [ wheel ] - sudo: [ "ALL=(ALL) NOPASSWD:ALL" ] - shell: /bin/bash - ssh-authorized-keys: - - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF6/+q2dBHEfETomAH7rvDPXHulSXnhcV/szxTAWiKZA cadey@defoko - - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJIH27tCTJZwJxp+bcWRALtrk7SqnwXZ4/Ae67jIO0eF cadey@shiroko - name: cloudinitdisk
\ No newline at end of file |
