diff --git a/Dockerfile b/Dockerfile index 4fedbea..38a67eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:latest -RUN apk add --no-cache ca-certificates +RUN apk add --no-cache ca-certificates ffmpeg COPY bin/evening_detective_server /usr/local/bin/evening_detective_server RUN chmod +x /usr/local/bin/evening_detective_server CMD ["/usr/local/bin/evening_detective_server"] diff --git a/bin/evening_detective_server b/bin/evening_detective_server index f420cd9..f0b18d5 100755 Binary files a/bin/evening_detective_server and b/bin/evening_detective_server differ diff --git a/docs/GAME_PACKAGE.md b/docs/GAME_PACKAGE.md index fc02977..c1dd6f9 100644 --- a/docs/GAME_PACKAGE.md +++ b/docs/GAME_PACKAGE.md @@ -66,6 +66,26 @@ JWT/refresh-токены, действия, полученные улики, о содержимого; поддерживаются PDF, JPEG, PNG, GIF, WebP, MP3, Ogg, WAV, MP4 и WebM. Ссылки внутри копии `Data` заменяются на пути `files/...`; входные данные не меняются. +При подготовке нового пакета статичные JPEG/PNG/WebP преобразуются в WebP +(качество 85), аудио — в MP3 (128 кбит/с), видео — в MP4 (H.264, CRF 23, +preset fast, yuv420p, звук AAC 128 кбит/с, faststart). Фото и видео уменьшаются +до 1920 пикселей по длинной стороне с сохранением пропорций; маленькие файлы +не увеличиваются. Прозрачность и ориентация изображений сохраняются. PDF, GIF, +APNG и анимированные WebP переносятся без изменений. + +Оптимизированная копия используется только если она меньше исходной; при +ошибке, отсутствии кодека/утилиты или превышении двух минут на файл сохраняется +исходник. Общий срок подготовки пакета остаётся 30 минут. Отмена подготовки +прерывает конвертер. Преобразование последовательное, до двух потоков +декодирования/кодирования и фильтров, только из локальных временных файлов; +сетевые протоколы FFmpeg запрещены. В манифест записываются MIME, размер и SHA-256 +окончательных байтов. Исходники в хранилище и уже готовые ZIP не меняются. + +Для запуска вне Docker установите `ffmpeg` и `ffprobe` в `PATH`, со сборкой +кодеков libwebp, libmp3lame, libx264 и AAC. Оба серверных Dockerfile устанавливают +пакет `ffmpeg`. Сжатие с потерями может ухудшить различимость мелких улик; если +преобразование не удалось, размер пакета может остаться прежним. + Относительное имя и полный URL с настроенным префиксом хранилища открываются через переданный источник локальных файлов. Остальные HTTP/HTTPS-ссылки скачиваются при сборке. Один URL скачивается один раз; одинаковое содержимое diff --git a/internal/services/game_package_service/media.go b/internal/services/game_package_service/media.go new file mode 100644 index 0000000..e243235 --- /dev/null +++ b/internal/services/game_package_service/media.go @@ -0,0 +1,122 @@ +package game_package_service + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "os" + "os/exec" + "strconv" + "time" +) + +// Optimization is best effort; cancellation of the export itself is never ignored. +func optimizeMedia(ctx context.Context, filename string, original FileInfo) (FileInfo, string, error) { + info, optimized, err := convertMedia(ctx, filename, original) + if ctx.Err() != nil { + return FileInfo{}, "", ctx.Err() + } + if err != nil || optimized == "" { + return original, filename, nil + } + if err := os.Remove(filename); err != nil { + return FileInfo{}, "", err + } + return info, optimized, nil +} + +func convertMedia(parent context.Context, filename string, original FileInfo) (FileInfo, string, error) { + ctx, cancel := context.WithTimeout(parent, 2*time.Minute) + defer cancel() + var codec []string + var ext string + switch original.MIME { + case "image/jpeg", "image/png", "image/webp": + // Decode at most two frames so APNG and animated WebP remain untouched. + probe, err := exec.CommandContext(ctx, "ffprobe", "-v", "error", "-protocol_whitelist", "file,pipe", + "-threads", "2", "-select_streams", "v:0", "-read_intervals", "%+#2", "-count_frames", + "-show_entries", "stream=nb_read_frames", "-of", "json", filename).Output() + if err != nil { + return FileInfo{}, "", err + } + var frames struct { + Streams []struct { + Count string `json:"nb_read_frames"` + } `json:"streams"` + } + if err := json.Unmarshal(probe, &frames); err != nil { + return FileInfo{}, "", err + } + if len(frames.Streams) != 1 || frames.Streams[0].Count != "1" { + return FileInfo{}, "", nil + } + ext = ".webp" + codec = []string{"-map", "0:v:0", "-frames:v", "1", "-vf", "scale=w=min(1920\\,iw):h=min(1920\\,ih):force_original_aspect_ratio=decrease", + "-c:v", "libwebp", "-quality", "85"} + case "audio/mpeg", "application/ogg", "audio/wave", "audio/x-wav": + ext = ".mp3" + codec = []string{"-map", "0:a:0", "-c:a", "libmp3lame", "-b:a", "128k"} + case "video/mp4", "video/webm": + ext = ".mp4" + codec = []string{"-map", "0:v:0", "-map", "0:a:0?", + "-vf", "scale=w=min(1920\\,iw):h=min(1920\\,ih):force_original_aspect_ratio=decrease:force_divisible_by=2", + "-c:v", "libx264", "-crf", "23", "-preset", "fast", "-pix_fmt", "yuv420p", + "-c:a", "aac", "-b:a", "128k", "-movflags", "+faststart"} + default: + return FileInfo{}, "", nil + } + output := filename + "-optimized" + ext + keep := false + defer func() { + if !keep { + _ = os.Remove(output) + } + }() + args := []string{"-nostdin", "-v", "error", "-y", "-protocol_whitelist", "file,pipe", "-threads", "2", "-i", filename, + "-map_metadata", "-1", "-filter_threads", "2"} + args = append(args, codec...) + // An oversized output is discarded, including files stopped early by -fs. + args = append(args, "-threads", "2", "-fs", strconv.FormatInt(original.Size+65536, 10), output) + if err := exec.CommandContext(ctx, "ffmpeg", args...).Run(); err != nil { + return FileInfo{}, "", err + } + file, err := os.Open(output) + if err != nil { + return FileInfo{}, "", err + } + defer func() { _ = file.Close() }() + stat, err := file.Stat() + if err != nil { + return FileInfo{}, "", err + } + if stat.Size() == 0 || stat.Size() >= original.Size { + return FileInfo{}, "", nil + } + header := make([]byte, 512) + n, err := file.Read(header) + if err != nil && !errors.Is(err, io.EOF) { + return FileInfo{}, "", err + } + mime, actualExt := mediaType(header[:n]) + if actualExt != ext { + return FileInfo{}, "", nil + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return FileInfo{}, "", err + } + hash := sha256.New() + if _, err := copyLimited(ctx, hash, file, original.Size); err != nil { + return FileInfo{}, "", err + } + digest := hex.EncodeToString(hash.Sum(nil)) + keep = true + return FileInfo{ + Path: "files/" + digest + ext, + Size: stat.Size(), + MIME: mime, + SHA256: digest, + }, output, nil +} diff --git a/internal/services/game_package_service/media_test.go b/internal/services/game_package_service/media_test.go new file mode 100644 index 0000000..b0e9d04 --- /dev/null +++ b/internal/services/game_package_service/media_test.go @@ -0,0 +1,286 @@ +package game_package_service + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "image" + "image/color" + "image/gif" + "image/png" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" +) + +func requireFFmpeg(t *testing.T) { + t.Helper() + for _, name := range []string{"ffmpeg", "ffprobe"} { + if _, err := exec.LookPath(name); err != nil { + t.Skip(name + " is required for media integration tests") + } + } +} + +func makeMedia(t *testing.T, ext string, args ...string) []byte { + t.Helper() + filename := filepath.Join(t.TempDir(), "source"+ext) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + command := append([]string{"-v", "error", "-nostdin", "-threads", "2", "-filter_threads", "2"}, args...) + command = append(command, "-threads", "2", filename) + if output, err := exec.CommandContext(ctx, "ffmpeg", command...).CombinedOutput(); err != nil { + t.Fatalf("generate fixture: %v: %s", err, output) + } + raw, err := os.ReadFile(filename) + if err != nil { + t.Fatal(err) + } + return raw +} + +func mediaPNG(t *testing.T, width, height int) []byte { + t.Helper() + im := image.NewNRGBA(image.Rect(0, 0, width, height)) + var seed uint32 = 42 + for y := 0; y < height; y++ { + for x := 0; x < width; x++ { + seed = 1664525*seed + 1013904223 + im.SetNRGBA(x, y, color.NRGBA{R: byte(seed >> 24), G: byte(seed >> 16), B: byte(seed >> 8), A: 128}) + } + } + var output bytes.Buffer + if err := png.Encode(&output, im); err != nil { + t.Fatal(err) + } + return output.Bytes() +} + +func TestOptimizedMediaRoundTrip(t *testing.T) { + requireFFmpeg(t) + for _, tc := range []struct { + name, ext, mime string + width, height int + args []string + }{ + {name: "large transparent image", ext: ".webp", mime: "image/webp", width: 1920, height: 960}, + {name: "small transparent image", ext: ".webp", mime: "image/webp", width: 160, height: 80}, + {name: "portrait image", ext: ".webp", mime: "image/webp", width: 960, height: 1920}, + {name: "stereo audio", ext: ".mp3", mime: "audio/mpeg", args: []string{"-f", "lavfi", "-i", "sine=frequency=440:duration=2", "-ac", "2", "-c:a", "pcm_s16le"}}, + {name: "silent video", ext: ".mp4", mime: "video/mp4", width: 1920, height: 960, + args: []string{"-f", "lavfi", "-i", "testsrc2=size=2048x1024:rate=8:duration=1", "-c:v", "libx264", "-qp", "0", "-preset", "ultrafast"}}, + {name: "video with audio", ext: ".mp4", mime: "video/mp4", width: 320, height: 160, + args: []string{"-f", "lavfi", "-i", "testsrc2=size=320x160:rate=8:duration=1", "-f", "lavfi", "-i", "sine=frequency=440:duration=1", "-c:v", "libx264", "-qp", "0", "-preset", "ultrafast", "-c:a", "aac"}}, + {name: "portrait video", ext: ".mp4", mime: "video/mp4", width: 960, height: 1920, + args: []string{"-f", "lavfi", "-i", "testsrc2=size=1024x2048:rate=8:duration=1", "-c:v", "libx264", "-qp", "0", "-preset", "ultrafast"}}, + } { + t.Run(tc.name, func(t *testing.T) { + snapshot, files := fixture() + var raw []byte + ref := "cover.png" + switch tc.mime { + case "image/webp": + width, height := tc.width, tc.height + if width == 1920 || height == 1920 { + width, height = width*16/15, height*16/15 + } + raw = mediaPNG(t, width, height) + case "audio/mpeg": + ref = "music.mp3" + raw = makeMedia(t, ".wav", tc.args...) + case "video/mp4": + ref = "video.webm" + raw = makeMedia(t, ".mp4", tc.args...) + } + files[ref] = raw + archive := buildPackage(t, NewGamePackageService(""), snapshot, files) + bundle, err := NewGamePackageService("").Unpack(context.Background(), bytes.NewReader(archive), int64(len(archive)), t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = os.RemoveAll(bundle.Dir) }() + var path string + switch tc.mime { + case "image/webp": + path = bundle.Data.Scenario.Image + if path != bundle.Data.Scenario.Story.Background || path != bundle.Data.Scenario.Story.Places[0].Image { + t.Fatal("shared image references diverged") + } + case "audio/mpeg": + path = bundle.Data.Scenario.Story.Introduction.Audio + if path != bundle.Data.Playlists[0].Tracks[0].Filename { + t.Fatal("shared audio references diverged") + } + case "video/mp4": + path = bundle.Data.Scenario.Story.Conclusion.Video + } + if !strings.HasSuffix(path, tc.ext) { + t.Fatalf("unexpected optimized path: %s", path) + } + filename := bundle.Files[path] + optimized, err := os.ReadFile(filename) + if err != nil { + t.Fatal(err) + } + if len(optimized) >= len(raw) { + t.Fatalf("media did not shrink: %d >= %d", len(optimized), len(raw)) + } + probe, err := exec.Command("ffprobe", "-v", "error", "-show_streams", "-of", "json", filename).Output() + if err != nil { + t.Fatal(err) + } + var metadata struct { + Streams []struct { + Width int `json:"width"` + Height int `json:"height"` + Channels int `json:"channels"` + PixelFormat string `json:"pix_fmt"` + Duration string `json:"duration"` + } `json:"streams"` + } + if err := json.Unmarshal(probe, &metadata); err != nil || len(metadata.Streams) == 0 { + t.Fatalf("invalid probe: %s: %v", probe, err) + } + stream := metadata.Streams[0] + if stream.Width != tc.width || stream.Height != tc.height { + t.Fatalf("unexpected dimensions: %dx%d", stream.Width, stream.Height) + } + if tc.mime == "image/webp" && !strings.Contains(stream.PixelFormat, "a") { + t.Fatalf("transparency lost: %s", stream.PixelFormat) + } + if tc.mime == "audio/mpeg" && stream.Channels != 2 { + t.Fatalf("stereo lost: %d channels", stream.Channels) + } + if tc.mime == "audio/mpeg" || tc.mime == "video/mp4" { + duration, err := strconv.ParseFloat(stream.Duration, 64) + want := 1.0 + if tc.mime == "audio/mpeg" { + want = 2 + } + if err != nil || duration < want-0.1 || duration > want+0.1 { + t.Fatalf("duration changed: %s", stream.Duration) + } + } + t.Logf("%d -> %d bytes", len(raw), len(optimized)) + }) + } +} + +func TestMediaPreservesAnimationsAndLargerResults(t *testing.T) { + requireFFmpeg(t) + for _, tc := range []struct { + name, ext string + args []string + }{ + {name: "APNG", ext: ".png", args: []string{"-f", "lavfi", "-i", "testsrc2=size=32x32:rate=2:duration=1", "-plays", "0", "-f", "apng"}}, + {name: "animated WebP", ext: ".webp", args: []string{"-f", "lavfi", "-i", "testsrc2=size=32x32:rate=2:duration=1", "-c:v", "libwebp_anim", "-loop", "0"}}, + {name: "larger encoded audio", ext: ".wav", args: []string{"-f", "lavfi", "-i", "sine=duration=0.001", "-c:a", "pcm_s16le"}}, + } { + t.Run(tc.name, func(t *testing.T) { + raw := makeMedia(t, tc.ext, tc.args...) + dir := t.TempDir() + filename := filepath.Join(dir, "source") + if err := os.WriteFile(filename, raw, 0600); err != nil { + t.Fatal(err) + } + mime, _ := mediaType(raw[:min(512, len(raw))]) + original := describeBytes("original", mime, raw) + info, path, err := optimizeMedia(context.Background(), filename, original) + if err != nil || info != original || path != filename { + t.Fatalf("original not preserved: %+v %q %v", info, path, err) + } + entries, err := os.ReadDir(dir) + if err != nil || len(entries) != 1 { + t.Fatalf("temporary output remains: %v %v", entries, err) + } + }) + } +} + +func TestMediaCancellationStopsProcess(t *testing.T) { + for _, tc := range []struct{ name string }{{name: "running converter"}} { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "ffmpeg"), []byte("#!/bin/sh\nexec /bin/sleep 10\n"), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir) + filename := filepath.Join(dir, "source") + raw := []byte("ID3original") + if err := os.WriteFile(filename, raw, 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + start := time.Now() + _, _, err := optimizeMedia(ctx, filename, describeBytes("original", "audio/mpeg", raw)) + if !errors.Is(err, context.DeadlineExceeded) || time.Since(start) > 2*time.Second { + t.Fatalf("converter did not stop on cancellation: %v", err) + } + }) + } +} + +func TestMediaOptimizationFallback(t *testing.T) { + for _, tc := range []struct { + name string + raw []byte + missingTools, cancelled bool + }{ + {name: "PDF", raw: []byte("%PDF-1.7\noriginal")}, + {name: "invalid image", raw: []byte("\x89PNG\r\n\x1a\ninvalid")}, + {name: "missing tools", missingTools: true}, + {name: "invalid audio", raw: []byte("ID3\x04\x00\x00tiny")}, + {name: "cancelled", cancelled: true}, + {name: "animated GIF"}, + } { + t.Run(tc.name, func(t *testing.T) { + raw := tc.raw + if tc.name == "animated GIF" { + frame := image.NewPaletted(image.Rect(0, 0, 2, 2), color.Palette{color.Black, color.White}) + var buffer bytes.Buffer + if err := gif.EncodeAll(&buffer, &gif.GIF{Image: []*image.Paletted{frame, frame}, Delay: []int{10, 10}}); err != nil { + t.Fatal(err) + } + raw = buffer.Bytes() + } + if raw == nil { + raw = mediaPNG(t, 16, 16) + } + if tc.missingTools { + t.Setenv("PATH", t.TempDir()) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if tc.cancelled { + cancel() + } + dir := t.TempDir() + filename := filepath.Join(dir, "source") + if err := os.WriteFile(filename, raw, 0600); err != nil { + t.Fatal(err) + } + mime, _ := mediaType(raw[:min(512, len(raw))]) + original := describeBytes("original", mime, raw) + info, path, err := optimizeMedia(ctx, filename, original) + if tc.cancelled { + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected cancellation, got %v", err) + } + return + } + if err != nil || info != original || path != filename { + t.Fatalf("original not preserved: %+v %q %v", info, path, err) + } + entries, err := os.ReadDir(dir) + if err != nil || len(entries) != 1 { + t.Fatalf("temporary output remains: %v %v", entries, err) + } + }) + } +} diff --git a/internal/services/game_package_service/service.go b/internal/services/game_package_service/service.go index 26d4059..307998d 100644 --- a/internal/services/game_package_service/service.go +++ b/internal/services/game_package_service/service.go @@ -317,12 +317,13 @@ func (s *GamePackageService) stageMedia(ctx context.Context, dir, ref string, op return FileInfo{}, "", invalid("неподдерживаемое содержимое медиафайла") } digest := hex.EncodeToString(hash.Sum(nil)) - return FileInfo{ + info := FileInfo{ Path: "files/" + digest + ext, Size: n, MIME: mime, SHA256: digest, - }, file.Name(), nil + } + return optimizeMedia(ctx, file.Name(), info) } func mediaType(header []byte) (string, string) {