diff --git a/backend/src/routes/videos.ts b/backend/src/routes/videos.ts index 58a7335..9f1425b 100644 --- a/backend/src/routes/videos.ts +++ b/backend/src/routes/videos.ts @@ -13,7 +13,10 @@ import { uploadObject, deleteObject, getObjectUrl, - getPresignedUploadUrl, + createMultipartUpload, + getPresignedUploadPartUrl, + completeMultipartUpload, + abortMultipartUpload, } from "../services/rustfs"; import { parseMtvFile } from "../services/mtv-parser"; import { getMapInfo } from "../services/momentum-api"; @@ -75,14 +78,23 @@ export const videoRoutes = new Elysia({ prefix: "/api/videos" }) const videoFileName = formData.get("videoFileName") as string | null; const videoContentType = (formData.get("videoContentType") as string | null) || "video/webm"; + const videoFileSizeStr = formData.get("videoFileSize") as string | null; const runDate = formData.get("runDate") as string | null; - if (!mtv || !videoFileName) { + if (!mtv || !videoFileName || !videoFileSizeStr) { return status(400, { - error: "mtv file and videoFileName are required", + error: "mtv file, videoFileName, and videoFileSize are required", }); } + const videoFileSize = parseInt(videoFileSizeStr, 10); + if (isNaN(videoFileSize) || videoFileSize <= 0) { + return status(400, { error: "Invalid videoFileSize" }); + } + + const CHUNK_SIZE = 80 * 1024 * 1024; + const numParts = Math.ceil(videoFileSize / CHUNK_SIZE); + const mtvBuffer = await mtv.arrayBuffer(); const metadata = parseMtvFile(mtvBuffer); if (!metadata) { @@ -176,13 +188,30 @@ export const videoRoutes = new Elysia({ prefix: "/api/videos" }) updateVideoPB(id, entry, previousPbs); - const presignedUrls = { - video: await getPresignedUploadUrl(videoKey, videoContentType), - }; + const { uploadId } = await createMultipartUpload( + videoKey, + videoContentType, + ); + + const parts = await Promise.all( + Array.from({ length: numParts }, (_, i) => i + 1).map( + async (partNumber) => ({ + partNumber, + url: await getPresignedUploadPartUrl( + videoKey, + uploadId, + partNumber, + ), + }), + ), + ); return { ...entry, - presignedUrls, + presignedUrls: { + parts, + uploadId, + }, videoUrl: getObjectUrl(videoKey), mtvUrl: getObjectUrl(mtvKey), thumbnailUrl: thumbnailKey @@ -251,23 +280,57 @@ export const videoRoutes = new Elysia({ prefix: "/api/videos" }) insertVideo(entry); - const presignedUrls = { - video: await getPresignedUploadUrl(videoKey, videoContentType), - }; + const { uploadId } = await createMultipartUpload( + videoKey, + videoContentType, + ); + + const parts = await Promise.all( + Array.from({ length: numParts }, (_, i) => i + 1).map( + async (partNumber) => ({ + partNumber, + url: await getPresignedUploadPartUrl( + videoKey, + uploadId, + partNumber, + ), + }), + ), + ); return { ...entry, - presignedUrls, + presignedUrls: { + parts, + uploadId, + }, videoUrl: getObjectUrl(videoKey), mtvUrl: getObjectUrl(mtvKey), thumbnailUrl: thumbnailKey ? getObjectUrl(thumbnailKey) : undefined, }; }) - .post("/:id/complete", async ({ params: { id } }) => { + .post("/:id/complete", async ({ params: { id }, body }) => { const video = getVideoById(id); if (!video) return status(404, { error: "Not found" }); + const { parts, uploadId } = body as { + parts: { partNumber: number; eTag: string }[]; + uploadId: string; + }; + if (!parts || !Array.isArray(parts) || !uploadId) { + return status(400, { + error: "parts array and uploadId are required", + }); + } + + const s3Parts = parts.map((p) => ({ + PartNumber: p.partNumber, + ETag: p.eTag, + })); + + await completeMultipartUpload(video.videoKey, uploadId, s3Parts); + return { ...video, videoUrl: getObjectUrl(video.videoKey), diff --git a/backend/src/services/rustfs.ts b/backend/src/services/rustfs.ts index 213c4ad..e1ff736 100644 --- a/backend/src/services/rustfs.ts +++ b/backend/src/services/rustfs.ts @@ -2,6 +2,10 @@ import { S3Client, PutObjectCommand, DeleteObjectCommand, + CreateMultipartUploadCommand, + UploadPartCommand, + CompleteMultipartUploadCommand, + AbortMultipartUploadCommand, } from "@aws-sdk/client-s3"; import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; @@ -80,3 +84,69 @@ export async function getPresignedUploadUrl( }); return getSignedUrl(s3, command, { expiresIn: expiresInSeconds }); } + +export async function createMultipartUpload( + key: string, + contentType: string, +): Promise<{ uploadId: string }> { + const s3 = getRustFsClient(); + const result = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: RUSTFS_BUCKET, + Key: key, + ContentType: contentType, + }), + ); + if (!result.UploadId) { + throw new Error("Failed to initiate multipart upload"); + } + return { uploadId: result.UploadId }; +} + +export async function getPresignedUploadPartUrl( + key: string, + uploadId: string, + partNumber: number, + expiresInSeconds = 3600, +): Promise { + const s3 = getRustFsClient(); + const command = new UploadPartCommand({ + Bucket: RUSTFS_BUCKET, + Key: key, + UploadId: uploadId, + PartNumber: partNumber, + }); + return getSignedUrl(s3, command, { expiresIn: expiresInSeconds }); +} + +export async function completeMultipartUpload( + key: string, + uploadId: string, + parts: { PartNumber: number; ETag: string }[], +): Promise { + const s3 = getRustFsClient(); + await s3.send( + new CompleteMultipartUploadCommand({ + Bucket: RUSTFS_BUCKET, + Key: key, + UploadId: uploadId, + MultipartUpload: { + Parts: parts.sort((a, b) => a.PartNumber - b.PartNumber), + }, + }), + ); +} + +export async function abortMultipartUpload( + key: string, + uploadId: string, +): Promise { + const s3 = getRustFsClient(); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: RUSTFS_BUCKET, + Key: key, + UploadId: uploadId, + }), + ); +} diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 689a403..5c535ab 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -26,10 +26,16 @@ export async function fetchVideo(id: string): Promise { return res.json(); } +export interface UploadPartUrl { + partNumber: number; + url: string; +} + export interface UploadInitResponse { id: string; presignedUrls: { - video: string; + parts: UploadPartUrl[]; + uploadId: string; }; videoUrl: string; mtvUrl: string; @@ -45,12 +51,14 @@ export async function uploadVideoInit( mtvFile: File, videoFileName: string, videoContentType: string, + videoFileSize: string, runDate?: string, ): Promise { const formData = new FormData(); formData.append("mtv", mtvFile); formData.append("videoFileName", videoFileName); formData.append("videoContentType", videoContentType); + formData.append("videoFileSize", videoFileSize); if (runDate) formData.append("runDate", runDate); const res = await fetch(`${API_BASE}/videos/upload-url`, { @@ -67,10 +75,18 @@ export async function uploadVideoInit( return res.json(); } -export async function uploadVideoComplete(id: string): Promise { +export async function uploadVideoComplete( + id: string, + parts: { partNumber: number; eTag: string }[], + uploadId: string, +): Promise { const res = await fetch(`${API_BASE}/videos/${id}/complete`, { method: "POST", - headers: authHeaders(), + headers: { + ...authHeaders(), + "Content-Type": "application/json", + }, + body: JSON.stringify({ parts, uploadId }), }); if (!res.ok) { if (res.status === 401) diff --git a/frontend/src/components/UploadForm.tsx b/frontend/src/components/UploadForm.tsx index bdc4dfa..405e55a 100644 --- a/frontend/src/components/UploadForm.tsx +++ b/frontend/src/components/UploadForm.tsx @@ -252,42 +252,67 @@ export default function UploadForm() { mtvFile, videoFile.name, videoFile.type || "video/webm", + videoFile.size.toString(), runDate, ); + const { parts, uploadId } = initRes.presignedUrls; const totalSize = videoFile.size; + const CHUNK_SIZE = 80 * 1024 * 1024; + const uploadedParts: { partNumber: number; eTag: string }[] = []; - await new Promise((resolve, reject) => { - const xhr = new XMLHttpRequest(); - xhr.upload.addEventListener("progress", (e) => { - if (e.lengthComputable) { - setProgress( - 5 + Math.round((e.loaded / totalSize) * 90), - ); - } + for (let i = 0; i < parts.length; i++) { + const part = parts[i]!; + const start = i * CHUNK_SIZE; + const end = Math.min(start + CHUNK_SIZE, totalSize); + const chunk = videoFile.slice(start, end); + + const eTag = await new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest(); + xhr.upload.addEventListener("progress", (e) => { + if (e.lengthComputable) { + const chunkProgress = e.loaded / e.total; + const overallUploaded = start + e.loaded; + setProgress( + 5 + + Math.round( + (overallUploaded / totalSize) * 90, + ), + ); + } + }); + xhr.addEventListener("load", () => { + if (xhr.status >= 200 && xhr.status < 300) { + const etag = xhr.getResponseHeader("ETag"); + if (etag) resolve(etag); + else + reject( + new Error("No ETag returned from upload"), + ); + } else { + reject( + new Error( + `Upload failed with status ${xhr.status}`, + ), + ); + } + }); + xhr.addEventListener("error", () => + reject(new Error("Upload failed")), + ); + xhr.open("PUT", part.url); + xhr.setRequestHeader( + "Content-Type", + videoFile.type || "video/webm", + ); + xhr.send(chunk); }); - xhr.addEventListener("load", () => { - if (xhr.status >= 200 && xhr.status < 300) resolve(); - else - reject( - new Error( - `Upload failed with status ${xhr.status}`, - ), - ); - }); - xhr.addEventListener("error", () => - reject(new Error("Upload failed")), - ); - xhr.open("PUT", initRes.presignedUrls.video); - xhr.setRequestHeader( - "Content-Type", - videoFile.type || "video/webm", - ); - xhr.send(videoFile); - }); + + uploadedParts.push({ partNumber: part.partNumber, eTag }); + } setProgress(98); - await uploadVideoComplete(initRes.id); + await uploadVideoComplete(initRes.id, uploadedParts, uploadId); setProgress(100); setSuccess(true);