S3 multipart upload for large videos, EISDIR fix

backend/src/routes/videos.ts

@@ -13,7 +13,10 @@ import {
     uploadObject,
     deleteObject,
     getObjectUrl,
-    getPresignedUploadUrl,
+    createMultipartUpload,
+    getPresignedUploadPartUrl,
+    completeMultipartUpload,
+    abortMultipartUpload,
 } from "../services/rustfs";
 import { parseMtvFile } from "../services/mtv-parser";
 import { getMapInfo } from "../services/momentum-api";
@@ -75,14 +78,23 @@ export const videoRoutes = new Elysia({ prefix: "/api/videos" })
         const videoFileName = formData.get("videoFileName") as string | null;
         const videoContentType =
             (formData.get("videoContentType") as string | null) || "video/webm";
+        const videoFileSizeStr = formData.get("videoFileSize") as string | null;
         const runDate = formData.get("runDate") as string | null;
 
-        if (!mtv || !videoFileName) {
+        if (!mtv || !videoFileName || !videoFileSizeStr) {
             return status(400, {
-                error: "mtv file and videoFileName are required",
+                error: "mtv file, videoFileName, and videoFileSize are required",
             });
         }
 
+        const videoFileSize = parseInt(videoFileSizeStr, 10);
+        if (isNaN(videoFileSize) || videoFileSize <= 0) {
+            return status(400, { error: "Invalid videoFileSize" });
+        }
+
+        const CHUNK_SIZE = 80 * 1024 * 1024;
+        const numParts = Math.ceil(videoFileSize / CHUNK_SIZE);
+
         const mtvBuffer = await mtv.arrayBuffer();
         const metadata = parseMtvFile(mtvBuffer);
         if (!metadata) {
@@ -176,13 +188,30 @@ export const videoRoutes = new Elysia({ prefix: "/api/videos" })
 
             updateVideoPB(id, entry, previousPbs);
 
-            const presignedUrls = {
+            const { uploadId } = await createMultipartUpload(
-                video: await getPresignedUploadUrl(videoKey, videoContentType),
+                videoKey,
-            };
+                videoContentType,
+            );
+
+            const parts = await Promise.all(
+                Array.from({ length: numParts }, (_, i) => i + 1).map(
+                    async (partNumber) => ({
+                        partNumber,
+                        url: await getPresignedUploadPartUrl(
+                            videoKey,
+                            uploadId,
+                            partNumber,
+                        ),
+                    }),
+                ),
+            );
 
             return {
                 ...entry,
-                presignedUrls,
+                presignedUrls: {
+                    parts,
+                    uploadId,
+                },
                 videoUrl: getObjectUrl(videoKey),
                 mtvUrl: getObjectUrl(mtvKey),
                 thumbnailUrl: thumbnailKey
@@ -251,23 +280,57 @@ export const videoRoutes = new Elysia({ prefix: "/api/videos" })
 
         insertVideo(entry);
 
-        const presignedUrls = {
+        const { uploadId } = await createMultipartUpload(
-            video: await getPresignedUploadUrl(videoKey, videoContentType),
+            videoKey,
-        };
+            videoContentType,
+        );
+
+        const parts = await Promise.all(
+            Array.from({ length: numParts }, (_, i) => i + 1).map(
+                async (partNumber) => ({
+                    partNumber,
+                    url: await getPresignedUploadPartUrl(
+                        videoKey,
+                        uploadId,
+                        partNumber,
+                    ),
+                }),
+            ),
+        );
 
         return {
             ...entry,
-            presignedUrls,
+            presignedUrls: {
+                parts,
+                uploadId,
+            },
             videoUrl: getObjectUrl(videoKey),
             mtvUrl: getObjectUrl(mtvKey),
             thumbnailUrl: thumbnailKey ? getObjectUrl(thumbnailKey) : undefined,
         };
     })
 
-    .post("/:id/complete", async ({ params: { id } }) => {
+    .post("/:id/complete", async ({ params: { id }, body }) => {
         const video = getVideoById(id);
         if (!video) return status(404, { error: "Not found" });
 
+        const { parts, uploadId } = body as {
+            parts: { partNumber: number; eTag: string }[];
+            uploadId: string;
+        };
+        if (!parts || !Array.isArray(parts) || !uploadId) {
+            return status(400, {
+                error: "parts array and uploadId are required",
+            });
+        }
+
+        const s3Parts = parts.map((p) => ({
+            PartNumber: p.partNumber,
+            ETag: p.eTag,
+        }));
+
+        await completeMultipartUpload(video.videoKey, uploadId, s3Parts);
+
         return {
             ...video,
             videoUrl: getObjectUrl(video.videoKey),

backend/src/services/rustfs.ts

@@ -2,6 +2,10 @@ import {
     S3Client,
     PutObjectCommand,
     DeleteObjectCommand,
+    CreateMultipartUploadCommand,
+    UploadPartCommand,
+    CompleteMultipartUploadCommand,
+    AbortMultipartUploadCommand,
 } from "@aws-sdk/client-s3";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
 
@@ -80,3 +84,69 @@ export async function getPresignedUploadUrl(
     });
     return getSignedUrl(s3, command, { expiresIn: expiresInSeconds });
 }
+
+export async function createMultipartUpload(
+    key: string,
+    contentType: string,
+): Promise<{ uploadId: string }> {
+    const s3 = getRustFsClient();
+    const result = await s3.send(
+        new CreateMultipartUploadCommand({
+            Bucket: RUSTFS_BUCKET,
+            Key: key,
+            ContentType: contentType,
+        }),
+    );
+    if (!result.UploadId) {
+        throw new Error("Failed to initiate multipart upload");
+    }
+    return { uploadId: result.UploadId };
+}
+
+export async function getPresignedUploadPartUrl(
+    key: string,
+    uploadId: string,
+    partNumber: number,
+    expiresInSeconds = 3600,
+): Promise<string> {
+    const s3 = getRustFsClient();
+    const command = new UploadPartCommand({
+        Bucket: RUSTFS_BUCKET,
+        Key: key,
+        UploadId: uploadId,
+        PartNumber: partNumber,
+    });
+    return getSignedUrl(s3, command, { expiresIn: expiresInSeconds });
+}
+
+export async function completeMultipartUpload(
+    key: string,
+    uploadId: string,
+    parts: { PartNumber: number; ETag: string }[],
+): Promise<void> {
+    const s3 = getRustFsClient();
+    await s3.send(
+        new CompleteMultipartUploadCommand({
+            Bucket: RUSTFS_BUCKET,
+            Key: key,
+            UploadId: uploadId,
+            MultipartUpload: {
+                Parts: parts.sort((a, b) => a.PartNumber - b.PartNumber),
+            },
+        }),
+    );
+}
+
+export async function abortMultipartUpload(
+    key: string,
+    uploadId: string,
+): Promise<void> {
+    const s3 = getRustFsClient();
+    await s3.send(
+        new AbortMultipartUploadCommand({
+            Bucket: RUSTFS_BUCKET,
+            Key: key,
+            UploadId: uploadId,
+        }),
+    );
+}

frontend/src/api/client.ts

@@ -26,10 +26,16 @@ export async function fetchVideo(id: string): Promise<VideoDetail> {
     return res.json();
 }
 
+export interface UploadPartUrl {
+    partNumber: number;
+    url: string;
+}
+
 export interface UploadInitResponse {
     id: string;
     presignedUrls: {
-        video: string;
+        parts: UploadPartUrl[];
+        uploadId: string;
     };
     videoUrl: string;
     mtvUrl: string;
@@ -45,12 +51,14 @@ export async function uploadVideoInit(
     mtvFile: File,
     videoFileName: string,
     videoContentType: string,
+    videoFileSize: string,
     runDate?: string,
 ): Promise<UploadInitResponse> {
     const formData = new FormData();
     formData.append("mtv", mtvFile);
     formData.append("videoFileName", videoFileName);
     formData.append("videoContentType", videoContentType);
+    formData.append("videoFileSize", videoFileSize);
     if (runDate) formData.append("runDate", runDate);
 
     const res = await fetch(`${API_BASE}/videos/upload-url`, {
@@ -67,10 +75,18 @@ export async function uploadVideoInit(
     return res.json();
 }
 
-export async function uploadVideoComplete(id: string): Promise<VideoDetail> {
+export async function uploadVideoComplete(
+    id: string,
+    parts: { partNumber: number; eTag: string }[],
+    uploadId: string,
+): Promise<VideoDetail> {
     const res = await fetch(`${API_BASE}/videos/${id}/complete`, {
         method: "POST",
-        headers: authHeaders(),
+        headers: {
+            ...authHeaders(),
+            "Content-Type": "application/json",
+        },
+        body: JSON.stringify({ parts, uploadId }),
     });
     if (!res.ok) {
         if (res.status === 401)

frontend/src/components/UploadForm.tsx

@@ -252,42 +252,67 @@ export default function UploadForm() {
                 mtvFile,
                 videoFile.name,
                 videoFile.type || "video/webm",
+                videoFile.size.toString(),
                 runDate,
             );
 
+            const { parts, uploadId } = initRes.presignedUrls;
             const totalSize = videoFile.size;
+            const CHUNK_SIZE = 80 * 1024 * 1024;
+            const uploadedParts: { partNumber: number; eTag: string }[] = [];
 
-            await new Promise<void>((resolve, reject) => {
+            for (let i = 0; i < parts.length; i++) {
-                const xhr = new XMLHttpRequest();
+                const part = parts[i]!;
-                xhr.upload.addEventListener("progress", (e) => {
+                const start = i * CHUNK_SIZE;
-                    if (e.lengthComputable) {
+                const end = Math.min(start + CHUNK_SIZE, totalSize);
-                        setProgress(
+                const chunk = videoFile.slice(start, end);
-                            5 + Math.round((e.loaded / totalSize) * 90),
+
-                        );
+                const eTag = await new Promise<string>((resolve, reject) => {
-                    }
+                    const xhr = new XMLHttpRequest();
+                    xhr.upload.addEventListener("progress", (e) => {
+                        if (e.lengthComputable) {
+                            const chunkProgress = e.loaded / e.total;
+                            const overallUploaded = start + e.loaded;
+                            setProgress(
+                                5 +
+                                    Math.round(
+                                        (overallUploaded / totalSize) * 90,
+                                    ),
+                            );
+                        }
+                    });
+                    xhr.addEventListener("load", () => {
+                        if (xhr.status >= 200 && xhr.status < 300) {
+                            const etag = xhr.getResponseHeader("ETag");
+                            if (etag) resolve(etag);
+                            else
+                                reject(
+                                    new Error("No ETag returned from upload"),
+                                );
+                        } else {
+                            reject(
+                                new Error(
+                                    `Upload failed with status ${xhr.status}`,
+                                ),
+                            );
+                        }
+                    });
+                    xhr.addEventListener("error", () =>
+                        reject(new Error("Upload failed")),
+                    );
+                    xhr.open("PUT", part.url);
+                    xhr.setRequestHeader(
+                        "Content-Type",
+                        videoFile.type || "video/webm",
+                    );
+                    xhr.send(chunk);
                 });
-                xhr.addEventListener("load", () => {
+
-                    if (xhr.status >= 200 && xhr.status < 300) resolve();
+                uploadedParts.push({ partNumber: part.partNumber, eTag });
-                    else
+            }
-                        reject(
-                            new Error(
-                                `Upload failed with status ${xhr.status}`,
-                            ),
-                        );
-                });
-                xhr.addEventListener("error", () =>
-                    reject(new Error("Upload failed")),
-                );
-                xhr.open("PUT", initRes.presignedUrls.video);
-                xhr.setRequestHeader(
-                    "Content-Type",
-                    videoFile.type || "video/webm",
-                );
-                xhr.send(videoFile);
-            });
 
             setProgress(98);
-            await uploadVideoComplete(initRes.id);
+            await uploadVideoComplete(initRes.id, uploadedParts, uploadId);
             setProgress(100);
 
             setSuccess(true);