S3 multipart upload for large videos, EISDIR fix

backend/src/routes/videos.ts

@@ -13,7 +13,10 @@ import {
     uploadObject,
     deleteObject,
     getObjectUrl,
-    getPresignedUploadUrl,
+    createMultipartUpload,
+    getPresignedUploadPartUrl,
+    completeMultipartUpload,
+    abortMultipartUpload,
 } from "../services/rustfs";
 import { parseMtvFile } from "../services/mtv-parser";
 import { getMapInfo } from "../services/momentum-api";
@@ -75,14 +78,23 @@ export const videoRoutes = new Elysia({ prefix: "/api/videos" })
         const videoFileName = formData.get("videoFileName") as string | null;
         const videoContentType =
             (formData.get("videoContentType") as string | null) || "video/webm";
+        const videoFileSizeStr = formData.get("videoFileSize") as string | null;
         const runDate = formData.get("runDate") as string | null;
 
-        if (!mtv || !videoFileName) {
+        if (!mtv || !videoFileName || !videoFileSizeStr) {
             return status(400, {
-                error: "mtv file and videoFileName are required",
+                error: "mtv file, videoFileName, and videoFileSize are required",
             });
         }
 
+        const videoFileSize = parseInt(videoFileSizeStr, 10);
+        if (isNaN(videoFileSize) || videoFileSize <= 0) {
+            return status(400, { error: "Invalid videoFileSize" });
+        }
+
+        const CHUNK_SIZE = 80 * 1024 * 1024;
+        const numParts = Math.ceil(videoFileSize / CHUNK_SIZE);
+
         const mtvBuffer = await mtv.arrayBuffer();
         const metadata = parseMtvFile(mtvBuffer);
         if (!metadata) {
@@ -176,13 +188,30 @@ export const videoRoutes = new Elysia({ prefix: "/api/videos" })
 
             updateVideoPB(id, entry, previousPbs);
 
-            const presignedUrls = {
-                video: await getPresignedUploadUrl(videoKey, videoContentType),
-            };
+            const { uploadId } = await createMultipartUpload(
+                videoKey,
+                videoContentType,
+            );
+
+            const parts = await Promise.all(
+                Array.from({ length: numParts }, (_, i) => i + 1).map(
+                    async (partNumber) => ({
+                        partNumber,
+                        url: await getPresignedUploadPartUrl(
+                            videoKey,
+                            uploadId,
+                            partNumber,
+                        ),
+                    }),
+                ),
+            );
 
             return {
                 ...entry,
-                presignedUrls,
+                presignedUrls: {
+                    parts,
+                    uploadId,
+                },
                 videoUrl: getObjectUrl(videoKey),
                 mtvUrl: getObjectUrl(mtvKey),
                 thumbnailUrl: thumbnailKey
@@ -251,23 +280,57 @@ export const videoRoutes = new Elysia({ prefix: "/api/videos" })
 
         insertVideo(entry);
 
-        const presignedUrls = {
-            video: await getPresignedUploadUrl(videoKey, videoContentType),
-        };
+        const { uploadId } = await createMultipartUpload(
+            videoKey,
+            videoContentType,
+        );
+
+        const parts = await Promise.all(
+            Array.from({ length: numParts }, (_, i) => i + 1).map(
+                async (partNumber) => ({
+                    partNumber,
+                    url: await getPresignedUploadPartUrl(
+                        videoKey,
+                        uploadId,
+                        partNumber,
+                    ),
+                }),
+            ),
+        );
 
         return {
             ...entry,
-            presignedUrls,
+            presignedUrls: {
+                parts,
+                uploadId,
+            },
             videoUrl: getObjectUrl(videoKey),
             mtvUrl: getObjectUrl(mtvKey),
             thumbnailUrl: thumbnailKey ? getObjectUrl(thumbnailKey) : undefined,
         };
     })
 
-    .post("/:id/complete", async ({ params: { id } }) => {
+    .post("/:id/complete", async ({ params: { id }, body }) => {
         const video = getVideoById(id);
         if (!video) return status(404, { error: "Not found" });
 
+        const { parts, uploadId } = body as {
+            parts: { partNumber: number; eTag: string }[];
+            uploadId: string;
+        };
+        if (!parts || !Array.isArray(parts) || !uploadId) {
+            return status(400, {
+                error: "parts array and uploadId are required",
+            });
+        }
+
+        const s3Parts = parts.map((p) => ({
+            PartNumber: p.partNumber,
+            ETag: p.eTag,
+        }));
+
+        await completeMultipartUpload(video.videoKey, uploadId, s3Parts);
+
         return {
             ...video,
             videoUrl: getObjectUrl(video.videoKey),

backend/src/services/rustfs.ts

@@ -2,6 +2,10 @@ import {
     S3Client,
     PutObjectCommand,
     DeleteObjectCommand,
+    CreateMultipartUploadCommand,
+    UploadPartCommand,
+    CompleteMultipartUploadCommand,
+    AbortMultipartUploadCommand,
 } from "@aws-sdk/client-s3";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
 
@@ -80,3 +84,69 @@ export async function getPresignedUploadUrl(
     });
     return getSignedUrl(s3, command, { expiresIn: expiresInSeconds });
 }
+
+export async function createMultipartUpload(
+    key: string,
+    contentType: string,
+): Promise<{ uploadId: string }> {
+    const s3 = getRustFsClient();
+    const result = await s3.send(
+        new CreateMultipartUploadCommand({
+            Bucket: RUSTFS_BUCKET,
+            Key: key,
+            ContentType: contentType,
+        }),
+    );
+    if (!result.UploadId) {
+        throw new Error("Failed to initiate multipart upload");
+    }
+    return { uploadId: result.UploadId };
+}
+
+export async function getPresignedUploadPartUrl(
+    key: string,
+    uploadId: string,
+    partNumber: number,
+    expiresInSeconds = 3600,
+): Promise<string> {
+    const s3 = getRustFsClient();
+    const command = new UploadPartCommand({
+        Bucket: RUSTFS_BUCKET,
+        Key: key,
+        UploadId: uploadId,
+        PartNumber: partNumber,
+    });
+    return getSignedUrl(s3, command, { expiresIn: expiresInSeconds });
+}
+
+export async function completeMultipartUpload(
+    key: string,
+    uploadId: string,
+    parts: { PartNumber: number; ETag: string }[],
+): Promise<void> {
+    const s3 = getRustFsClient();
+    await s3.send(
+        new CompleteMultipartUploadCommand({
+            Bucket: RUSTFS_BUCKET,
+            Key: key,
+            UploadId: uploadId,
+            MultipartUpload: {
+                Parts: parts.sort((a, b) => a.PartNumber - b.PartNumber),
+            },
+        }),
+    );
+}
+
+export async function abortMultipartUpload(
+    key: string,
+    uploadId: string,
+): Promise<void> {
+    const s3 = getRustFsClient();
+    await s3.send(
+        new AbortMultipartUploadCommand({
+            Bucket: RUSTFS_BUCKET,
+            Key: key,
+            UploadId: uploadId,
+        }),
+    );
+}