Nathan/surfnathanrip
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

S3 multipart upload for large videos, EISDIR fix

Browse Source
This commit is contained in:
CallMeVerity
2026-06-03 03:30:54 +01:00
parent d6cd848257
commit 1693b3849b
4 changed files with 217 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/routes/videos.ts
+75 -12
View File
@@ -13,7 +13,10 @@ import {
uploadObject,
deleteObject,
getObjectUrl,
getPresignedUploadUrl,
createMultipartUpload,
getPresignedUploadPartUrl,
completeMultipartUpload,
abortMultipartUpload,
} from "../services/rustfs";
import { parseMtvFile } from "../services/mtv-parser";
import { getMapInfo } from "../services/momentum-api";
@@ -75,14 +78,23 @@ export const videoRoutes = new Elysia({ prefix: "/api/videos" })
const videoFileName = formData.get("videoFileName") as string | null;
const videoContentType =
(formData.get("videoContentType") as string | null) || "video/webm";
const videoFileSizeStr = formData.get("videoFileSize") as string | null;
const runDate = formData.get("runDate") as string | null;
if (!mtv || !videoFileName) {
if (!mtv || !videoFileName || !videoFileSizeStr) {
return status(400, {
error: "mtv file and videoFileName are required",
error: "mtv file, videoFileName, and videoFileSize are required",
});
}
const videoFileSize = parseInt(videoFileSizeStr, 10);
if (isNaN(videoFileSize) || videoFileSize <= 0) {
return status(400, { error: "Invalid videoFileSize" });
}
const CHUNK_SIZE = 80 * 1024 * 1024;
const numParts = Math.ceil(videoFileSize / CHUNK_SIZE);
const mtvBuffer = await mtv.arrayBuffer();
const metadata = parseMtvFile(mtvBuffer);
if (!metadata) {
@@ -176,13 +188,30 @@ export const videoRoutes = new Elysia({ prefix: "/api/videos" })
updateVideoPB(id, entry, previousPbs);
const presignedUrls = {
video: await getPresignedUploadUrl(videoKey, videoContentType),
};
const { uploadId } = await createMultipartUpload(
videoKey,
videoContentType,
);
const parts = await Promise.all(
Array.from({ length: numParts }, (_, i) => i + 1).map(
async (partNumber) => ({
partNumber,
url: await getPresignedUploadPartUrl(
videoKey,
uploadId,
partNumber,
),
}),
),
);
return {
...entry,
presignedUrls,
presignedUrls: {
parts,
uploadId,
},
videoUrl: getObjectUrl(videoKey),
mtvUrl: getObjectUrl(mtvKey),
thumbnailUrl: thumbnailKey
@@ -251,23 +280,57 @@ export const videoRoutes = new Elysia({ prefix: "/api/videos" })
insertVideo(entry);
const presignedUrls = {
video: await getPresignedUploadUrl(videoKey, videoContentType),
};
const { uploadId } = await createMultipartUpload(
videoKey,
videoContentType,
);
const parts = await Promise.all(
Array.from({ length: numParts }, (_, i) => i + 1).map(
async (partNumber) => ({
partNumber,
url: await getPresignedUploadPartUrl(
videoKey,
uploadId,
partNumber,
),
}),
),
);
return {
...entry,
presignedUrls,
presignedUrls: {
parts,
uploadId,
},
videoUrl: getObjectUrl(videoKey),
mtvUrl: getObjectUrl(mtvKey),
thumbnailUrl: thumbnailKey ? getObjectUrl(thumbnailKey) : undefined,
};
})
.post("/:id/complete", async ({ params: { id } }) => {
.post("/:id/complete", async ({ params: { id }, body }) => {
const video = getVideoById(id);
if (!video) return status(404, { error: "Not found" });
const { parts, uploadId } = body as {
parts: { partNumber: number; eTag: string }[];
uploadId: string;
};
if (!parts || !Array.isArray(parts) || !uploadId) {
return status(400, {
error: "parts array and uploadId are required",
});
}
const s3Parts = parts.map((p) => ({
PartNumber: p.partNumber,
ETag: p.eTag,
}));
await completeMultipartUpload(video.videoKey, uploadId, s3Parts);
return {
...video,
videoUrl: getObjectUrl(video.videoKey),
backend/src/services/rustfs.ts
+70
View File
@@ -2,6 +2,10 @@ import {
S3Client,
PutObjectCommand,
DeleteObjectCommand,
CreateMultipartUploadCommand,
UploadPartCommand,
CompleteMultipartUploadCommand,
AbortMultipartUploadCommand,
} from "@aws-sdk/client-s3";
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
@@ -80,3 +84,69 @@ export async function getPresignedUploadUrl(
});
return getSignedUrl(s3, command, { expiresIn: expiresInSeconds });
}
export async function createMultipartUpload(
key: string,
contentType: string,
): Promise<{ uploadId: string }> {
const s3 = getRustFsClient();
const result = await s3.send(
new CreateMultipartUploadCommand({
Bucket: RUSTFS_BUCKET,
Key: key,
ContentType: contentType,
}),
);
if (!result.UploadId) {
throw new Error("Failed to initiate multipart upload");
}
return { uploadId: result.UploadId };
}
export async function getPresignedUploadPartUrl(
key: string,
uploadId: string,
partNumber: number,
expiresInSeconds = 3600,
): Promise<string> {
const s3 = getRustFsClient();
const command = new UploadPartCommand({
Bucket: RUSTFS_BUCKET,
Key: key,
UploadId: uploadId,
PartNumber: partNumber,
});
return getSignedUrl(s3, command, { expiresIn: expiresInSeconds });
}
export async function completeMultipartUpload(
key: string,
uploadId: string,
parts: { PartNumber: number; ETag: string }[],
): Promise<void> {
const s3 = getRustFsClient();
await s3.send(
new CompleteMultipartUploadCommand({
Bucket: RUSTFS_BUCKET,
Key: key,
UploadId: uploadId,
MultipartUpload: {
Parts: parts.sort((a, b) => a.PartNumber - b.PartNumber),
},
}),
);
}
export async function abortMultipartUpload(
key: string,
uploadId: string,
): Promise<void> {
const s3 = getRustFsClient();
await s3.send(
new AbortMultipartUploadCommand({
Bucket: RUSTFS_BUCKET,
Key: key,
UploadId: uploadId,
}),
);
}
frontend/src/api/client.ts
+19 -3
View File
@@ -26,10 +26,16 @@ export async function fetchVideo(id: string): Promise<VideoDetail> {
return res.json();
}
export interface UploadPartUrl {
partNumber: number;
url: string;
}
export interface UploadInitResponse {
id: string;
presignedUrls: {
video: string;
parts: UploadPartUrl[];
uploadId: string;
};
videoUrl: string;
mtvUrl: string;
@@ -45,12 +51,14 @@ export async function uploadVideoInit(
mtvFile: File,
videoFileName: string,
videoContentType: string,
videoFileSize: string,
runDate?: string,
): Promise<UploadInitResponse> {
const formData = new FormData();
formData.append("mtv", mtvFile);
formData.append("videoFileName", videoFileName);
formData.append("videoContentType", videoContentType);
formData.append("videoFileSize", videoFileSize);
if (runDate) formData.append("runDate", runDate);
const res = await fetch(`${API_BASE}/videos/upload-url`, {
@@ -67,10 +75,18 @@ export async function uploadVideoInit(
return res.json();
}
export async function uploadVideoComplete(id: string): Promise<VideoDetail> {
export async function uploadVideoComplete(
id: string,
parts: { partNumber: number; eTag: string }[],
uploadId: string,
): Promise<VideoDetail> {
const res = await fetch(`${API_BASE}/videos/${id}/complete`, {
method: "POST",
headers: authHeaders(),
headers: {
...authHeaders(),
"Content-Type": "application/json",
},
body: JSON.stringify({ parts, uploadId }),
});
if (!res.ok) {
if (res.status === 401)
frontend/src/components/UploadForm.tsx
+53 -28
View File
@@ -252,42 +252,67 @@ export default function UploadForm() {
mtvFile,
videoFile.name,
videoFile.type || "video/webm",
videoFile.size.toString(),
runDate,
);
const { parts, uploadId } = initRes.presignedUrls;
const totalSize = videoFile.size;
const CHUNK_SIZE = 80 * 1024 * 1024;
const uploadedParts: { partNumber: number; eTag: string }[] = [];
await new Promise<void>((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.upload.addEventListener("progress", (e) => {
if (e.lengthComputable) {
setProgress(
5 + Math.round((e.loaded / totalSize) * 90),
);
}
for (let i = 0; i < parts.length; i++) {
const part = parts[i]!;
const start = i * CHUNK_SIZE;
const end = Math.min(start + CHUNK_SIZE, totalSize);
const chunk = videoFile.slice(start, end);
const eTag = await new Promise<string>((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.upload.addEventListener("progress", (e) => {
if (e.lengthComputable) {
const chunkProgress = e.loaded / e.total;
const overallUploaded = start + e.loaded;
setProgress(
5 +
Math.round(
(overallUploaded / totalSize) * 90,
),
);
}
});
xhr.addEventListener("load", () => {
if (xhr.status >= 200 && xhr.status < 300) {
const etag = xhr.getResponseHeader("ETag");
if (etag) resolve(etag);
else
reject(
new Error("No ETag returned from upload"),
);
} else {
reject(
new Error(
`Upload failed with status ${xhr.status}`,
),
);
}
});
xhr.addEventListener("error", () =>
reject(new Error("Upload failed")),
);
xhr.open("PUT", part.url);
xhr.setRequestHeader(
"Content-Type",
videoFile.type || "video/webm",
);
xhr.send(chunk);
});
xhr.addEventListener("load", () => {
if (xhr.status >= 200 && xhr.status < 300) resolve();
else
reject(
new Error(
`Upload failed with status ${xhr.status}`,
),
);
});
xhr.addEventListener("error", () =>
reject(new Error("Upload failed")),
);
xhr.open("PUT", initRes.presignedUrls.video);
xhr.setRequestHeader(
"Content-Type",
videoFile.type || "video/webm",
);
xhr.send(videoFile);
});
uploadedParts.push({ partNumber: part.partNumber, eTag });
}
setProgress(98);
await uploadVideoComplete(initRes.id);
await uploadVideoComplete(initRes.id, uploadedParts, uploadId);
setProgress(100);
setSuccess(true);