How to implement a slice upload service
A requirement encountered in a recent project requires that if the upload fails due to network outage or other circumstances, the next time you start uploading the same file, you can start uploading from the breakpoint. Baidu, found that to achieve this function, need backend cooperation, so I use KOA to achieve a simple slice upload service, used for the development of the front end of the trial. Now write down the implementation process for a reminder.
Train of thought
To implement breakpoint continuation, do the following:
- Gets the unique identity of the file
- Gets the length of the file
- Record the length that has been uploaded
- Record the data
- Slice up the file and upload it
- Merge slicing files
- Verify file integrity
These need to be done together with the back end and the front end.
implementation
Based on the above points, let’s look at how to implement a slice upload interface.
Record file metadata
We need to provide an interface for the front end to call, upload the metadata of the file, and generate an upload task according to the metadata. If the task is interrupted abnormally later, we can also obtain the progress of the current task according to the metadata. Metadata includes file name, unique file identifier, file length, and slice size. The unique identifier of the file is calculated by the hash algorithm. Here we choose the hash algorithm md5, which is a very common hash encryption algorithm, characterized by fast and stable.
The front-end code
/** * input file onChange callback */
async function onFileChange(e) {
const files = e.target.files;
const file = files[0];
const fileMetadata = await getFileMetadata(file); // Get file metadata
const task = await getTaskInfo(fileMetadata); // Upload metadata to get task information
const chunks = await getFileChunks(file, task.chunkSize); // Slice the file
readyUploadFiles[task.hash] = { task, chunks }; // Save task and slice information locally
updateTable();
}
/** * Get file meta information *@param {File}} file
*/
async function getFileMetadata(file) {
const hash = await getFileMd5(file); // Get file hash; The spark-MD5 library is used
const fileName = file.name;
const fileType = file.type;
const fileSize = file.size;
return { hash, fileName, fileType, fileSize };
}
/** * Get upload task information *@param {{hash: string, fileName: string, fileType: string, fileSize: number}} metadata
*/
async function getTaskInfo(metadata) {
return fetch("http://127.0.0.1:38080/api/task", {
method: "POST".body: JSON.stringify(metadata),
headers: { "Content-Type": "application/json" },
}).then((res) = > res.json());
}
Copy the codeBack-end interface code
import Koa from "koa";
import KoaRouter from "@koa/router";
const router = new KoaRouter({ prefix: "/api" });
const upload_map = {};
router.post("/task".(ctx) = > {
const metadata = ctx.request.body;
// Create a temporary folder to store the chunks file for subsequent data consolidation
makeTempDirByFileHash(metadata.hash);
let task = upload_map[metadata.hash];
if(! task) {// Save the task information for subsequent breakpoint continuations
task = { chunkSize: 500.currentChunk: 0.done: false. metadata }; upload_map[metadata.hash] = task; } ctx.body = task; });const app = new Koa();
app.use(router.routes());
Copy the codeUploading file slices
Once you get the upload task, slice the file according to the chunkSize in the task and upload it.
The front-end code
By recursively calling the function, chunks are uploaded in turn.
/** * Slice the file according to chunkSize *@param {File} file
* @param {number} chunkSize* /
async function getFileChunks(file, chunkSize) {
const result = [];
const chunks = Math.ceil(file.size / chunkSize);
for (let index = 0; index < chunks; index++) {
const start = index * chunkSize,
end = start + chunkSize >= file.size ? file.size : start + chunkSize;
result.push(file.slice(start, end));
}
return result;
}
/** * Start uploading slices *@param {*} task
* @param {*} chunks* /
async function beginUploadChunks(task, chunks) {
if (task.done) {
return;
}
const start = task.currentChunk * task.chunkSize;
const end =
start + task.chunkSize >= task.fileSize
? task.fileSize
: start + task.chunkSize;
try {
const nextTask = await uploadChunks(
task.hash,
chunks[task.currentChunk],
start,
end
);
readyUploadFiles[task.hash].task = nextTask;
updateTable();
await beginUploadChunks(nextTask, chunks);
} catch (error) {
console.error(error); }}/** * Upload chunk data *@param {string} hash
* @param {Blob} chunk
* @param {number} start
* @param {number} end* /
async function uploadChunks(hash, chunk, start, end) {
const data = new FormData();
data.append("hash", hash);
data.append("chunk", chunk);
data.append("start", start);
data.append("end", end);
const res = await fetch("http://127.0.0.1:38080/api/upload_chunk", {
method: "POST".body: data,
}).then((res) = > res.json());
if (res.error) {
throw new Error(res.error);
} else {
returnres; }}Copy the codeThe back-end code
The back end uses the KOA-body library to parse multipart/form-data data
import KoaBody from "koa-body";
app.use(KoaBody({ multipart: true }));
// Receive the chunk uploaded
router.post("/upload_chunk".async (ctx) => {
const upload = ctx.request.body;
const files = ctx.request.files;
if(! files) {return;
}
const { hash, start, end } = upload;
const { chunk } = files;
// Koa-body will automatically write the file in the form-data to the hard disk. We need to get the path to the file and write it to the temporary folder we created
let filePath;
if (chunk instanceof Array) {
filePath = chunk[0].path;
} else {
filePath = chunk.path;
}
const task = upload_map[hash];
if(task && ! task.done) {// Save chunk to a temporary folder
const chunkPath = getTempDirByHash(hash) + ` /${start}-${end}`;
const fileRead = fs.createReadStream(filePath);
const chunkWrite = fs.createWriteStream(chunkPath);
fileRead.pipe(chunkWrite);
// Wait for the write to complete
await new Promise((resolve) = > fileRead.on("end", resolve));
// Delete the temporary file koa-body saved for us
await fs.promises.unlink(filePath);
// Index of the next chunk
task.currentChunk++;
if (task.currentChunk >= Math.ceil(task.fileSize / task.chunkSize)) {
// Chunk uploads all task status to complete
(task.done as any) = true;
(task.currentChunk as any) = null;
}
ctx.body = task;
} else {
ctx.status = 400;
ctx.body = { error: "Task not created"}; }});Copy the codeFile merge and verification
After all slices have been uploaded, you can merge the slices and verify the integrity of the file
The front-end code
async function concatChunks(hash) {
return fetch("http://127.0.0.1:38080/api/concat_chunk", {
method: "POST".body: JSON.stringify({ hash }),
headers: { "Content-Type": "application/json" },
}).then((res) = > res.json());
}
Copy the codeThe back-end code
In the final merge step, we check the integrity of the file through the data
router.post("/concat_chunk".async (ctx) => {
const hash = ctx.request.body.hash;
const task = upload_map[hash];
if(! task) { ctx.body = {error: "Mission not found" };
ctx.status = 400;
return;
}
if(! task.done) { ctx.body = {error: "Not all files uploaded" };
ctx.status = 400;
return;
}
// Check whether the number of chunks is consistent
const chunkDir = getTempDirByHash(hash);
const chunkCount = Math.ceil(task.fileSize / task.chunkSize);
const chunkPaths = await fs.promises.readdir(chunkDir);
if(chunkCount ! == chunkPaths.length) { ctx.body = {error: "Inconsistent file slice verification" };
ctx.status = 400;
return;
}
const chunkFullPaths = chunkPaths
.sort((a, b) = > {
const a1 = a.split("-") [0];
const b1 = b.split("-") [0];
return Number(a1) - Number(b1);
})
.map((chunkPath) = > path.join(chunkDir, chunkPath));
const filePath = path.resolve(
path.join(__dirname, ".. /upload".`/file/${task.fileName}`));// Merge files
await concatChunks(filePath, chunkFullPaths);
const stat = await fs.promises.stat(filePath);
// Verify file size
if(stat.size ! == task.fileSize) { ctx.body = {error: "Inconsistent file size verification" };
ctx.status = 400;
return;
}
// Finally verify the hash
const fileHash = await getFileMd5(filePath);
if(fileHash ! == task.hash) { ctx.body = {error: "File hash check inconsistent" };
ctx.status = 400;
return;
}
// The task and temporary folder are deleted
upload_map[task.hash] = undefined;
ctx.body = { ok: true };
});
Copy the codeconclusion
Finally, release the complete code
If it is helpful to you, I hope to point a star~