Commit 48ae0e48 authored by lijingang's avatar lijingang

修复uploadFolderToS3函数支持递归上传子文件夹,新增deleteS3Folder函数删除远程文件夹

parent 2fa4efdb
Pipeline #299 canceled with stages
......@@ -44,9 +44,13 @@ ss3ops upload <bucket> <key> <source>
# Upload a folder
ss3ops upload-folder <bucket> <prefix> <folder>
> ss3ops upload-folder zhipinku audit-4520a6ae9d60a234fe-zi .\audit-4520a6ae9d60a234fe-zi
# Download a folder
ss3ops download-folder <bucket> <prefix> <destination>
ss3ops download-folder zhipinku audit-4520a6ae9d60a234fe-zi ./xxx
# List objects in a bucket
ss3ops list <bucket> [prefix]
```
......
#!/usr/bin/env node
const { s3, downloadFileFromS3, uploadFileToS3, uploadFolderToS3, downloadS3Folder } = require('../lib/index.js');
const { s3, downloadFileFromS3, uploadFileToS3, uploadFolderToS3, downloadS3Folder, deleteS3Folder } = require('../lib/index.js');
const fs = require('fs');
const path = require('path');
......@@ -15,6 +15,7 @@ if (!command) {
console.error(' upload <bucket> <key> <source> Upload a file to S3');
console.error(' upload-folder <bucket> <prefix> <folder> Upload a folder to S3');
console.error(' download-folder <bucket> <prefix> <destination> Download a folder from S3');
console.error(' delete-folder <bucket> <prefix> Delete a folder from S3');
console.error(' list <bucket> [prefix] List objects in bucket');
process.exit(1);
}
......@@ -63,6 +64,16 @@ async function main() {
await downloadS3Folder(dlBucket, dlPrefix, dlDestination);
console.log(`Downloaded folder ${dlPrefix} to ${dlDestination}`);
break;
case 'delete-folder':
if (args.length < 3) {
console.error('Error: missing arguments');
console.error('Usage: ss3ops delete-folder <bucket> <prefix>');
process.exit(1);
}
const [delBucket, delPrefix] = args.slice(1);
const deletedCount = await deleteS3Folder(delBucket, delPrefix);
console.log(`Deleted folder ${delPrefix} from ${delBucket}. Total objects deleted: ${deletedCount}`);
break;
case 'list':
if (args.length < 2) {
console.error('Error: missing bucket');
......
......@@ -7,4 +7,5 @@ module.exports.s3 = s3Client.s3;
module.exports.downloadFileFromS3 = s3Client.downloadFileFromS3;
module.exports.uploadFileToS3 = s3Client.uploadFileToS3;
module.exports.uploadFolderToS3 = s3Client.uploadFolderToS3;
module.exports.downloadS3Folder = s3Client.downloadS3Folder;
\ No newline at end of file
module.exports.downloadS3Folder = s3Client.downloadS3Folder;
module.exports.deleteS3Folder = s3Client.deleteS3Folder;
\ No newline at end of file
......@@ -82,26 +82,97 @@ const uploadFileToS3 = async (bucket, key, body) => {
* @param {string} bucket - The name of the S3 bucket.
* @param {string} prefix - The prefix to add to the S3 keys (optional).
*/
/**
* Recursively uploads all files and subfolders from a local folder to an S3 bucket.
* @param {string} folderPath - The local folder path to upload.
* @param {string} bucket - The name of the S3 bucket.
* @param {string} prefix - The prefix to add to the S3 keys (optional).
*/
const uploadFolderToS3 = async (folderPath, bucket, prefix = "") => {
const { PutObjectCommand } = require("@aws-sdk/client-s3");
const files = fs.readdirSync(folderPath);
try {
// Check if the folder exists
if (!fs.existsSync(folderPath)) {
throw new Error(`Folder does not exist: ${folderPath}`);
}
for (const file of files) {
const filePath = path.join(folderPath, file);
const fileKey = path.join(prefix, file);
// Read all items in the folder
const items = fs.readdirSync(folderPath);
for (const item of items) {
const itemPath = path.join(folderPath, item);
const itemKey = prefix ? path.join(prefix, item) : item;
const stats = fs.statSync(itemPath);
if (stats.isFile()) {
// Upload file using the existing uploadFileToS3 function
const fileContent = fs.readFileSync(itemPath);
await uploadFileToS3(bucket, itemKey, fileContent);
console.log(`Uploaded file: ${itemKey}`);
} else if (stats.isDirectory()) {
// Recursively upload subfolder
console.log(`Processing directory: ${itemKey}/`);
await uploadFolderToS3(itemPath, bucket, itemKey);
}
}
console.log(`Successfully uploaded folder: ${folderPath} to bucket: ${bucket}${prefix ? ` with prefix: ${prefix}` : ''}`);
} catch (error) {
console.error(`Error uploading folder ${folderPath}:`, error);
throw error;
}
};
/**
* Deletes a folder (prefix) and all its contents from S3
* @param {string} bucket - The name of the S3 bucket
* @param {string} prefix - The folder prefix to delete
*/
const deleteS3Folder = async (bucket, prefix) => {
try {
const { ListObjectsV2Command, DeleteObjectsCommand } = require("@aws-sdk/client-s3");
// Ensure prefix ends with '/' for folder deletion
const folderPrefix = prefix.endsWith('/') ? prefix : prefix + '/';
console.log(`Deleting folder: ${folderPrefix} from bucket: ${bucket}`);
let continuationToken;
let deletedCount = 0;
let objectsToDelete = [];
do {
const listParams = {
Bucket: bucket,
Prefix: folderPrefix,
ContinuationToken: continuationToken
};
if (fs.statSync(filePath).isFile()) {
const fileContent = fs.readFileSync(filePath);
await s3.send(
new PutObjectCommand({
const listResponse = await s3.send(new ListObjectsV2Command(listParams));
if (listResponse.Contents && listResponse.Contents.length > 0) {
// Add objects to delete list
objectsToDelete = listResponse.Contents.map(obj => ({ Key: obj.Key }));
// Delete objects in batches (S3 allows up to 1000 objects per DeleteObjects request)
const deleteParams = {
Bucket: bucket,
Key: fileKey,
Body: fileContent,
})
);
console.log(`Uploaded ${fileKey}`);
}
Delete: { Objects: objectsToDelete }
};
const deleteResponse = await s3.send(new DeleteObjectsCommand(deleteParams));
deletedCount += deleteResponse.Deleted ? deleteResponse.Deleted.length : 0;
console.log(`Deleted ${deleteResponse.Deleted ? deleteResponse.Deleted.length : 0} objects from ${folderPrefix}`);
}
continuationToken = listResponse.NextContinuationToken;
} while (continuationToken);
console.log(`Successfully deleted folder ${folderPrefix} from bucket ${bucket}. Total objects deleted: ${deletedCount}`);
return deletedCount;
} catch (error) {
console.error(`Error deleting folder ${prefix} from bucket ${bucket}:`, error);
throw error;
}
};
......@@ -192,7 +263,7 @@ async function downloadS3Folder(bucketName, folderPrefix, localDestination) {
console.log(`\n完成!共下载 ${downloadedCount} 个文件。`);
}
module.exports = { s3, downloadFileFromS3, uploadFileToS3, uploadFolderToS3, downloadS3Folder }; // Export the S3 client, download, and upload functions
module.exports = { s3, downloadFileFromS3, uploadFileToS3, uploadFolderToS3, downloadS3Folder, deleteS3Folder }; // Export the S3 client, download, and upload functions
// Example usage - commented out to prevent auto-execution:
/*
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment