Skip to content

C
client-s3
Commit 64268fea authored by lijingang's avatar lijingang
Browse files
Options

支持通过命令行参数和环境变量覆盖S3配置，新增createS3Client和reconfigureS3Client函数

parent 174c2ef0
Show whitespace changes
Inline Side-by-side
Showing with 170 additions and 48 deletions
.gitignore
View file @ 64268fea
...@@ -8,3 +8,4 @@ npm-debug.log* ...@@ -8,3 +8,4 @@ npm-debug.log*
yarn-debug.log* yarn-debug.log*
yarn-error.log* yarn-error.log*
*.tgz *.tgz
package-lock.json
\ No newline at end of file
README.md
View file @ 64268fea
...@@ -99,6 +99,15 @@ npm publish ...@@ -99,6 +99,15 @@ npm publish
Ensure you have proper permissions and `publishConfig` points to the correct registry. Ensure you have proper permissions and `publishConfig` points to the correct registry.
方案2：使用 npm link
这个方案是在 ../framework 项目中通过 npm link 链接到本地开发的包。
步骤：
1. 在我们的 ss3ops 目录运行 npm link 创建全局链接
2. 在 ../framework 目录运行 npm link @youdatasum/client-s3 链接到本地包
## License ## License
MIT MIT
\ No newline at end of file
bin/ss3ops.js
View file @ 64268fea
#!/usr/bin/env node #!/usr/bin/env node
const { s3, downloadFileFromS3, uploadFileToS3, uploadFolderToS3, downloadS3Folder, deleteS3Folder } = require('../lib/index.js'); const { s3, downloadFileFromS3, uploadFileToS3, uploadFolderToS3, downloadS3Folder, deleteS3Folder, createS3Client } = require('../lib/index.js');
const fs = require('fs'); const fs = require('fs');
const path = require('path'); const path = require('path');
// Parse command line arguments // Parse command line arguments
const args = process.argv.slice(2); const args = process.argv.slice(2);
const command = args[0];
// Global options
const options = {
endpoint: null,
accessKeyId: null,
secretAccessKey: null,
region: null
};
// Parse global options (before command)
let i = 0;
while (i < args.length && args[i].startsWith('--')) {
const arg = args[i];
if (arg === '--endpoint' && i + 1 < args.length) {
options.endpoint = args[i + 1];
i += 2;
} else if (arg === '--access-key' && i + 1 < args.length) {
options.accessKeyId = args[i + 1];
i += 2;
} else if (arg === '--secret-key' && i + 1 < args.length) {
options.secretAccessKey = args[i + 1];
i += 2;
} else if (arg === '--region' && i + 1 < args.length) {
options.region = args[i + 1];
i += 2;
} else if (arg.startsWith('--endpoint=')) {
options.endpoint = arg.substring('--endpoint='.length);
i += 1;
} else if (arg.startsWith('--access-key=')) {
options.accessKeyId = arg.substring('--access-key='.length);
i += 1;
} else if (arg.startsWith('--secret-key=')) {
options.secretAccessKey = arg.substring('--secret-key='.length);
i += 1;
} else if (arg.startsWith('--region=')) {
options.region = arg.substring('--region='.length);
i += 1;
} else {
console.error(`Unknown option: ${arg}`);
process.exit(1);
}
}
// Remaining arguments after options
const remainingArgs = args.slice(i);
const command = remainingArgs[0];
if (!command) { if (!command) {
console.error('Usage: ss3ops <command> [options]'); console.error('Usage: ss3ops [options] <command> [args]');
console.error('Global options:');
console.error(' --endpoint <url> S3 endpoint URL (default: http://192.168.2.253:9000)');
console.error(' --access-key <key> S3 access key ID');
console.error(' --secret-key <key> S3 secret access key');
console.error(' --region <region> S3 region (default: us-east-1)');
console.error('');
console.error('Commands:'); console.error('Commands:');
console.error(' download <bucket> <key> <destination> Download a file from S3'); console.error(' download <bucket> <key> <destination> Download a file from S3');
console.error(' upload <bucket> <key> <source> Upload a file to S3'); console.error(' upload <bucket> <key> <source> Upload a file to S3');
...@@ -22,66 +74,85 @@ if (!command) { ...@@ -22,66 +74,85 @@ if (!command) {
async function main() { async function main() {
try { try {
// Reconfigure S3 client if options are provided
const config = {};
if (options.endpoint) config.endpoint = options.endpoint;
if (options.region) config.region = options.region;
if (options.accessKeyId || options.secretAccessKey) {
config.credentials = {};
if (options.accessKeyId) config.credentials.accessKeyId = options.accessKeyId;
if (options.secretAccessKey) config.credentials.secretAccessKey = options.secretAccessKey;
}
// Only reconfigure if any options were provided
if (Object.keys(config).length > 0) {
console.log('Reconfiguring S3 client with provided options');
reconfigureS3Client(config);
}
// Get command arguments from remainingArgs (skip the command itself)
const cmdArgs = remainingArgs.slice(1);
switch (command) { switch (command) {
case 'download': case 'download':
if (args.length < 4) { if (cmdArgs.length < 3) {
console.error('Error: missing arguments'); console.error('Error: missing arguments');
console.error('Usage: ss3ops download <bucket> <key> <destination>'); console.error('Usage: ss3ops [options] download <bucket> <key> <destination>');
process.exit(1); process.exit(1);
} }
const [bucket, key, destination] = args.slice(1); const [bucket, key, destination] = cmdArgs;
await downloadFileFromS3(bucket, key, destination); await downloadFileFromS3(bucket, key, destination);
console.log(`Downloaded ${key} to ${destination}`); console.log(`Downloaded ${key} to ${destination}`);
break; break;
case 'upload': case 'upload':
if (args.length < 4) { if (cmdArgs.length < 3) {
console.error('Error: missing arguments'); console.error('Error: missing arguments');
console.error('Usage: ss3ops upload <bucket> <key> <source>'); console.error('Usage: ss3ops [options] upload <bucket> <key> <source>');
process.exit(1); process.exit(1);
} }
const [upBucket, upKey, source] = args.slice(1); const [upBucket, upKey, source] = cmdArgs;
const body = fs.readFileSync(source); const body = fs.readFileSync(source);
await uploadFileToS3(upBucket, upKey, body); await uploadFileToS3(upBucket, upKey, body);
console.log(`Uploaded ${source} to ${upBucket}/${upKey}`); console.log(`Uploaded ${source} to ${upBucket}/${upKey}`);
break; break;
case 'upload-folder': case 'upload-folder':
if (args.length < 4) { if (cmdArgs.length < 3) {
console.error('Error: missing arguments'); console.error('Error: missing arguments');
console.error('Usage: ss3ops upload-folder <bucket> <prefix> <folder>'); console.error('Usage: ss3ops [options] upload-folder <bucket> <prefix> <folder>');
process.exit(1); process.exit(1);
} }
const [folderBucket, prefix, folderPath] = args.slice(1); const [folderBucket, prefix, folderPath] = cmdArgs;
await uploadFolderToS3(folderPath, folderBucket, prefix); await uploadFolderToS3(folderPath, folderBucket, prefix);
console.log(`Uploaded folder ${folderPath} to ${folderBucket}/${prefix}`); console.log(`Uploaded folder ${folderPath} to ${folderBucket}/${prefix}`);
break; break;
case 'download-folder': case 'download-folder':
if (args.length < 4) { if (cmdArgs.length < 3) {
console.error('Error: missing arguments'); console.error('Error: missing arguments');
console.error('Usage: ss3ops download-folder <bucket> <prefix> <destination>'); console.error('Usage: ss3ops [options] download-folder <bucket> <prefix> <destination>');
process.exit(1); process.exit(1);
} }
const [dlBucket, dlPrefix, dlDestination] = args.slice(1); const [dlBucket, dlPrefix, dlDestination] = cmdArgs;
await downloadS3Folder(dlBucket, dlPrefix, dlDestination); await downloadS3Folder(dlBucket, dlPrefix, dlDestination);
console.log(`Downloaded folder ${dlPrefix} to ${dlDestination}`); console.log(`Downloaded folder ${dlPrefix} to ${dlDestination}`);
break; break;
case 'delete-folder': case 'delete-folder':
if (args.length < 3) { if (cmdArgs.length < 2) {
console.error('Error: missing arguments'); console.error('Error: missing arguments');
console.error('Usage: ss3ops delete-folder <bucket> <prefix>'); console.error('Usage: ss3ops [options] delete-folder <bucket> <prefix>');
process.exit(1); process.exit(1);
} }
const [delBucket, delPrefix] = args.slice(1); const [delBucket, delPrefix] = cmdArgs;
const deletedCount = await deleteS3Folder(delBucket, delPrefix); const deletedCount = await deleteS3Folder(delBucket, delPrefix);
console.log(`Deleted folder ${delPrefix} from ${delBucket}. Total objects deleted: ${deletedCount}`); console.log(`Deleted folder ${delPrefix} from ${delBucket}. Total objects deleted: ${deletedCount}`);
break; break;
case 'list': case 'list':
if (args.length < 2) { if (cmdArgs.length < 1) {
console.error('Error: missing bucket'); console.error('Error: missing bucket');
console.error('Usage: ss3ops list <bucket> [prefix]'); console.error('Usage: ss3ops [options] list <bucket> [prefix]');
process.exit(1); process.exit(1);
} }
const listBucket = args[1]; const listBucket = cmdArgs[0];
const listPrefix = args[2] || ''; const listPrefix = cmdArgs[1] || '';
const { ListObjectsV2Command } = require('@aws-sdk/client-s3'); const { ListObjectsV2Command } = require('@aws-sdk/client-s3');
const response = await s3.send(new ListObjectsV2Command({ Bucket: listBucket, Prefix: listPrefix })); const response = await s3.send(new ListObjectsV2Command({ Bucket: listBucket, Prefix: listPrefix }));
const contents = response.Contents || []; const contents = response.Contents || [];
......
lib/index.js
View file @ 64268fea
...@@ -4,6 +4,8 @@ module.exports = s3Client; ...@@ -4,6 +4,8 @@ module.exports = s3Client;
// Also export individual functions for convenience // Also export individual functions for convenience
module.exports.s3 = s3Client.s3; module.exports.s3 = s3Client.s3;
module.exports.createS3Client = s3Client.createS3Client;
module.exports.reconfigureS3Client = s3Client.reconfigureS3Client;
module.exports.downloadFileFromS3 = s3Client.downloadFileFromS3; module.exports.downloadFileFromS3 = s3Client.downloadFileFromS3;
module.exports.uploadFileToS3 = s3Client.uploadFileToS3; module.exports.uploadFileToS3 = s3Client.uploadFileToS3;
module.exports.uploadFolderToS3 = s3Client.uploadFolderToS3; module.exports.uploadFolderToS3 = s3Client.uploadFolderToS3;
......
lib/s3-client.js
View file @ 64268fea
...@@ -11,18 +11,19 @@ const { pipeline } = require("stream/promises"); // Node.js 15+ 支持，用于 ...@@ -11,18 +11,19 @@ const { pipeline } = require("stream/promises"); // Node.js 15+ 支持，用于
// import { readFileSync } from "fs"; // import { readFileSync } from "fs";
// 动态创建S3客户端 // 动态创建S3客户端
let s3; const createS3Client = (config = {}) => {
try { try {
// 动态引入第三方库 // 动态引入第三方库
({ S3Client } = require("@aws-sdk/client-s3")); const { S3Client } = require("@aws-sdk/client-s3");
({ NodeHttpHandler } = require("@smithy/node-http-handler")); const { NodeHttpHandler } = require("@smithy/node-http-handler");
s3 = new S3Client({ // 默认配置
endpoint: "http://127.0.0.1:9000", // RustFS endpoint const defaultConfig = {
region: "us-east-1", // Any value is accepted endpoint: process.env.S3_ENDPOINT || "http://192.168.2.253:9000",
region: process.env.S3_REGION || "us-east-1",
credentials: { credentials: {
accessKeyId: "ERfTPtr9H5VdyasN0hCn", accessKeyId: process.env.S3_ACCESS_KEY_ID || "ERfTPtr9H5VdyasN0hCn",
secretAccessKey: "lOA7cJSzTY2nEgVeCR5WQhIf1tN9HvDjGmrpoP0w", secretAccessKey: process.env.S3_SECRET_ACCESS_KEY || "lOA7cJSzTY2nEgVeCR5WQhIf1tN9HvDjGmrpoP0w",
}, },
forcePathStyle: true, // Must be enabled for RustFS compatibility forcePathStyle: true, // Must be enabled for RustFS compatibility
maxAttempts: 3, maxAttempts: 3,
...@@ -30,11 +31,27 @@ try { ...@@ -30,11 +31,27 @@ try {
connectionTimeout: 10000, // Increased from 3000 to 10000 ms connectionTimeout: 10000, // Increased from 3000 to 10000 ms
socketTimeout: 30000, // Increased from 5000 to 30000 ms socketTimeout: 30000, // Increased from 5000 to 30000 ms
}), }),
}); };
} catch (error) {
// 合并配置
const finalConfig = {
...defaultConfig,
...config,
credentials: {
...defaultConfig.credentials,
...(config.credentials || {})
}
};
return new S3Client(finalConfig);
} catch (error) {
console.error("Failed to initialize S3 client:", error.message); console.error("Failed to initialize S3 client:", error.message);
s3 = null; return null;
} }
};
// 默认客户端（向后兼容）
let s3 = createS3Client();
/** /**
* Downloads a file from the specified S3 bucket and key, and writes it to the local file system. * Downloads a file from the specified S3 bucket and key, and writes it to the local file system.
...@@ -281,7 +298,29 @@ async function downloadS3Folder(bucketName, folderPrefix, localDestination) { ...@@ -281,7 +298,29 @@ async function downloadS3Folder(bucketName, folderPrefix, localDestination) {
console.log(`\n完成！共下载 ${downloadedCount} 个文件。`); console.log(`\n完成！共下载 ${downloadedCount} 个文件。`);
} }
module.exports = { s3, downloadFileFromS3, uploadFileToS3, uploadFolderToS3, downloadS3Folder, deleteS3Folder }; // Export the S3 client, download, and upload functions /**
* Reconfigure the global S3 client with new settings
* @param {Object} config - Configuration object
* @returns {Object} The new S3 client
*/
const reconfigureS3Client = (config = {}) => {
const newClient = createS3Client(config);
if (newClient) {
s3 = newClient;
}
return s3;
};
module.exports = {
s3,
createS3Client,
reconfigureS3Client,
downloadFileFromS3,
uploadFileToS3,
uploadFolderToS3,
downloadS3Folder,
deleteS3Folder
}; // Export the S3 client, download, and upload functions
// Example usage - commented out to prevent auto-execution: // Example usage - commented out to prevent auto-execution:
/* /*
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment