Commit 174c2ef0 authored by lijingang's avatar lijingang

增加超时时间并优化deleteS3Folder分批处理,避免连接超时

parent 48ae0e48
Pipeline #300 failed with stages
......@@ -25,9 +25,10 @@ try {
secretAccessKey: "lOA7cJSzTY2nEgVeCR5WQhIf1tN9HvDjGmrpoP0w",
},
forcePathStyle: true, // Must be enabled for RustFS compatibility
maxAttempts: 3,
requestHandler: new NodeHttpHandler({
connectionTimeout: 3000,
socketTimeout: 5000,
connectionTimeout: 10000, // Increased from 3000 to 10000 ms
socketTimeout: 30000, // Increased from 5000 to 30000 ms
}),
});
} catch (error) {
......@@ -151,18 +152,35 @@ const deleteS3Folder = async (bucket, prefix) => {
const listResponse = await s3.send(new ListObjectsV2Command(listParams));
if (listResponse.Contents && listResponse.Contents.length > 0) {
// Add objects to delete list
objectsToDelete = listResponse.Contents.map(obj => ({ Key: obj.Key }));
// Convert objects to delete format
const allObjects = listResponse.Contents.map(obj => ({ Key: obj.Key }));
// Delete objects in batches (S3 allows up to 1000 objects per DeleteObjects request)
// Process in batches of max 1000 objects per request (S3 limit)
const batchSize = 1000;
for (let i = 0; i < allObjects.length; i += batchSize) {
const batch = allObjects.slice(i, i + batchSize);
try {
const deleteParams = {
Bucket: bucket,
Delete: { Objects: objectsToDelete }
Delete: { Objects: batch }
};
const deleteResponse = await s3.send(new DeleteObjectsCommand(deleteParams));
deletedCount += deleteResponse.Deleted ? deleteResponse.Deleted.length : 0;
console.log(`Deleted ${deleteResponse.Deleted ? deleteResponse.Deleted.length : 0} objects from ${folderPrefix}`);
const deletedInBatch = deleteResponse.Deleted ? deleteResponse.Deleted.length : 0;
deletedCount += deletedInBatch;
console.log(`Deleted ${deletedInBatch} objects from ${folderPrefix} (batch ${Math.floor(i/batchSize) + 1})`);
// Small delay between batches to avoid overwhelming the server
if (i + batchSize < allObjects.length) {
await new Promise(resolve => setTimeout(resolve, 100));
}
} catch (batchError) {
console.error(`Error deleting batch ${Math.floor(i/batchSize) + 1}:`, batchError.message);
// Continue with next batch instead of failing completely
console.log('Continuing with remaining batches...');
}
}
}
continuationToken = listResponse.NextContinuationToken;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment