两个minio之间文件同步

代码仅供参考

package com.minio.config;

import io.minio.*;
import io.minio.errors.MinioException;
import io.minio.messages.Item;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

public class MinioSyncUtil {
    private static final Logger logger = LoggerFactory.getLogger(MinioSyncUtil.class);

    private final MinioClient sourceMinioClient;
    private final MinioClient targetMinioClient;
    private final String sourceBucketName;
    private final String targetBucketName;
    private final ExecutorService executorService;

    private static final int MAX_RETRIES = 3;
    private static final long RETRY_INTERVAL_MS = 1000;

    public MinioSyncUtil(String sourceEndpoint, String targetEndpoint,
                         String sourceAccessKey, String sourceSecretKey,
                         String targetAccessKey, String targetSecretKey,
                         String sourceBucketName, String targetBucketName) {
        this.sourceMinioClient = MinioClient.builder().endpoint(sourceEndpoint).credentials(sourceAccessKey, sourceSecretKey).build();
        this.targetMinioClient = MinioClient.builder().endpoint(targetEndpoint).credentials(targetAccessKey, targetSecretKey).build();
        this.sourceBucketName = sourceBucketName;
        this.targetBucketName = targetBucketName;
        this.executorService = Executors.newFixedThreadPool(10); // 使用线程池进行并行操作
    }

    /**
     * 从源桶同步文件到目标桶
     */
    public void syncFiles() {
        try {
            long startTime = System.currentTimeMillis();
            logger.info("开始同步文件，源桶: {}, 目标桶: {}", sourceBucketName, targetBucketName);

            // 确保源桶存在
            if (!sourceMinioClient.bucketExists(BucketExistsArgs.builder().bucket(sourceBucketName).build())) {
                logger.error("源桶 {} 不存在！", sourceBucketName);
                return; // 终止同步
            }

            // 如果目标桶不存在，则创建
            if (!targetMinioClient.bucketExists(BucketExistsArgs.builder().bucket(targetBucketName).build())) {
                targetMinioClient.makeBucket(MakeBucketArgs.builder().bucket(targetBucketName).build());
                logger.info("目标桶 {} 不存在，已创建", targetBucketName);
            }

            // 获取源桶中的所有对象（文件和文件夹）
            Iterable<Result<Item>> sourceObjects = sourceMinioClient.listObjects(
                    ListObjectsArgs.builder().bucket(sourceBucketName).recursive(true).build());

            int filesCopied = 0;

            for (Result<Item> result : sourceObjects) {
                Item item = result.get();
                String objectName = item.objectName();

                // 处理空文件夹（如果 MinIO 里有）
                if (objectName.endsWith("/")) {
                    executorService.submit(() -> createFolderPlaceholder(objectName));
                } else {
                    executorService.submit(() -> copyFileToTargetBucket(objectName));
                    filesCopied++;
                }
            }

            // 等待所有任务执行完毕
            executorService.shutdown();
            while (!executorService.isTerminated()) {
                // 等待线程池中的任务完成
            }

            long endTime = System.currentTimeMillis();
            logger.info("同步完成，共同步 {} 个文件，耗时 {} 毫秒", filesCopied, (endTime - startTime));
        } catch (MinioException | IOException e) {
            logger.error("同步文件失败", e);
        } catch (NoSuchAlgorithmException | InvalidKeyException e) {
            throw new RuntimeException(e);
        }
    }

    /**
     * 检查目标桶中是否已经存在同名文件
     */
    private boolean fileExistsInTargetBucket(String objectName) {
        try {
            // 使用 statObject 检查文件是否存在
            targetMinioClient.statObject(StatObjectArgs.builder().bucket(targetBucketName).object(objectName).build());
            return true; // 如果文件存在
        } catch (Exception e) {
            return false; // 文件不存在
        }
    }

    /**
     * 创建文件夹占位符（如果 MinIO 中存在文件夹）
     */
    private void createFolderPlaceholder(String folderPath) {
        try {
            String placeholder = folderPath + ".keep"; // 创建一个 `.keep` 文件，确保文件夹结构

            targetMinioClient.putObject(
                    PutObjectArgs.builder()
                            .bucket(targetBucketName)
                            .object(placeholder) // 在 MinIO 里创建占位文件
                            .stream(new ByteArrayInputStream(new byte[0]), 0, -1)
                            .build()
            );

            logger.info("已同步文件夹：{}", folderPath);
        } catch (Exception e) {
            logger.error("创建文件夹 {} 失败", folderPath, e);
        }
    }

    /**
     * 将文件从源桶复制到目标桶
     */
    private void copyFileToTargetBucket(String objectName) {
        int attempts = 0;
        while (attempts < MAX_RETRIES) {
            try {
                // 先从源桶下载文件
                InputStream stream = sourceMinioClient.getObject(
                        GetObjectArgs.builder()
                                .bucket(sourceBucketName)
                                .object(objectName)
                                .build()
                );

                // 再上传到目标桶
                targetMinioClient.putObject(
                        PutObjectArgs.builder()
                                .bucket(targetBucketName)
                                .object(objectName)
                                .stream(stream, -1, 5242880) // -1 表示不限制文件大小，5242880 是5MB块大小
                                .build()
                );

                logger.info("文件 {} 已成功从源桶同步到目标桶", objectName);
                return; // 成功，退出重试循环

            } catch (MinioException | IOException e) {
                attempts++;
                logger.error("复制文件 {} 到目标桶失败，尝试次数 {}/{}", objectName, attempts, MAX_RETRIES, e);

                if (attempts < MAX_RETRIES) {
                    try {
                        Thread.sleep(RETRY_INTERVAL_MS); // 暂停后重试
                    } catch (InterruptedException ie) {
                        Thread.currentThread().interrupt(); // 恢复中断状态
                    }
                }
            } catch (NoSuchAlgorithmException e) {
                throw new RuntimeException(e);
            } catch (InvalidKeyException e) {
                throw new RuntimeException(e);
            }
        }
        logger.error("文件 {} 同步失败，已达到最大重试次数", objectName);
    }

    public static void main(String[] args) {
        // MinIO 配置
        String sourceEndpoint = "http://127.0.0.1:9000";
        String sourceAccessKey = "eUYIKb9zbVzBgq4Zjg7I";
        String sourceSecretKey = "KdJZwwpq2QGGvBkvPkqJHReEOfOVpTl6P81Vl50O";
        String sourceBucketName = "blog"; // 源桶名称
        String targetEndpoint = "http://127.0.0.1:10086";
        String targetBucketName = "test"; // 目标桶名称

        String targetAccessKey = "minio";
        String targetSecretKey = "minio123";

        // 创建同步工具实例
        MinioSyncUtil minioSyncUtil = new MinioSyncUtil(
                sourceEndpoint, targetEndpoint,
                sourceAccessKey, sourceSecretKey,
                targetAccessKey, targetSecretKey,
                sourceBucketName, targetBucketName);

        // 执行文件同步
        minioSyncUtil.syncFiles();
    }
}