S3-客户端测试

根据S3的Amazon S3 API Reference 和Amazon S3 Developer Guide，使用JAVA编写的putObject和getObject测试，同时测试了：MD5特性，ETAG,RANGE。

 

package amazons3;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.Closeable;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.security.MessageDigest;
import java.util.Date;
import java.util.Map;

import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;

import org.apache.commons.codec.binary.Base64;
import org.apache.commons.httpclient.util.DateUtil;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;

import sun.misc.BASE64Encoder;

public class S3ClientSimpleTest {

    private static final Logger logger = Logger.getLogger(S3ClientSimpleTest.class);

    static String accessKey = "your accessKey";
    static String secretKey = "your secretKey";
    static String bucket = "your bucket";

    public static void main(String[] args) throws Exception {
        File localFile = new File("D:/temp/s3testfile.txt");
        putObject(localFile);
        getObject("s3testfile.txt", "d:/temp/download", null, null, null);
    }

    public static void putObject(File localFile) throws Exception {
        HttpURLConnection conn = null;
        BufferedInputStream in = null;
        BufferedOutputStream out = null;
        try {
            URL url = new URL("http://" + bucket + ".s3.amazonaws.com/" + localFile.getName());
            conn = (HttpURLConnection) url.openConnection();
            conn.setRequestMethod("PUT");
            conn.setDoOutput(true);
            conn.setDoInput(true);

            String contentMD5 = md5file(localFile);
            logger.info("ContentMD5: " + contentMD5);
            String contentType = "application/xml";
            Date date = new Date();
            String dateString = DateUtil.formatDate(date, DateUtil.PATTERN_RFC1036);
            String sign = sign("PUT", contentMD5, contentType, dateString, "/" + bucket + "/" + localFile.getName(), null);
            conn.setRequestProperty("Date", dateString);
            conn.setRequestProperty("Authorization", sign);
            conn.setRequestProperty("Content-Type", contentType);
            conn.setRequestProperty("Content-MD5", contentMD5);

            out = new BufferedOutputStream(conn.getOutputStream());
            in = new BufferedInputStream(new FileInputStream(localFile));

            byte[] buffer = new byte[1024];
            int p = 0;
            while ((p = in.read(buffer)) != -1) {
                out.write(buffer, 0, p);
                out.flush();
            }

            int status = conn.getResponseCode();
            logger.info("http status: " + status);
            logger.info("after:\n" + conn.getHeaderFields());

        } catch (Exception e) {
            e.printStackTrace();
            throw new RuntimeException(e);
        } finally {
            close(in);
            close(out);
        }
    }

    public static File getObject(String objectName, String rootPath, Long start, Long end, String etag) {

        HttpURLConnection conn = null;
        BufferedInputStream in = null;
        BufferedOutputStream out = null;
        try {
            URL url = new URL("http://" + bucket + ".s3.amazonaws.com/" + objectName);
            conn = (HttpURLConnection) url.openConnection();
            conn.setRequestMethod("GET");
            conn.setDoOutput(true);

            String contentType = "application/xml";
            Date date = new Date();
            String dateString = DateUtil.formatDate(date, DateUtil.PATTERN_RFC1036);
            String sign = sign("GET", "", contentType, dateString, "/" + bucket + "/" + objectName, null);
            conn.setRequestProperty("Date", dateString);
            conn.setRequestProperty("Authorization", sign);
            conn.setRequestProperty("Content-Type", contentType);

            // Range 特性
            if (start != null && end != null) {
                conn.setRequestProperty("Range", "bytes=" + start + "-" + end);
            }

            // Etag 特性
            if (StringUtils.isNotBlank(etag)) {
                conn.setRequestProperty("If-None-Match", etag);
            }

            int status = conn.getResponseCode();
            logger.info("http status: " + status);
            if (status == 304) {
                // ETAG未变化，文件未变化，服务器返回空body
                logger.info("after:\n" + conn.getHeaderFields());
                return null;
            }

            in = new BufferedInputStream(conn.getInputStream());
            File localFile = new File(rootPath + "/" + objectName);
            if (!localFile.getParentFile().exists()) {
                localFile.getParentFile().mkdirs();
            }
            out = new BufferedOutputStream(new FileOutputStream(localFile, false));

            byte[] buffer = new byte[1024];
            int p = 0;
            while ((p = in.read(buffer)) != -1) {
                out.write(buffer, 0, p);
                out.flush();
            }
            logger.info("after:\n" + conn.getHeaderFields());
            return localFile;
        } catch (Exception e) {
            e.printStackTrace();
            throw new RuntimeException(e);
        } finally {
            close(in);
            close(out);
        }

    }

    private static void close(Closeable c) {
        try {
            if (c != null) {
                c.close();
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    /**
     * MD5文件
     *
     * @param file
     * @return
     * @throws Exception
     */
    public static String md5file(File file) throws Exception {
        MessageDigest messageDigest = MessageDigest.getInstance("MD5");
        BufferedInputStream in = new BufferedInputStream(new FileInputStream(file));
        byte[] buf = new byte[1024 * 100];
        int p = 0;
        while ((p = in.read(buf)) != -1) {
            messageDigest.update(buf, 0, p);
        }
        in.close();
        byte[] digest = messageDigest.digest();

        BASE64Encoder encoder = new BASE64Encoder();
        return encoder.encode(digest);
    }

    /**
     * 计算签名
     *
     * @param httpVerb
     * @param contentMD5
     * @param contentType
     * @param date
     * @param resource
     * @param metas
     * @return
     */
    public static String sign(String httpVerb, String contentMD5, String contentType, String date, String resource, Map metas) {

        String stringToSign = httpVerb + "\n" + StringUtils.trimToEmpty(contentMD5) + "\n" + StringUtils.trimToEmpty(contentType) + "\n" + date + "\n";
        if (metas != null) {
            for (Map.Entry entity : metas.entrySet()) {
                stringToSign += StringUtils.trimToEmpty(entity.getKey()) + ":" + StringUtils.trimToEmpty(entity.getValue()) + "\n";
            }
        }
        stringToSign += resource;
        try {
            Mac mac = Mac.getInstance("HmacSHA1");
            byte[] keyBytes = secretKey.getBytes("UTF8");
            SecretKeySpec signingKey = new SecretKeySpec(keyBytes, "HmacSHA1");
            mac.init(signingKey);
            byte[] signBytes = mac.doFinal(stringToSign.getBytes("UTF8"));
            String signature = encodeBase64(signBytes);
            return "AWS" + " " + accessKey + ":" + signature;
        } catch (Exception e) {
            throw new RuntimeException("MAC CALC FAILED.");
        }

    }

    private static String encodeBase64(byte[] data) {
        String base64 = new String(Base64.encodeBase64(data));
        if (base64.endsWith("\r\n"))
            base64 = base64.substring(0, base64.length() - 2);
        if (base64.endsWith("\n"))
            base64 = base64.substring(0, base64.length() - 1);

        return base64;
    }

}