s3 api接口的调用-白红宇

s3 api接口的调用

阅读量：5128 次

发布时间：2019-06-13

本文共 16369 字，大约阅读时间需要 54 分钟。

最近公司使用s3做文件存储服务器，因此在程序中需要调用s3的api，目前程序中使用了python和java版本的s3的api，简单做下记录，方便以后使用。

一、s3 api使用python版

1.安装boto3的sdk

pip install boto3==1.4.7

2.创建s3的api客户端

import boto3

from botocore.client import Config

import os

import sys

import threading

#创建s3链接，如果s3服务器是第四代，则需要设置signature_version='s3v4'

s3_client = boto3.client('s3', endpoint_url='s3服务器地址',

aws_access_key_id='s3服务器的access_key_id',

aws_secret_access_key='s3服务器的secret_access_key',

region_name='s3服务器的时区，这里可以填写cn-north-1',

config=Config(signature_version='s3'))

3.获取s3中bucket列表

bucket_list = s3_client.list_buckets()

print bucket_list

4.创建bucket

bucket = s3_client.create_bucket(Bucket='bucket的名称')

print bucket

5.获取bucket信息

bucket_info = s3_client.head_bucket(Bucket='bucket的名称')

print bucket_info

6.删除bucket

#bucket_delete = s3_client.delete_bucket(Bucket='bucket的名称')

7.上传文件到s3服务器

# s3_client.upload_file("上传的源文件地址", "bucket名称", "上传文件在s3上对应的key名称", ExtraArgs={'ACL': 'public-read'})

或者

s3_client.upload_file("上传的源文件地址", "bucket名称", "上传文件在s3上对应的key名称",

　　　　　　ExtraArgs={'ACL': 'public-read'},

　　　　　　Callback=UploadProgressPercentage("上传的源文件地址"))

Callback属性对应的类方法如下，该类方法在控制台中打印了上传文件的进度

class UploadProgressPercentage(object):

def __init__(self, filename):

self._filename = filename

self._size = float(os.path.getsize(filename))

self._seen_so_far = 0

self._lock = threading.Lock()

def __call__(self, bytes_amount):

# To simplify we'll assume this is hooked up

# to a single filename.

with self._lock:

self._seen_so_far += bytes_amount

percentage = (self._seen_so_far / self._size) * 100

sys.stdout.write(

"\r%s %s / %s (%.2f%%)" % (

self._filename, self._seen_so_far, self._size,

percentage))

sys.stdout.flush()

8.获取bucket下文件列表

object_list = s3_client.list_objects(Bucket='bucket名称')

print object_list

9.查看bucket下的某个文件信息

object_info = s3_client.get_object(Bucket='bucket名称', Key='文件对应的key名称')

print object_info

10.删除文件

object_delete = s3_client.delete_object(Bucket='bucket名称', Key='文件对应的key名称')

11.下载文件

s3_client.download_file("bucket名称", "文件对应的key名称", "文件下载到的地址")

或者

s3client.download_file("bucket名称", "文件对应的key名称", "文件下载到的地址",Callback=DownloadProgressPercentage("文件下载到的地址"))

Callback属性对应的类方法如下，该类方法在控制台中打印了下载文件的进度

class _DownloadProgressPercentage(object):

def __init__(self, filename):

self._filename = filename

self._seen_so_far = 0

self._lock = threading.Lock()

def __call__(self, bytes_amount):

# To simplify we'll assume this is hooked up

# to a single filename.

with self._lock:

self._seen_so_far += bytes_amount

sys.stdout.write(

"\r%s --> %s bytes transferred" % (

self._filename, self._seen_so_far))

sys.stdout.flush()

参考文档

boto3 api官方文档：http://boto3.readthedocs.io/en/latest/guide/quickstart.html

1.s3的client的api

2.s3的client的api上传下载示例

二、3 api使用java版

1.在pom.xml中增加依赖包

　　<groupId>com.amazonaws</groupId>

</dependency>

2.java中使用s3的api的demo

package org.jenkinsci.plugins.s3_step;

import java.io.BufferedInputStream;

import java.io.BufferedOutputStream;

import java.io.ByteArrayInputStream;

import java.io.File;

import java.io.FileOutputStream;

import java.io.IOException;

import java.io.InputStream;

import java.io.OutputStream;

import java.net.URL;

import java.security.SecureRandom;

import java.util.ArrayList;

import java.util.List;

import com.amazonaws.ClientConfiguration;

import com.amazonaws.auth.AWSCredentials;

import com.amazonaws.auth.BasicAWSCredentials;

import com.amazonaws.regions.Region;

import com.amazonaws.regions.Regions;

import com.amazonaws.services.s3.AmazonS3;

import com.amazonaws.services.s3.AmazonS3Client;

import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;

import com.amazonaws.services.s3.model.Bucket;

import com.amazonaws.services.s3.model.BucketPolicy;

import com.amazonaws.services.s3.model.CannedAccessControlList;

import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;

import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;

import com.amazonaws.services.s3.model.CopyObjectResult;

import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;

import com.amazonaws.services.s3.model.GetBucketPolicyRequest;

import com.amazonaws.services.s3.model.GetObjectRequest;

import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;

import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;

import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;

import com.amazonaws.services.s3.model.ListObjectsRequest;

import com.amazonaws.services.s3.model.ListPartsRequest;

import com.amazonaws.services.s3.model.MultipartUpload;

import com.amazonaws.services.s3.model.MultipartUploadListing;

import com.amazonaws.services.s3.model.ObjectListing;

import com.amazonaws.services.s3.model.ObjectMetadata;

import com.amazonaws.services.s3.model.PartETag;

import com.amazonaws.services.s3.model.PartListing;

import com.amazonaws.services.s3.model.PartSummary;

import com.amazonaws.services.s3.model.PutObjectRequest;

import com.amazonaws.services.s3.model.PutObjectResult;

import com.amazonaws.services.s3.model.S3Object;

import com.amazonaws.services.s3.model.S3ObjectInputStream;

import com.amazonaws.services.s3.model.S3ObjectSummary;

import com.amazonaws.services.s3.model.UploadPartRequest;

import com.amazonaws.services.s3.model.UploadPartResult;

public class Demo {

AWSCredentials credentials = null;

public AmazonS3 getS3Client(String access_key, String secret_key,

String endpoint) {

ClientConfiguration conf = null;

if (conf == null) {

conf = new ClientConfiguration();

credentials = new BasicAWSCredentials(access_key, secret_key);

}

AmazonS3 s3 = new AmazonS3Client(credentials, conf);

s3.setRegion(Region.getRegion(Regions.CN_NORTH_1));

if (endpoint != null)

s3.setEndpoint(endpoint);

return s3;

}

// ============================ bucket ================================

public void testCreateBucket(AmazonS3 s3Client, String bucketName) {

System.out.println("======================创建 Bucket==========================");

Bucket bucket = s3Client.createBucket(bucketName);

System.out.println(bucket);

System.out.println("创建 Bucket : " + bucket.getName());

System.out.println("======================创建 Bucket==========================");

}

public void testDeleteBucket(AmazonS3 s3Client, String bucketName) {

System.out.println("======================删除 Bucket==========================");

System.out.println("删除 Bucket : " + bucketName);

s3Client.deleteBucket(bucketName);

System.out.println("======================删除 Bucket==========================");

}

public void testDoesBucketExist(AmazonS3 s3Client, String bucketName) {

System.out.println("======================Bucket 是否存在==========================");

boolean bucketExist = s3Client.doesBucketExist(bucketName);

System.out.println(String.format("%s : %s", bucketName, bucketExist));

System.out.println("======================Bucket 是否存在==========================");

}

public void testListBuckets(AmazonS3 s3Client) {

System.out.println("======================枚举 Buckets==========================");

List<Bucket> buckets = s3Client.listBuckets();

for (Bucket bucket : buckets) {

System.out.println(String.format("%s - %s - %s", bucket.getName(), bucket.getOwner(), bucket.getCreationDate()));

}

System.out.println("======================枚举 Buckets==========================");

}

public void testListObjects(AmazonS3 s3Client, String bucketName) {

System.out.println("======================枚举 Objects==========================");

List<String> objectKeys = new ArrayList<String>();

String prefix = "/s3demo/test/";

// for (int i = 0; i < 5; i++) {

// objectKeys.add(prefix + i);

// }

// for (String objectKey : objectKeys) {

// testPutObject(s3Client, bucketName, objectKey);

// }

ObjectListing objects = s3Client.listObjects(bucketName);

List<S3ObjectSummary> s3Objects = objects.getObjectSummaries();

for (S3ObjectSummary s3os : s3Objects) {

System.out.println(String.format("%s - %s - %s", s3os.getKey(), s3os.getETag(), s3os.getSize()));

}

// prefix

ListObjectsRequest listObjReq = new ListObjectsRequest();

listObjReq.setBucketName(bucketName);

listObjReq.setDelimiter("/");

listObjReq.setPrefix(prefix);

ObjectListing preobjects = s3Client.listObjects(listObjReq);

List<String> preobjs = preobjects.getCommonPrefixes();

for (String preobj : preobjs) {

System.out.println(preobj + "\n");

}

System.out.println("======================枚举 Objects==========================");

}

// ============================ bucket ================================

// ============================ object ================================

public void testPutObject(AmazonS3 s3Client, String bucketName, String objectKey,String file_source_address) {

System.out.println("======================上传 object==========================");

s3Client.putObject(new PutObjectRequest(bucketName, objectKey, new File(file_source_address)).withCannedAcl(CannedAccessControlList.PublicRead));

System.out.println("======================上传 object==========================");

}

public void testGetObject(AmazonS3 s3Client, String bucketName,String objectKey) throws Exception {

System.out.println("======================下载 object==========================");

try {

S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucketName,objectKey));

//下载文件到指定地方

InputStream reader = new BufferedInputStream(s3Object.getObjectContent());

File file = new File("localFilename");

FileOutputStream fos = new FileOutputStream(file);

OutputStream writer = new BufferedOutputStream(fos);

int read = -1;

while ( ( read = reader.read() ) != -1 ) {

writer.write(read);

}

writer.flush();

writer.close();

reader.close();

//获取一个request，生成一个下载的url地址，可以在浏览器上下载

GeneratePresignedUrlRequest urlRequest = new GeneratePresignedUrlRequest(bucketName, objectKey);

//生成公用的url

URL url = s3Client.generatePresignedUrl(urlRequest);

System.out.println("=========URL=================" + url + "============URL=============");

if (url == null) {

throw new Exception("can't get s3 file url!");

}

System.out.println("URL=" + url.toString());

} catch (Exception e) {

e.printStackTrace();

}

System.out.println("======================下载 object==========================");

}

public void testDeleteObject(AmazonS3 s3Client, String bucketName,String objectKey) {

System.out.println("======================删除 object==========================");

s3Client.deleteObject(bucketName, objectKey);

System.out.println("======================删除 object==========================");

}

public void testCopyObject(AmazonS3 s3Client, String srcBucketName, String srcObjectKey, String dstBucketName, String dstObjectKey) {

System.out.println("======================复制 object==========================");

CopyObjectResult cpres = s3Client.copyObject(srcBucketName, srcObjectKey, dstBucketName, dstObjectKey);

String text = String.format("test copy objcet %s from %s to %s.%s", srcObjectKey, srcBucketName, dstBucketName, dstObjectKey);

System.out.println(text);

System.out.println("object etag : " + cpres.getETag());

System.out.println("======================复制 objects==========================");

}

public void testMultiUpload(AmazonS3 s3Client, String bucketName, String objectKey) {

System.out.println("======================分块上传==========================");

System.out.println("初始化分块上传");

InitiateMultipartUploadRequest initMultiUp = new InitiateMultipartUploadRequest(bucketName, objectKey);

InitiateMultipartUploadResult initMultiUpRes = s3Client.initiateMultipartUpload(initMultiUp);

String uploadId = initMultiUpRes.getUploadId();

System.out.println("分块上传 MultiUpload Id : " + uploadId);

System.out.println("上传块");

List<PartETag> partETags = new ArrayList<PartETag>();

for (int i = 1; i <= 3; i++) {

UploadPartRequest uploadPartReq = new UploadPartRequest();

uploadPartReq.setBucketName(bucketName);

uploadPartReq.setKey(objectKey);

uploadPartReq.setUploadId(uploadId);

uploadPartReq.setPartNumber(i);

byte[] part = new byte[1024 * 1024 * 6]; // 分块上传一个块必须大于等于5M

if (i == 3) {

uploadPartReq.setLastPart(true);

part = new byte[1024 * 1024 * 3]; // 最后一个块可以小于5M

}

SecureRandom random = new SecureRandom();

random.nextBytes(part);

System.out.println("块大小 = " + part.length);

uploadPartReq.setPartSize(part.length);

InputStream inputStream = new ByteArrayInputStream(part);

uploadPartReq.setInputStream(inputStream);

UploadPartResult uploadPartRes = s3Client.uploadPart(uploadPartReq);

PartETag partETag = uploadPartRes.getPartETag();

partETags.add(partETag);

System.out.println(String.format("上传块 %s - %s", partETag.getPartNumber(), partETag.getETag()));

}

CompleteMultipartUploadRequest comMultiReq = new CompleteMultipartUploadRequest(

bucketName, objectKey, uploadId, partETags);

CompleteMultipartUploadResult comMultires = s3Client.completeMultipartUpload(comMultiReq);

System.out.println("完成上传 etag : " + comMultires.getETag());

System.out.println("测试终止上传");

String objectKey1 = "test_abort_multi_upload";

InitiateMultipartUploadRequest initMultiUp1 = new InitiateMultipartUploadRequest(

bucketName, objectKey1);

InitiateMultipartUploadResult initMultiUpRes1 = s3Client

.initiateMultipartUpload(initMultiUp1);

String uploadId1 = initMultiUpRes1.getUploadId();

System.out.println("终止上传 upload id : " + uploadId1);

AbortMultipartUploadRequest abortMultiUp = new AbortMultipartUploadRequest(bucketName, objectKey1, uploadId1);

s3Client.abortMultipartUpload(abortMultiUp);

System.out.println("======================分块上传==========================");

}

public void testListUploadIds(AmazonS3 s3Client, String bucketName) {

System.out.println("======================枚举分块上传uploadId==========================");

String keyPrefix = "/demo/listupids";

for (int i = 1; i < 5; i++) {

String objectKey = keyPrefix + i;

InitiateMultipartUploadRequest initMultiUp = new InitiateMultipartUploadRequest(bucketName, objectKey);

s3Client.initiateMultipartUpload(initMultiUp);

}

ListMultipartUploadsRequest listMultiUpReq = new ListMultipartUploadsRequest(bucketName);

MultipartUploadListing multiUploads = s3Client.listMultipartUploads(listMultiUpReq);

List<MultipartUpload> uploads = multiUploads.getMultipartUploads();

for (MultipartUpload up : uploads) {

System.out.println(String.format("uploadId : %s - key : %s", up.getUploadId(), up.getKey()));

}

System.out.println("======================枚举分块上传uploadId==========================");

}

public void testListMultiParts(AmazonS3 s3Client, String bucketName, String objectkey, String uploadId) {

System.out.println("======================枚举分块上传part==========================");

ListPartsRequest listPartsReq = new ListPartsRequest(bucketName, objectkey, uploadId);

PartListing parts = s3Client.listParts(listPartsReq);

List<PartSummary> ps = parts.getParts();

for (PartSummary p : ps) {

System.out.println(String.format("partNum : %s - partEtag : %s", p.getPartNumber(), p.getETag()));

}

System.out.println("======================枚举分块上传part==========================");

}

public void testIsFileExists(AmazonS3 s3Client, String s3Path,String keyName){

System.out.println("======================判断文件是否已经存在==========================");

try{

boolean flag = false;

S3Object s3Object = s3Client.getObject(s3Path, keyName);

if(s3Object!=null){

flag = true;

}

if(flag){

System.out.println("File exists，filename:"+keyName);

}

}catch(Exception e){

e.printStackTrace();

}

// ============================ object ================================

public static void main(String[] args) throws IOException {

String access_key = "s3的access key";

String secret_key = "s3的secret key";

String endpoint = "s3的地址";

Demo d = new Demo();

AmazonS3 s3Client = d.getS3Client(access_key, secret_key, endpoint);

String bucketName = "demo";

String bucketName1 = "demo1";

//创建bucket

// d.testCreateBucket(s3Client, bucketName);

//查询bucket列表

// d.testListBuckets(s3Client);

//判断bucket是否存在

// d.testDoesBucketExist(s3Client, bucketName);

//删除bucket

// d.testDeleteBucket(s3Client, bucketName2);

//上传文件

// String objectKey = "var/lib/pom.xml";

// String objectKey1 = "var/lib/pom.xml";

// String file_source_address = "E:/demo/test1/pom.xml";

// d.testPutObject(s3Client, bucketName, objectKey,file_source_address);

//下载文件

// d.testGetObject(s3Client, bucketName, objectKey);

// //复制文件

// d.testCopyObject(s3Client, bucketName, objectKey, bucketName1, objectKey1);

//bucket下面文件列表

// d.testListObjects(s3Client, bucketName);

//判断文件是否存在

// d.testIsFileExists(s3Client, bucketName, objectKey);

}

转载于:https://www.cnblogs.com/web424/p/7779059.html

你可能感兴趣的文章

Eclipse Python插件 PyDev

查看>>

selenium+python3模拟键盘实现粘贴、复制

SDWebImage源码解读之SDWebImageDownloaderOperation

Iroha and a Grid AtCoder - 1974（思维水题）

[LeetCode] Palindrome Number

查看>>

我对于脚本程序的理解——百度轻应用有感

查看>>