使用boto3上传大文件至cloudflare R2

发布于 2022-12-30  853 次阅读


import os
from tqdm import tqdm
import botocore
import boto3
from boto3.session import Session
import threading
import sys
import warnings
warnings.filterwarnings("ignore")
class ProgressPercentage:
    def __init__(self,filename):
        self.filename = filename
        self.size = float(os.path.getsize(filename))
        self._seen_so_far = 0
        self._lock = threading.Lock()

    def __call__(self, bytes_amount):
        with self._lock:
            self._seen_so_far += bytes_amount
            percentage = (self._seen_so_far / self.size) * 100
            print(
                "\r%s %s / %s  (%.2f%%)" % (
                    self.filename, self._seen_so_far, self.size,
                    percentage),end="")

class R2:
    def __init__(self,access_key=None,secret_key=None,endpoint_url=None,bucket=None):
        self.access_key = access_key
        self.secret_key = secret_key
        self.endpoint_url = endpoint_url
        self.bucket = bucket
        self.client = None

    def initialize(self):
        if self.access_key is None or self.secret_key is None or self.endpoint_url is None:
            print("Please provide access_key, secret_key and endpoint_url")
            return
        self.client = boto3.client('s3',
                                    verify=False,
                  endpoint_url=self.endpoint_url, 
                  aws_access_key_id=self.access_key,
                  aws_secret_access_key=self.secret_key)
        return self

    def upload_file(self,local_file,cloud_path,bucket=None,progress_callback=None):
        if bucket is None and self.bucket is None:
            print("Please provide bucket name")
            return
        if bucket is None:
            bucket = self.bucket
        if progress_callback is None:
            try:
                self.client.upload_file(local_file,bucket,cloud_path+'/'+os.path.basename(local_file))
            except botocore.exceptions.ClientError as e:
                print(e)
        else:
            try:
                self.client.upload_file(local_file,bucket,cloud_path+'/'+os.path.basename(local_file),Callback=progress_callback)
            except botocore.exceptions.ClientError as e:
                print(e)

    def upload_folder(self,local_folder,cloud_path,bucket=None):
        if bucket is None and self.bucket is None:
            print("Please provide bucket name")
            return
        if bucket is None:
            bucket = self.bucket
        stream = tqdm(os.walk(local_folder))
        for root, dirs, files in stream:
           for file in files:
               localFilePath = os.path.join(root, file)
               relativePath = os.path.relpath(localFilePath, local_folder)
               cloudPath = os.path.join(cloud_path, relativePath)
               cloudPath = cloudPath.replace('\\', '/')
               try:
                   self.client.upload_file(localFilePath, bucket, cloudPath, Callback=ProgressPercentage(localFilePath))
                   print()
               except botocore.exceptions.ClientError as e:
                   print(e)