oss_storage.h

Go to the documentation of this file.

 
#ifndef FLEX_UTILS_REMOTE_OSS_STORAGE_H_
#define FLEX_UTILS_REMOTE_OSS_STORAGE_H_
 
#ifdef BUILD_WITH_OSS
 
#include <filesystem>
#include <string>
#include "flex/third_party/aliyun-oss-cpp-sdk/sdk/include/alibabacloud/oss/OssClient.h"
#include "flex/third_party/aliyun-oss-cpp-sdk/sdk/include/alibabacloud/oss/client/ClientConfiguration.h"
#include "flex/utils/remote/remote_storage.h"
#include "flex/utils/result.h"
 
namespace gs {
 
struct OSSConf {
  static constexpr const char* kOSSAccessKeyId = "OSS_ACCESS_KEY_ID";
  static constexpr const char* kOSSAccessKeySecret = "OSS_ACCESS_KEY_SECRET";
  static constexpr const char* kOSSEndpoint = "OSS_ENDPOINT";
  static constexpr const char* kOSSBucketName = "OSS_BUCKET_NAME";
  static constexpr const char* kOSSConcurrency = "OSS_CONCURRENCY";
  // Avoid storing or printing the accesskey_id and accesskey_secret
  std::string accesskey_id_;
  std::string accesskey_secret_;
  std::string endpoint_;
  std::string bucket_name_;
  int32_t concurrency_ = 4;
  uint64_t partition_size_ = 1024 * 1024 * 128;
  AlibabaCloud::OSS::ClientConfiguration client_conf_;
 
  void load_conf_from_env();
};
 
class OSSRemoteStorageUploader : public RemoteStorageUploader {
 public:
  OSSRemoteStorageUploader(OSSConf conf = {}) : conf_(conf) {}
 
  ~OSSRemoteStorageUploader() override = default;
 
  // Will try to load the accesskey_id and accesskey_secret from the environment
  // variables if they are not set in the OSSConf.
  Status Open() override;
 
  Status Put(const std::string& local_path, const std::string& remote_path,
             bool override = false) override;
 
  Status Delete(const std::string& remote_path) override;
 
  Status Close() override;
 
 private:
  OSSConf conf_;
  std::shared_ptr<AlibabaCloud::OSS::OssClient> client_;
};
 
class OSSRemoteStorageDownloader : public RemoteStorageDownloader {
 public:
  OSSRemoteStorageDownloader(OSSConf conf = {}) : conf_(conf) {}
 
  ~OSSRemoteStorageDownloader() override = default;
 
  Status Open() override;
 
  Status Get(const std::string& remote_path,
             const std::string& local_path) override;
 
  Status List(const std::string& remote_path,
              std::vector<std::string>& list) override;
 
  Status Close() override;
 
 private:
  bool get_metadata_etag(const std::string& remote_path, std::string& etag);
  OSSConf conf_;
  std::shared_ptr<AlibabaCloud::OSS::OssClient> client_;
};
}  // namespace gs
 
#endif  // BUILD_WITH_OSS
 
#endif  // FLEX_UTILS_REMOTE_OSS_STORAGE_H_