Flex  0.17.9
oss_storage.h
Go to the documentation of this file.
1 
16 #ifndef FLEX_UTILS_REMOTE_OSS_STORAGE_H_
17 #define FLEX_UTILS_REMOTE_OSS_STORAGE_H_
18 
19 #ifdef BUILD_WITH_OSS
20 
21 #include <filesystem>
22 #include <string>
23 #include "flex/third_party/aliyun-oss-cpp-sdk/sdk/include/alibabacloud/oss/OssClient.h"
24 #include "flex/third_party/aliyun-oss-cpp-sdk/sdk/include/alibabacloud/oss/client/ClientConfiguration.h"
26 #include "flex/utils/result.h"
27 
28 namespace gs {
29 
30 struct OSSConf {
31  static constexpr const char* kOSSAccessKeyId = "OSS_ACCESS_KEY_ID";
32  static constexpr const char* kOSSAccessKeySecret = "OSS_ACCESS_KEY_SECRET";
33  static constexpr const char* kOSSEndpoint = "OSS_ENDPOINT";
34  static constexpr const char* kOSSBucketName = "OSS_BUCKET_NAME";
35  // Avoid storing or printing the accesskey_id and accesskey_secret
36  std::string accesskey_id_;
37  std::string accesskey_secret_;
38  std::string endpoint_;
39  std::string bucket_name_;
40  int32_t uploading_concurrency_ = 4;
41  uint64_t partition_size_ = 1024 * 1024 * 128;
42  AlibabaCloud::OSS::ClientConfiguration client_conf_;
43 
44  void load_conf_from_env();
45 };
46 
47 class OSSRemoteStorageUploader : public RemoteStorageUploader {
48  public:
49  OSSRemoteStorageUploader(OSSConf conf = {}) : conf_(conf) {}
50 
51  ~OSSRemoteStorageUploader() override = default;
52 
53  // Will try to load the accesskey_id and accesskey_secret from the environment
54  // variables if they are not set in the OSSConf.
55  Status Open() override;
56 
57  Status Put(const std::string& local_path, const std::string& remote_path,
58  bool override = false) override;
59 
60  Status Delete(const std::string& remote_path) override;
61 
62  Status Close() override;
63 
64  private:
65  OSSConf conf_;
66  std::shared_ptr<AlibabaCloud::OSS::OssClient> client_;
67 };
68 
69 class OSSRemoteStorageDownloader : public RemoteStorageDownloader {
70  public:
71  OSSRemoteStorageDownloader(OSSConf conf = {}) : conf_(conf) {}
72 
73  ~OSSRemoteStorageDownloader() override = default;
74 
75  Status Open() override;
76 
77  Status Get(const std::string& remote_path,
78  const std::string& local_path) override;
79 
80  Status List(const std::string& remote_path,
81  std::vector<std::string>& list) override;
82 
83  Status Close() override;
84 
85  private:
86  bool get_metadata_etag(const std::string& remote_path, std::string& etag);
87  OSSConf conf_;
88  std::shared_ptr<AlibabaCloud::OSS::OssClient> client_;
89 };
90 } // namespace gs
91 
92 #endif // BUILD_WITH_OSS
93 
94 #endif // FLEX_UTILS_REMOTE_OSS_STORAGE_H_
gs
Definition: adj_list.h:23
remote_storage.h
result.h