file_names.h

Go to the documentation of this file.

 
#ifndef STORAGES_RT_MUTABLE_GRAPH_FILE_NAMES_H_
#define STORAGES_RT_MUTABLE_GRAPH_FILE_NAMES_H_
 
#include <assert.h>
#include <fcntl.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <filesystem>
 
#include "glog/logging.h"
 
namespace gs {
// clang-format off
/*
    ├── schema
    ├── runtime
    │   ├── allocator                                // allocator dir
    │   ├── tails                                    // tails (mutable parts) of tables 
    │   │   ├── vertex_table_PERSON.col_0 
    │   │   ├── vertex_table_PERSON.col_1.data 
    │   │   └── vertex_table_PERSON.col_1.items
    │   └── tmp                                      // tmp dir, used for touched vertex maps, vertex tables and adjlists of csrs 
    │       ├── ie_PERSON_KNOWS_PERSON.adj 
    │       ├── oe_PERSON_KNOWS_PERSON.adj 
    │       ├── vertex_map_PERSON.indices 
    │       ├── vertex_map_PERSON.keys 
    │       ├── vertex_table_PERSON.col_0 
    │       ├── vertex_table_PERSON.col_1.data
    │       └── vertex_table_PERSON.col_1.items 
            └── bulk_load_progress.log               // bulk load progress file
    ├── snapshots // snapshots dir 
    │   ├── 0 
    │   │   ├── ie_PERSON_KNOWS_PERSON.deg 
    │   │   ├── ie_PERSON_KNOWS_PERSON.nbr 
    │   │   ├── oe_PERSON_KNOWS_PERSON.deg 
    │   │   ├── oe_PERSON_KNOWS_PERSON.nbr 
    │   │   ├── vertex_map_PERSON.indices 
    │   │   ├── vertex_map_PERSON.keys 
    │   │   ├── vertex_map_PERSON.meta 
    │   │   ├── vertex_table_PERSON.col_0 
    │   │   ├── vertex_table_PERSON.col_1.data 
    │   │   └── vertex_table_PERSON.col_1.items
    │   ├── 1234567
    │   │   ├── ie_PERSON_KNOWS_PERSON.deg
    │   │   ├── ie_PERSON_KNOWS_PERSON.nbr
    │   │   ├── oe_PERSON_KNOWS_PERSON.deg
    │   │   ├── oe_PERSON_KNOWS_PERSON.nbr
    │   │   ├── vertex_map_PERSON.indices
    │   │   ├── vertex_map_PERSON.keys
    │   │   ├── vertex_map_PERSON.meta
    │   │   ├── vertex_table_PERSON.col_0
    │   │   ├── vertex_table_PERSON.col_1.data
    │   │   └── vertex_table_PERSON.col_1.items
    │   ├── ...
    │   └── VERSION
    └── wal                                         // wal dir
        ├── log_0
        ├── log_1
        └── ...
*/
// clang-format on
 
inline void copy_file(const std::string& src, const std::string& dst) {
  if (!std::filesystem::exists(src)) {
    LOG(ERROR) << "file not exists: " << src;
    return;
  }
#if USE_COPY_FILE_RANGE
  size_t len = std::filesystem::file_size(src);
  int src_fd = open(src.c_str(), O_RDONLY, 0777);
  bool creat = false;
  if (!std::filesystem::exists(dst)) {
    creat = true;
  }
  int dst_fd = open(dst.c_str(), O_WRONLY | O_CREAT, 0777);
  if (creat) {
    std::filesystem::perms readWritePermission =
        std::filesystem::perms::owner_read |
        std::filesystem::perms::owner_write;
    std::error_code errorCode;
    std::filesystem::permissions(dst, readWritePermission,
                                 std::filesystem::perm_options::add, errorCode);
    if (errorCode) {
      LOG(ERROR) << "Failed to set read/write permission for file: " << dst
                 << " " << errorCode.message() << std::endl;
    }
 
    // For a newly created file, you may need to close and then reopen it,
    // otherwise you may encounter a copy_file_range "Invalid cross-device link"
    // error, one possible cause of the error could be that the
    // file's metadata has not yet been flushed to the file system.
    close(dst_fd);
    dst_fd = open(dst.c_str(), O_WRONLY, 0777);
  }
  ssize_t ret;
  do {
    ret = copy_file_range(src_fd, NULL, dst_fd, NULL, len, 0);
    if (ret == -1) {
      perror("copy_file_range");
      return;
    }
    len -= ret;
  } while (len > 0 && ret > 0);
  close(src_fd);
  close(dst_fd);
#else
  bool creat = false;
  if (!std::filesystem::exists(dst)) {
    creat = true;
  }
  std::error_code errorCode;
  std::filesystem::copy_file(
      src, dst, std::filesystem::copy_options::overwrite_existing, errorCode);
  if (errorCode) {
    LOG(ERROR) << "Failed to copy file from " << src << " to " << dst << " "
               << errorCode.message() << std::endl;
  }
  if (creat) {
    std::filesystem::perms readWritePermission =
        std::filesystem::perms::owner_read |
        std::filesystem::perms::owner_write;
    std::error_code errorCode;
    std::filesystem::permissions(dst, readWritePermission,
                                 std::filesystem::perm_options::add, errorCode);
    if (errorCode) {
      LOG(INFO) << "Failed to set read/write permission for file: " << dst
                << " " << errorCode.message() << std::endl;
    }
  }
 
#endif
}
 
inline std::string schema_path(const std::string& work_dir) {
  return work_dir + "/schema";
}
 
inline std::string snapshots_dir(const std::string& work_dir) {
  return work_dir + "/snapshots/";
}
 
inline std::string snapshot_version_path(const std::string& work_dir) {
  return snapshots_dir(work_dir) + "/VERSION";
}
 
inline std::string get_latest_snapshot(const std::string& work_dir) {
  std::string snapshots_dir = work_dir + "/snapshots";
  uint32_t version;
  {
    FILE* fin = fopen((snapshots_dir + "/VERSION").c_str(), "r");
    CHECK_EQ(fread(&version, sizeof(uint32_t), 1, fin), 1);
    fclose(fin);
  }
  return snapshots_dir + "/" + std::to_string(version);
}
 
inline uint32_t get_snapshot_version(const std::string& work_dir) {
  std::string version_path = snapshot_version_path(work_dir);
  FILE* version_file = fopen(version_path.c_str(), "rb");
  uint32_t version = 0;
  CHECK_EQ(fread(&version, sizeof(uint32_t), 1, version_file), 1);
  fclose(version_file);
  return version;
}
 
inline void set_snapshot_version(const std::string& work_dir,
                                 uint32_t version) {
  std::string version_path = snapshot_version_path(work_dir);
  FILE* version_file = fopen(version_path.c_str(), "wb");
  CHECK_EQ(fwrite(&version, sizeof(uint32_t), 1, version_file), 1);
  fflush(version_file);
  fclose(version_file);
}
 
inline std::string snapshot_dir(const std::string& work_dir, uint32_t version) {
  return snapshots_dir(work_dir) + std::to_string(version) + "/";
}
 
inline std::string wal_dir(const std::string& work_dir) {
  return work_dir + "/wal/";
}
 
inline std::string runtime_dir(const std::string& work_dir) {
  return work_dir + "/runtime/";
}
 
inline std::string update_txn_dir(const std::string& work_dir,
                                  uint32_t version) {
  return runtime_dir(work_dir) + "update_txn_" + std::to_string(version) + "/";
}
 
inline std::string allocator_dir(const std::string& work_dir) {
  return runtime_dir(work_dir) + "allocator/";
}
 
inline std::string tmp_dir(const std::string& work_dir) {
  return runtime_dir(work_dir) + "tmp/";
}
 
inline std::string bulk_load_progress_file(const std::string& work_dir) {
  return tmp_dir(work_dir) + "bulk_load_progress.log";
}
 
inline void clear_tmp(const std::string& work_dir) {
  std::string tmp_dir_str = tmp_dir(work_dir);
  if (std::filesystem::exists(tmp_dir_str)) {
    assert(std::filesystem::is_directory(tmp_dir_str));
    if (std::filesystem::directory_iterator(tmp_dir_str) !=
        std::filesystem::directory_iterator()) {
      for (const auto& entry :
           std::filesystem::directory_iterator(tmp_dir_str)) {
        std::filesystem::remove_all(entry.path());
      }
    }
  }
}
 
inline std::string vertex_map_prefix(const std::string& label) {
  return "vertex_map_" + label;
}
 
inline std::string ie_prefix(const std::string& src_label,
                             const std::string& dst_label,
                             const std::string edge_label) {
  return "ie_" + src_label + "_" + edge_label + "_" + dst_label;
}
 
inline std::string oe_prefix(const std::string& src_label,
                             const std::string& dst_label,
                             const std::string edge_label) {
  return "oe_" + src_label + "_" + edge_label + "_" + dst_label;
}
 
inline std::string edata_prefix(const std::string& src_label,
                                const std::string& dst_label,
                                const std::string& edge_label) {
  return "e_" + src_label + "_" + edge_label + "_" + dst_label + "_data";
}
inline std::string vertex_table_prefix(const std::string& label) {
  return "vertex_table_" + label;
}
 
inline std::string thread_local_allocator_prefix(const std::string& work_dir,
                                                 int thread_id) {
  return allocator_dir(work_dir) + "allocator_" + std::to_string(thread_id) +
         "_";
}
 
}  // namespace gs
 
#endif  // STORAGES_RT_MUTABLE_GRAPH_FILE_NAMES_H_