csv_fragment_loader.h

Go to the documentation of this file.

 
#ifndef STORAGES_RT_MUTABLE_GRAPH_LOADER_CSV_FRAGMENT_LOADER_H_
#define STORAGES_RT_MUTABLE_GRAPH_LOADER_CSV_FRAGMENT_LOADER_H_
 
#include "flex/storages/rt_mutable_graph/loader/abstract_arrow_fragment_loader.h"
#include "flex/storages/rt_mutable_graph/loader/basic_fragment_loader.h"
#include "flex/storages/rt_mutable_graph/loader/i_fragment_loader.h"
#include "flex/storages/rt_mutable_graph/loader/loader_factory.h"
#include "flex/storages/rt_mutable_graph/loading_config.h"
#include "flex/storages/rt_mutable_graph/mutable_property_fragment.h"
 
#include <arrow/api.h>
#include <arrow/csv/api.h>
#include <arrow/io/api.h>
#include "arrow/util/value_parsing.h"
 
#include "grape/util.h"
 
namespace gs {
 
class CSVStreamRecordBatchSupplier : public IRecordBatchSupplier {
 public:
  CSVStreamRecordBatchSupplier(label_t label_id, const std::string& file_path,
                               arrow::csv::ConvertOptions convert_options,
                               arrow::csv::ReadOptions read_options,
                               arrow::csv::ParseOptions parse_options);
 
  std::shared_ptr<arrow::RecordBatch> GetNextBatch() override;
 
 private:
  std::string file_path_;
  std::shared_ptr<arrow::csv::StreamingReader> reader_;
};
 
class CSVTableRecordBatchSupplier : public IRecordBatchSupplier {
 public:
  CSVTableRecordBatchSupplier(label_t label_id, const std::string& file_path,
                              arrow::csv::ConvertOptions convert_options,
                              arrow::csv::ReadOptions read_options,
                              arrow::csv::ParseOptions parse_options);
 
  std::shared_ptr<arrow::RecordBatch> GetNextBatch() override;
 
 private:
  std::string file_path_;
  std::shared_ptr<arrow::Table> table_;
  std::shared_ptr<arrow::TableBatchReader> reader_;
};
 
// LoadFragment for csv files.
class CSVFragmentLoader : public AbstractArrowFragmentLoader {
 public:
  CSVFragmentLoader(const std::string& work_dir, const Schema& schema,
                    const LoadingConfig& loading_config)
      : AbstractArrowFragmentLoader(work_dir, schema, loading_config) {}
 
  static std::shared_ptr<IFragmentLoader> Make(
      const std::string& work_dir, const Schema& schema,
      const LoadingConfig& loading_config);
 
  ~CSVFragmentLoader() {}
 
  Result<bool> LoadFragment() override;
 
 private:
  void loadVertices();
 
  void loadEdges();
 
  void addVertices(label_t v_label_id, const std::vector<std::string>& v_files);
 
  void addEdges(label_t src_label_id, label_t dst_label_id, label_t e_label_id,
                const std::vector<std::string>& e_files);
 
  void fillEdgeReaderMeta(arrow::csv::ReadOptions& read_options,
                          arrow::csv::ParseOptions& parse_options,
                          arrow::csv::ConvertOptions& convert_options,
                          const std::string& e_file, label_t src_label_id,
                          label_t dst_label_id, label_t label_id) const;
 
  void fillVertexReaderMeta(arrow::csv::ReadOptions& read_options,
                            arrow::csv::ParseOptions& parse_options,
                            arrow::csv::ConvertOptions& convert_options,
                            const std::string& v_file, label_t v_label) const;
 
  static const bool registered_;
};
 
}  // namespace gs
 
#endif  // STORAGES_RT_MUTABLE_GRAPH_LOADER_CSV_FRAGMENT_LOADER_H_