csv_fragment_loader.h

Go to the documentation of this file.

 
#ifndef STORAGES_RT_MUTABLE_GRAPH_LOADER_CSV_FRAGMENT_LOADER_H_
#define STORAGES_RT_MUTABLE_GRAPH_LOADER_CSV_FRAGMENT_LOADER_H_
 
#include "flex/storages/rt_mutable_graph/loader/abstract_arrow_fragment_loader.h"
#include "flex/storages/rt_mutable_graph/loader/basic_fragment_loader.h"
#include "flex/storages/rt_mutable_graph/loader/i_fragment_loader.h"
#include "flex/storages/rt_mutable_graph/loader/loader_factory.h"
#include "flex/storages/rt_mutable_graph/loading_config.h"
#include "flex/storages/rt_mutable_graph/mutable_property_fragment.h"
 
#include <arrow/api.h>
#include <arrow/csv/api.h>
#include <arrow/io/api.h>
#include "arrow/util/value_parsing.h"
 
#include "grape/util.h"
 
namespace gs {
 
class CSVStreamRecordBatchSupplier : public IRecordBatchSupplier {
 public:
 CSVStreamRecordBatchSupplier(label_t label_id, const std::string& file_path,
 arrow::csv::ConvertOptions convert_options,
 arrow::csv::ReadOptions read_options,
 arrow::csv::ParseOptions parse_options);
 
 std::shared_ptr<arrow::RecordBatch> GetNextBatch() override;
 
 private:
 std::string file_path_;
 std::shared_ptr<arrow::csv::StreamingReader> reader_;
};
 
class CSVTableRecordBatchSupplier : public IRecordBatchSupplier {
 public:
 CSVTableRecordBatchSupplier(label_t label_id, const std::string& file_path,
 arrow::csv::ConvertOptions convert_options,
 arrow::csv::ReadOptions read_options,
 arrow::csv::ParseOptions parse_options);
 
 std::shared_ptr<arrow::RecordBatch> GetNextBatch() override;
 
 private:
 std::string file_path_;
 std::shared_ptr<arrow::Table> table_;
 std::shared_ptr<arrow::TableBatchReader> reader_;
};
 
// LoadFragment for csv files.
class CSVFragmentLoader : public AbstractArrowFragmentLoader {
 public:
 CSVFragmentLoader(const std::string& work_dir, const Schema& schema,
 const LoadingConfig& loading_config)
 : AbstractArrowFragmentLoader(work_dir, schema, loading_config) {}
 
 static std::shared_ptr<IFragmentLoader> Make(
 const std::string& work_dir, const Schema& schema,
 const LoadingConfig& loading_config);
 
 ~CSVFragmentLoader() {}
 
 Result<bool> LoadFragment() override;
 
 private:
 void loadVertices();
 
 void loadEdges();
 
 void addVertices(label_t v_label_id, const std::vector<std::string>& v_files);
 
 void addEdges(label_t src_label_id, label_t dst_label_id, label_t e_label_id,
 const std::vector<std::string>& e_files);
 
 void fillEdgeReaderMeta(arrow::csv::ReadOptions& read_options,
 arrow::csv::ParseOptions& parse_options,
 arrow::csv::ConvertOptions& convert_options,
 const std::string& e_file, label_t src_label_id,
 label_t dst_label_id, label_t label_id) const;
 
 void fillVertexReaderMeta(arrow::csv::ReadOptions& read_options,
 arrow::csv::ParseOptions& parse_options,
 arrow::csv::ConvertOptions& convert_options,
 const std::string& v_file, label_t v_label) const;
 
 static const bool registered_;
};
 
} // namespace gs
 
#endif // STORAGES_RT_MUTABLE_GRAPH_LOADER_CSV_FRAGMENT_LOADER_H_