gs::ODPSFragmentLoader Class Reference

#include <odps_fragment_loader.h>

Inheritance diagram for gs::ODPSFragmentLoader:

Collaboration diagram for gs::ODPSFragmentLoader:

Public Member Functions
	ODPSFragmentLoader (const std::string &work_dir, const Schema &schema, const LoadingConfig &loading_config)
	~ODPSFragmentLoader ()
Result< bool >	LoadFragment () override
Public Member Functions inherited from gs::AbstractArrowFragmentLoader
	AbstractArrowFragmentLoader (const std::string &work_dir, const Schema &schema, const LoadingConfig &loading_config)
	~AbstractArrowFragmentLoader ()
void	AddVerticesRecordBatch (label_t v_label_id, const std::vector< std::string > &input_paths, std::function< std::vector< std::shared_ptr< IRecordBatchSupplier >>(label_t, const std::string &, const LoadingConfig &, int)> supplier_creator)
void	AddEdgesRecordBatch (label_t src_label_id, label_t dst_label_id, label_t edge_label_id, const std::vector< std::string > &input_paths, std::function< std::vector< std::shared_ptr< IRecordBatchSupplier >>(label_t, label_t, label_t, const std::string &, const LoadingConfig &, int)> supplier_creator)
Public Member Functions inherited from gs::IFragmentLoader
virtual	~IFragmentLoader ()=default

Static Public Member Functions
static std::shared_ptr< IFragmentLoader >	Make (const std::string &work_dir, const Schema &schema, const LoadingConfig &loading_config)

Private Member Functions
void	init ()
void	parseLocation (const std::string &odps_table_path, TableIdentifier &table_identifier, std::vector< std::string > &partition_names, std::vector< std::string > &selected_partitions)
void	loadVertices ()
void	loadEdges ()
void	addVertices (label_t v_label_id, const std::vector< std::string > &v_files)
void	addEdges (label_t src_label_id, label_t dst_label_id, label_t e_label_id, const std::vector< std::string > &e_files)
std::vector< std::string >	columnMappingsToSelectedCols (const std::vector< std::tuple< size_t, std::string, std::string >> &column_mappings)

Private Attributes
ODPSReadClient	odps_read_client_

Static Private Attributes
static const bool	registered_

Additional Inherited Members
Protected Member Functions inherited from gs::AbstractArrowFragmentLoader
template<typename KEY_T >
void	addVertexBatchFromArray (label_t v_label_id, IdIndexer< KEY_T, vid_t > &indexer, std::shared_ptr< arrow::Array > &primary_key_col, const std::vector< std::shared_ptr< arrow::Array >> &property_cols)
template<typename KEY_T >
void	addVertexRecordBatchImpl (label_t v_label_id, const std::vector< std::string > &v_files, std::function< std::vector< std::shared_ptr< IRecordBatchSupplier >>(label_t, const std::string &, const LoadingConfig &, int)> supplier_creator)
template<typename SRC_PK_T , typename EDATA_T , typename VECTOR_T >
void	_append_edges (std::shared_ptr< arrow::Array > src_col, std::shared_ptr< arrow::Array > dst_col, const IndexerType &src_indexer, const IndexerType &dst_indexer, std::shared_ptr< arrow::Array > &property_cols, const PropertyType &edge_property, VECTOR_T &parsed_edges, std::vector< std::atomic< int32_t >> &ie_degree, std::vector< std::atomic< int32_t >> &oe_degree, size_t offset)
template<typename EDATA_T >
void	addEdgesRecordBatchImpl (label_t src_label_id, label_t dst_label_id, label_t e_label_id, const std::vector< std::string > &e_files, std::function< std::vector< std::shared_ptr< IRecordBatchSupplier >>(label_t, label_t, label_t, const std::string &, const LoadingConfig &, int)> supplier_creator)
template<typename EDATA_T , typename VECTOR_T >
void	addEdgesRecordBatchImplHelper (label_t src_label_id, label_t dst_label_id, label_t e_label_id, const std::vector< std::string > &e_files, std::function< std::vector< std::shared_ptr< IRecordBatchSupplier >>(label_t, label_t, label_t, const std::string &, const LoadingConfig &, int)> supplier_creator)
Protected Attributes inherited from gs::AbstractArrowFragmentLoader
const LoadingConfig &	loading_config_
const Schema &	schema_
size_t	vertex_label_num_
size_t	edge_label_num_
int32_t	thread_num_
std::mutex *	mtxs_
bool	build_csr_in_mem_
bool	use_mmap_vector_
BasicFragmentLoader	basic_fragment_loader_

Constructor & Destructor Documentation

ODPSFragmentLoader()

gs::ODPSFragmentLoader::ODPSFragmentLoader ( const std::string &  work_dir, const Schema &  schema, const LoadingConfig &  loading_config  )

inline

~ODPSFragmentLoader()

gs::ODPSFragmentLoader::~ODPSFragmentLoader ( )

inline

Member Function Documentation

addEdges()

void gs::ODPSFragmentLoader::addEdges ( label_t  src_label_id, label_t  dst_label_id, label_t  e_label_id, const std::vector< std::string > &  e_files  )

private

addVertices()

void gs::ODPSFragmentLoader::addVertices ( label_t  v_label_id, const std::vector< std::string > &  v_files  )

private

columnMappingsToSelectedCols()

std::vector< std::string > gs::ODPSFragmentLoader::columnMappingsToSelectedCols ( const std::vector< std::tuple< size_t, std::string, std::string >> &  column_mappings )

private

init()

void gs::ODPSFragmentLoader::init ( )

private

loadEdges()

void gs::ODPSFragmentLoader::loadEdges ( )

private

LoadFragment()

Result< bool > gs::ODPSFragmentLoader::LoadFragment ( )

overridevirtual

Implements gs::IFragmentLoader.

loadVertices()

void gs::ODPSFragmentLoader::loadVertices ( )

private

Make()

std::shared_ptr< IFragmentLoader > gs::ODPSFragmentLoader::Make ( const std::string &  work_dir, const Schema &  schema, const LoadingConfig &  loading_config  )

static

parseLocation()

void gs::ODPSFragmentLoader::parseLocation ( const std::string &  odps_table_path, TableIdentifier &  table_identifier, std::vector< std::string > &  partition_names, std::vector< std::string > &  selected_partitions  )

private

Member Data Documentation

odps_read_client_

ODPSReadClient gs::ODPSFragmentLoader::odps_read_client_

private

registered_

const bool gs::ODPSFragmentLoader::registered_

staticprivate

Initial value:

=
 LoaderFactory::Register("odps", "arrow",
 static_cast<LoaderFactory::loader_initializer_t>(
 &ODPSFragmentLoader::Make))

---

The documentation for this class was generated from the following files:

• /home/runner/work/GraphScope/GraphScope/flex/storages/rt_mutable_graph/loader/odps_fragment_loader.h
• /home/runner/work/GraphScope/GraphScope/flex/storages/rt_mutable_graph/loader/odps_fragment_loader.cc