Utilities Private Part (do not include from applications). More...

#include <stdio.h>
#include <stdlib.h>
#include <errno.h>

Macros
#define	CHECK_ONE_FN_OPTION(NAME, A, B)

#define	KMR_ALIGN(X) (((X)+((8)-1))&~((8)-1))
	Rounds up a given size to the alignment restriction (currently eight bytes). More...

#define	KMR_DEBUGX(X) ((void)0)

#define	KMR_EMPTY

#define	kmr_malloc(Z) kmr_safe_malloc((Z), #Z, __FILE__, __LINE__, __func__)
	Allocates memory, or aborts when failed. More...

#define	KMR_MAX_THREADS 200

#define	KMR_OMP_CRITICAL_ KMR_EMPTY

#define	KMR_OMP_FOR_ KMR_EMPTY

#define	KMR_OMP_GET_THREAD_NUM() 0

#define	KMR_OMP_PARALLEL_ KMR_EMPTY

#define	KMR_OMP_PARALLEL_FOR_ KMR_EMPTY

#define	KMR_OMP_PARALLEL_IF_(CC) KMR_EMPTY

#define	KMR_OMP_SINGLE_NOWAIT_ KMR_EMPTY

#define	KMR_OMP_TASK_ KMR_EMPTY

#define	KMR_OMP_TASK_FINAL_(CC) KMR_EMPTY

#define	KMR_OMP_TASKWAIT_ KMR_EMPTY

#define	kmr_realloc(P, Z) kmr_safe_realloc((P), (Z), #Z, __FILE__, __LINE__, __func__)
	Allocates memory, or aborts when failed. More...

#define	KMR_TAG_ATOA 100

#define	KMR_TAG_MAP_BY_RANK 200

#define	KMR_TAG_PEER(THRD) (KMR_TAG_PEER_0 + (THRD))

#define	KMR_TAG_PEER_0 400

#define	KMR_TAG_PEER_END (KMR_TAG_PEER_0 + KMR_MAX_THREADS)

#define	KMR_TAG_PUSHOFF 201

#define	KMR_TAG_REQ 399

#define	KMR_TICK_ONLY_FOR_AMD64_AND_SPARC 0

#define	KMR_TRACE_FILE_PREFIX "kmrlog"
	Prefix to Trace Files. More...

#define	xassert(X)

Typedefs
typedef unsigned short	kmr_k_position_t[4]
	Positions of node by (X,Y,Z,ABC), with ABC axes collapsed. More...

typedef int(*	kmr_qsorter_t) (const void p, const void q)

typedef int(*	kmr_record_sorter_t) (const struct kmr_keyed_record p, const struct kmr_keyed_record q)

typedef int(*	kmr_sorter_t) (const struct kmr_kv_box p, const struct kmr_kv_box q)

Functions
int	kmr_add_kv_done_pushoff (KMR_KVS *kvs)
	Marks finished adding key-value pairs, called from kmr_add_kv_done(). More...

int	kmr_add_kv_pushoff (KMR_KVS *kvs, const struct kmr_kv_box kv)
	Adds a key-value pair. More...

int	kmr_allgatherv (KMR mr, _Bool rankzeroonly, void sbuf, long scnt, void rbuf, long rcnts, long *rdsps)
	All-gathers data, or gathers data when RANKZEROONLY. More...

int	kmr_allocate_block (KMR_KVS *kvs, size_t size)

int	kmr_alltoallv (KMR mr, void sbuf, long scounts, long sdsps, void rbuf, long rcounts, long *rdsps)
	Does all-to-all-v, but it takes arguments of long-integers. More...

static void	kmr_assert_i_kvs_ok (KMR_KVS *kvi, _Bool irequired)

static void	kmr_assert_i_kvs_ok_at_site (KMR_KVS kvi, _Bool irequired, struct kmr_code_line site)

static void	kmr_assert_kv_sizes (KMR_KVS *kvs, const struct kmr_kv_box kv)

static void	kmr_assert_kvs_ok (KMR_KVS kvi, KMR_KVS kvo, _Bool irequired, _Bool orequired)

static void	kmr_assert_o_kvs_ok (KMR_KVS *kvo, _Bool orequired)

static void	kmr_assert_o_kvs_ok_at_site (KMR_KVS kvo, _Bool orequired, struct kmr_code_line site)

int	kmr_assert_sorted (KMR_KVS *kvi, _Bool locally, _Bool shuffling, _Bool ranking)
	Checks a key-value stream is sorted. More...

void *	kmr_bsearch (const void key, const void base, size_t nel, size_t size, int(compar)(const void , const void *))
	Searches a key entry like bsearch(3C), but returns a next greater entry instead of null on no match. More...

static _Bool	kmr_check_alignment (size_t sz)

static void	kmr_check_fn_options (KMR mr, struct kmr_option provide, struct kmr_option given, const char func)

int	kmr_check_options (KMR *mr, MPI_Info conf)

kmr_sorter_t	kmr_choose_sorter (const KMR_KVS *kvs)

void	kmr_ckpt_create_context (KMR *)
	Initialize checkpoint context. More...

int	kmr_ckpt_disable_ckpt (KMR *)
	It temporally disables checkpoint/restart. More...

int	kmr_ckpt_enable_ckpt (KMR *, int)
	It temporally enables checkpoint/restart which has been disabled by calling kmr_ckpt_disable_ckpt(). More...

int	kmr_ckpt_enabled (KMR *)
	Check if checkpoint/restart is enabled. More...

long	kmr_ckpt_first_unprocessed_kv (KMR *)
	It returns the index of the first unprocessed key-value in the input KVS. More...

void	kmr_ckpt_free_context (KMR *)
	Free checkpoint context. More...

void	kmr_ckpt_lock_finish (KMR *)
	Define the end position of code region that is referred when restart. More...

void	kmr_ckpt_lock_start (KMR *)
	Define the start position of code region that is referred when restart. More...

void	kmr_ckpt_progress_fin (KMR *)
	It finalizes the progress of MapReduce checkpointing. More...

int	kmr_ckpt_progress_init (KMR_KVS , KMR_KVS , struct kmr_option)
	It initializes a progress of MapReduce checkpointing. More...

void	kmr_ckpt_remove_ckpt (KMR_KVS *)
	It removes checkpoint data file. More...

void	kmr_ckpt_restore_ckpt (KMR_KVS *)
	It restores checkpoint data to kvs. More...

void	kmr_ckpt_save_kvo_block_add (KMR , KMR_KVS , long)
	It adds a new block of key-value pairs of the output KVS to the checkpoint data file. More...

void	kmr_ckpt_save_kvo_block_fin (KMR , KMR_KVS )
	It finalizes saving block of key-value pairs of the output KVS to the checkpoint data file. More...

void	kmr_ckpt_save_kvo_block_init (KMR , KMR_KVS )
	It initializes saving blocks of key-value pairs of the output KVS to a checkpoint data file. More...

void	kmr_ckpt_save_kvo_each_add (KMR , KMR_KVS , long)
	It adds new key-value pairs of the output KVS to the checkpoint data file. More...

void	kmr_ckpt_save_kvo_each_fin (KMR , KMR_KVS )
	It finalizes saving indexed key-value pairs of the output KVS to the checkpoint data file. More...

void	kmr_ckpt_save_kvo_each_init (KMR , KMR_KVS )
	It initializes saving indexed key-value pairs of the output KVS to a checkpoint data file. More...

void	kmr_ckpt_save_kvo_whole (KMR , KMR_KVS )
	It saves all key-value pairs in the output KVS to a checkpoint data file. More...

int	kmr_copy_mpi_info (MPI_Info src, MPI_Info dst)
	Copies contents of MPI_Info. More...

static struct kmr_option	kmr_copy_options_i_part (struct kmr_option opt)

static struct kmr_option	kmr_copy_options_m_part (struct kmr_option opt)

static struct kmr_option	kmr_copy_options_o_part (struct kmr_option opt)

static struct kmr_option	kmr_copy_s_option (struct kmr_option opt)

long	kmr_dblint_ff (double v)

int	kmr_dump_mpi_info (char *prefix, MPI_Info info)
	Dumps simply contents in MPI_Info. More...

void	kmr_error (KMR mr, char m)

void	kmr_error2 (KMR mr, char m, const char file, const int line, const char func)

void	kmr_error_at_site (KMR mr, char m, struct kmr_code_line *site)

void	kmr_error_kvs (KMR mr, char m, KMR_KVS *kvs)

void	kmr_error_kvs2 (KMR mr, char m, KMR_KVS kvs, const char file, const int line, const char *func)

void	kmr_error_kvs_at_site (KMR mr, char m, KMR_KVS kvs, struct kmr_code_line site)

void	kmr_error_mpi (KMR mr, char m, int errorcode)

int	kmr_exchange_sizes (KMR mr, long sbuf, long *rbuf)
	Calls all-to-all to exchange one long-integer. More...

int	kmr_fgetc (FILE *f)
	Does fgetc, avoiding EINTR. More...

static _Bool	kmr_fields_pointer_p (KMR_KVS *kvs)

struct kmr_kvs_entry *	kmr_find_kvs_last_entry (KMR_KVS *kvs)
	Finds the last entry of a key-value stream. More...

unsigned long	kmr_fix_bits_endian_ff (unsigned long b)
	Fixes little-endian bits used in Fortran to host-endian. More...

FILE *	kmr_fopen (const char n, const char m)
	Does fopen, avoiding EINTR. More...

static void	kmr_free (void *p, size_t sz)

int	kmr_free_kvs_pushoff (KMR_KVS *kvs, _Bool deallocate)

int	kmr_gather_sizes (KMR mr, long siz, long rbuf)
	Calls all-gather for collecting one long-integer. More...

int	kmr_get_key_type_ff (const KMR_KVS *kvs)

int	kmr_get_nprocs (const KMR *mr)

int	kmr_get_nprocs_ff (const KMR_KVS *kvs)

int	kmr_get_rank (const KMR *mr)

int	kmr_get_rank_ff (const KMR_KVS *kvs)

int	kmr_get_spawner_communicator_ff (KMR mr, long i, int comm)

int	kmr_get_value_type_ff (const KMR_KVS *kvs)

int	kmr_getdtablesize (KMR *mr)
	Does getdtablesize(); it is defined, because it is not Posix. More...

int	kmr_imax_one_fn (const struct kmr_kv_box kv[], const long n, const KMR_KVS kvi, KMR_KVS kvo, void *p)

void	kmr_init_kvs_oncore (KMR_KVS kvs, KMR mr)

int	kmr_install_watch_program (KMR mr, char msg)

double	kmr_intdbl_ff (long v)

void *	kmr_intptr_ff (long p)

int	kmr_intstr_ff (long p, char *s, int n)
	Fills the character array S by the contents at the pointer value integer P by the length N. More...

int	kmr_iogroup_distance (int a0, int a1)

int	kmr_iogroup_of_node (KMR *mr)
	Returns an I/O-group (an integer key) of a compute node. More...

int	kmr_iogroup_of_obd (int obdidx)
	Returns an I/O-group (an integer key) of a disk from an OBDIDX of Lustre file-system. More...

void	kmr_isort (void *a, size_t n, size_t es, int depth)
	Sorts by comparator on long integers. More...

int	kmr_isum_one_fn (const struct kmr_kv_box kv[], const long n, const KMR_KVS kvi, KMR_KVS kvo, void *p)

int	kmr_k_node (KMR *mr, kmr_k_position_t p)
	Gets TOFU position (physical coordinates) of the node. More...

static _Bool	kmr_key_pointer_p (KMR_KVS *kvs)

static size_t	kmr_kv_key_size (const KMR_KVS *kvs, const struct kmr_kv_box kv)

static size_t	kmr_kv_value_size (const KMR_KVS *kvs, const struct kmr_kv_box kv)

static struct kmr_kvs_entry *	kmr_kvs_adding_point (struct kmr_kvs_block *b)

static struct kmr_kvs_entry *	kmr_kvs_entry_at (KMR_KVS kvs, const struct kmr_kvs_block b, size_t bytes)

static _Bool	kmr_kvs_entry_fits_in_block (KMR_KVS kvs, struct kmr_kvs_block b, size_t sz)

static size_t	kmr_kvs_entry_netsize (const struct kmr_kvs_entry *e)

static size_t	kmr_kvs_entry_netsize_of_box (const struct kmr_kv_box kv)

static size_t	kmr_kvs_entry_size (KMR_KVS kvs, const struct kmr_kvs_entry e)

static size_t	kmr_kvs_entry_size_of_box (KMR_KVS *kvs, const struct kmr_kv_box kv)

static _Bool	kmr_kvs_entry_tail_p (struct kmr_kvs_entry *e)

static struct kmr_kvs_block *	kmr_kvs_first_block (KMR_KVS *kvs)

static struct kmr_kvs_entry *	kmr_kvs_first_entry (KMR_KVS kvs, const struct kmr_kvs_block b)

static void	kmr_kvs_insert_block (KMR_KVS kvs, struct kmr_kvs_block b)

static struct kmr_kvs_entry *	kmr_kvs_last_entry_limit (const KMR_KVS kvs, const struct kmr_kvs_block b)

static void	kmr_kvs_mark_entry_tail (struct kmr_kvs_entry *e)

static struct kmr_kvs_entry *	kmr_kvs_next (KMR_KVS kvs, const struct kmr_kvs_entry e, _Bool bound_in_block)

static struct kmr_kvs_entry *	kmr_kvs_next_entry (KMR_KVS kvs, const struct kmr_kvs_entry e)

static void	kmr_kvs_reset_block (KMR_KVS kvs, struct kmr_kvs_block b, size_t size, size_t netsize)

static void	kmr_link_kvs (KMR_KVS *kvs)

int	kmr_load_preference (KMR *mr, MPI_Info info)

int	kmr_load_properties (MPI_Info conf, char *filename)
	Loads properties into MPI_Info (in Latin1 characters). More...

void	kmr_log_map (KMR mr, KMR_KVS kvs, struct kmr_kv_box *ev, long i, long cnt, kmr_mapfn_t m, double dt)

void	kmr_log_reduce (KMR mr, KMR_KVS kvs, struct kmr_kv_box *ev, long n, kmr_redfn_t r, double dt)

int	kmr_map_via_spawn_ff (KMR_KVS kvi, KMR_KVS kvo, void *arg, int finfo, struct kmr_spawn_option opt, kmr_mapfn_t m)

void	kmr_mfree (void *p, size_t sz)

uint64_t	kmr_mpi_constant_value (char *s)

size_t	kmr_mpi_type_size (char *s)

int	kmr_msleep (int msec, int interval)

static size_t	kmr_ntuple_data_offset (int n)

static int	kmr_ntuple_entry_size (int len)

static void *	kmr_ntuple_insertion_point (struct kmr_ntuple *u)

static size_t	kmr_ntuple_nth_offset (struct kmr_ntuple *u, int nth)

void	kmr_open_log (KMR *mr)

int	kmr_pairing_fn (const struct kmr_kv_box kv, const KMR_KVS kvi, KMR_KVS kvo, void *p, const long i)

int	kmr_parse_boolean (char s, int r)

int	kmr_parse_int (char s, int r)

static struct kmr_kv_box	kmr_pick_kv (struct kmr_kvs_entry e, KMR_KVS kvs)
	Returns a handle to a key-value entry – a reverse of kmr_poke_kv(). More...

static struct kmr_kv_box	kmr_pick_kv2 (struct kmr_kvs_entry *e, enum kmr_kv_field keyf, enum kmr_kv_field valf)

int	kmr_pitch_rank (const struct kmr_kv_box kv, KMR_KVS *kvs)

static union kmr_unit_sized *	kmr_point_key (const struct kmr_kvs_entry *e)

static union kmr_unit_sized *	kmr_point_value (const struct kmr_kvs_entry *e)

static void	kmr_poke_kv (struct kmr_kvs_entry e, const struct kmr_kv_box kv, struct kmr_kv_box xkv, const KMR_KVS *kvs, _Bool reserve_space_only)
	Stores a key-value pair at the entry E in the store – a reverse of kmr_pick_kv(). More...

static void	kmr_poke_kv2 (struct kmr_kvs_entry e, const struct kmr_kv_box kv, struct kmr_kv_box xkv, enum kmr_kv_field keyf, enum kmr_kv_field valf, _Bool reserve_space_only)

void	kmr_print_file_options (struct kmr_file_option opt)

void	kmr_print_options (struct kmr_option opt)

void	kmr_print_spawn_options (struct kmr_spawn_option opt)

void	kmr_print_string (char msg, char s, int len)

long	kmr_ptrint_ff (void *p)

char *	kmr_ptrstr_ff (char *s)

int	kmr_pushoff_make_stationary (KMR_KVS *kvs)

int	kmr_reverse_fn (const struct kmr_kv_box kv, const KMR_KVS kvs, KMR_KVS kvo, void *p, const long i)

static void *	kmr_safe_malloc (size_t s, char szexpr, const char file, const int line, const char *func)

static void *	kmr_safe_realloc (void q, size_t s, char szexpr, const char file, const int line, const char func)

int	kmr_scan_argv_strings (KMR mr, char s, size_t len, int arglim, int argc, char argv, _Bool wssep, char msg)

int	kmr_set_option_by_strings (KMR mr, char k, char *v)

static _Bool	kmr_shuffle_compatible_data_p (enum kmr_kv_field d0, enum kmr_kv_field d1)

static _Bool	kmr_shuffle_compatible_p (KMR_KVS s0, KMR_KVS s1)

signed long	kmr_stable_key (const struct kmr_kv_box kv, const KMR_KVS *kvs)

void *	kmr_strdup (char *s)
	STRDUP, but aborts on failure. More...

void	kmr_string_truncation (KMR mr, size_t sz, char s)
	Modifies the string end with by "..." for indicating truncation, used on the result of snprintf. More...

char *	kmr_stringify_file_options (struct kmr_file_option o)
	Returns a print string of a single option, to check the bits are properly encoded in foreign language interfaces. More...

char *	kmr_stringify_options (struct kmr_option o)
	Returns a print string of a single option, to check the bits are properly encoded in foreign language interfaces. More...

char *	kmr_stringify_spawn_options (struct kmr_spawn_option o)
	Returns a print string of a single option, to check the bits are properly encoded in foreign language interfaces. More...

long	kmr_strint_ff (char *p)

char *	kmr_strptr_ff (char *s)
	Returns itself; this is for Fortran-binding. More...

static long	kmr_tick ()
	Returns the clock counter value. More...

static enum kmr_kv_field	kmr_unit_sized_or_opaque (enum kmr_kv_field data)

static _Bool	kmr_unit_sized_p (enum kmr_kv_field data)

static enum kmr_kv_field	kmr_unit_sized_with_unmanaged (enum kmr_kv_field data)

int	kmr_unpairing_fn (const struct kmr_kv_box kv, const KMR_KVS kvs, KMR_KVS kvo, void *p, const long i)

static _Bool	kmr_value_pointer_p (KMR_KVS *kvs)

void	kmr_warning (KMR mr, unsigned int mask, char m)

double	kmr_wtime (void)

Variables
int	kmr_kv_field_bad

int	kmr_kv_field_cstring

int	kmr_kv_field_float8

int	kmr_kv_field_integer

int	kmr_kv_field_opaque

int	kmr_kv_field_pointer_owned

int	kmr_kv_field_pointer_unmanaged

Detailed Description

Utilities Private Part (do not include from applications).

Definition in file kmrimpl.h.

Macro Definition Documentation

#define KMR_ALIGN ( X ) (((X)+((8)-1))&~((8)-1))

Rounds up a given size to the alignment restriction (currently eight bytes).

Definition at line 75 of file kmrimpl.h.

#define KMR_TRACE_FILE_PREFIX "kmrlog"

Prefix to Trace Files.

Definition at line 101 of file kmrimpl.h.

#define xassert ( X )

Value:

((X) \
     ? (void)(0) \
     : (fprintf(stderr, \
                "%s:%d: %s: Assertion '%s' failed.\n", \
                __FILE__, __LINE__, __func__, #X), \
        (void)MPI_Abort(MPI_COMM_WORLD, 1)))

Definition at line 107 of file kmrimpl.h.

#define kmr_malloc ( Z ) kmr_safe_malloc((Z), #Z, __FILE__, __LINE__, __func__)

Allocates memory, or aborts when failed.

Definition at line 177 of file kmrimpl.h.

#define kmr_realloc	(	P,
		Z
	)	kmr_safe_realloc((P), (Z), #Z, __FILE__, __LINE__, __func__)

Allocates memory, or aborts when failed.

Definition at line 181 of file kmrimpl.h.

#define CHECK_ONE_FN_OPTION	(	NAME,
		A,
		B
	)

Value:

if (A . NAME < B . NAME) { \
        char ee[80]; \
        snprintf(ee, sizeof(ee), \
                 "%s() does not support '" # NAME "' option", func); \
        kmr_error(mr, ee); \
    }

Definition at line 949 of file kmrimpl.h.

Typedef Documentation

typedef unsigned short kmr_k_position_t[4]

Positions of node by (X,Y,Z,ABC), with ABC axes collapsed.

Definition at line 126 of file kmrimpl.h.

Function Documentation

static long kmr_tick ( )

inlinestatic

Returns the clock counter value.

(See for well-written information on tick counters: http://www.fftw.org/cycle.h). "__thread" is only needed for an old compiler on K.

Definition at line 135 of file kmrimpl.h.

void kmr_string_truncation	(	KMR *	mr,
		size_t	sz,
		char *	s
	)

Modifies the string end with by "..." for indicating truncation, used on the result of snprintf.

Definition at line 193 of file kmrutil.c.

static struct kmr_kv_box kmr_pick_kv	(	struct kmr_kvs_entry *	e,
		KMR_KVS *	kvs
	)

static

Returns a handle to a key-value entry – a reverse of kmr_poke_kv().

An entry does not know its data-types but they are taken from a key-value stream KVS.

Definition at line 551 of file kmrimpl.h.

static void kmr_poke_kv	(	struct kmr_kvs_entry *	e,
		const struct kmr_kv_box	kv,
		struct kmr_kv_box *	xkv,
		const KMR_KVS *	kvs,
		_Bool	reserve_space_only
	)

inlinestatic

Stores a key-value pair at the entry E in the store – a reverse of kmr_pick_kv().

A key-value pair does not know its data-types but they are taken from a key-value stream. It modifies kmr_kv_box XKV (when non-null) to return the pointer to the opaque field when a key or a value is opaque. It does not move actual data when RESERVE_SPACE_ONLY=1.

Definition at line 599 of file kmrimpl.h.

int kmr_k_node	(	KMR *	mr,
		kmr_k_position_t	p
	)

Gets TOFU position (physical coordinates) of the node.

Errors are fatal (aborts).

Definition at line 441 of file kmrutil.c.

char* kmr_strptr_ff ( char * s )

Returns itself; this is for Fortran-binding.

Definition at line 208 of file kmrutil.c.

int kmr_intstr_ff	(	long	p,
		char *	s,
		int	n
	)

Fills the character array S by the contents at the pointer value integer P by the length N.

It returns the string length in C limited by N.

Definition at line 257 of file kmrutil.c.

unsigned long kmr_fix_bits_endian_ff ( unsigned long b )

Fixes little-endian bits used in Fortran to host-endian.

Definition at line 284 of file kmrutil.c.

int kmr_add_kv_pushoff	(	KMR_KVS *	kvs,
		const struct kmr_kv_box	kv
	)

Adds a key-value pair.

It is called from inside a mutex (OMP critical). It first stores a KV into a buffer, and then sends the buffer when it gets full. It sends an empty message as an end-of-stream marker. IT POLLS MESSAGES TOO OFTEN OR TOO SELDOM.

Definition at line 460 of file kmraltkvs.c.

int kmr_add_kv_done_pushoff ( KMR_KVS * kvs )

Marks finished adding key-value pairs, called from kmr_add_kv_done().

It flushes pending buffers and it is a collective operation in effect.

Definition at line 555 of file kmraltkvs.c.

int kmr_load_properties	(	MPI_Info	info,
		char *	filename
	)

Loads properties into MPI_Info (in Latin1 characters).

It runs only on the main-thread. It returns MPI_SUCCESS normally. It stores only Latin1 strings because MPI_Info does. Refer to the JDK document "java.util.Properties.load()" for the ".properties" format.

Definition at line 1851 of file kmrutil.c.

int kmr_copy_mpi_info	(	MPI_Info	src,
		MPI_Info	dst
	)

Copies contents of MPI_Info.

The destination info is modified.

Definition at line 2238 of file kmrutil.c.

int kmr_dump_mpi_info	(	char *	prefix,
		MPI_Info	info
	)

Dumps simply contents in MPI_Info.

Definition at line 2212 of file kmrutil.c.

void kmr_isort	(	void *	A,
		const size_t	N,
		const size_t	ES,
		int	depth
	)

Sorts by comparator on long integers.

The target array is of "struct kmr_isort_entry". DEPTH tells how deep OMP threading is enabled (using tasks), and it is decremented by one each recursion. Zero depth tells a sequential run.

Definition at line 292 of file kmrisort.c.

void* kmr_bsearch	(	const void *	key,
		const void *	base,
		size_t	nel,
		size_t	size,
		int()(const void , const void *)	compar
	)

Searches a key entry like bsearch(3C), but returns a next greater entry instead of null on no match.

Thus, it may return a pointer ((char *)base+(nel*size)) when a key is larger than all.

Definition at line 565 of file kmrutil.c.

int kmr_assert_sorted	(	KMR_KVS *	kvi,
		_Bool	locally,
		_Bool	shuffling,
		_Bool	ranking
	)

Checks a key-value stream is sorted.

When not LOCALLY, it collects all the entries to rank-zero for checking.

Definition at line 702 of file kmrutil.c.

struct kmr_kvs_entry* kmr_find_kvs_last_entry ( KMR_KVS * kvs )

Finds the last entry of a key-value stream.

It returns null when a key-value stream is empty. It sequentially scans all the entries and slow.

Definition at line 2759 of file kmrbase.c.

int kmr_exchange_sizes	(	KMR *	mr,
		long *	sbuf,
		long *	rbuf
	)

Calls all-to-all to exchange one long-integer.

Definition at line 46 of file kmratoa.c.

int kmr_alltoallv	(	KMR *	mr,
		void *	sbuf,
		long *	scnts,
		long *	sdsps,
		void *	rbuf,
		long *	rcnts,
		long *	rdsps
	)

Does all-to-all-v, but it takes arguments of long-integers.

Setting ATOA_THRESHOLD=0 forces to use MPI all-to-all-v.

Definition at line 116 of file kmratoa.c.

int kmr_gather_sizes	(	KMR *	mr,
		long	siz,
		long *	rbuf
	)

Calls all-gather for collecting one long-integer.

Definition at line 58 of file kmratoa.c.

int kmr_allgatherv	(	KMR *	mr,
		_Bool	rankzeroonly,
		void *	sbuf,
		long	scnt,
		void *	rbuf,
		long *	rcnts,
		long *	rdsps
	)

All-gathers data, or gathers data when RANKZEROONLY.

Definition at line 70 of file kmratoa.c.

int kmr_iogroup_of_node ( KMR * mr )

Returns an I/O-group (an integer key) of a compute node.

Definition at line 106 of file kmrfiles.c.

int kmr_iogroup_of_obd ( int obdidx )

Returns an I/O-group (an integer key) of a disk from an OBDIDX of Lustre file-system.

It uses magic expressions (x=obdidx/2048) and (y=(obdidx%2048)/64).

Definition at line 120 of file kmrfiles.c.

void* kmr_strdup ( char * s )

STRDUP, but aborts on failure.

Definition at line 592 of file kmrutil.c.

FILE* kmr_fopen	(	const char *	n,
		const char *	m
	)

Does fopen, avoiding EINTR.

Definition at line 610 of file kmrutil.c.

int kmr_fgetc ( FILE * f )

Does fgetc, avoiding EINTR.

Definition at line 622 of file kmrutil.c.

int kmr_getdtablesize ( KMR * mr )

Does getdtablesize(); it is defined, because it is not Posix.

Definition at line 635 of file kmrutil.c.

char* kmr_stringify_options ( struct kmr_option o )

Returns a print string of a single option, to check the bits are properly encoded in foreign language interfaces.

Definition at line 361 of file kmrutil.c.

char* kmr_stringify_file_options ( struct kmr_file_option o )

Returns a print string of a single option, to check the bits are properly encoded in foreign language interfaces.

Definition at line 386 of file kmrutil.c.

char* kmr_stringify_spawn_options ( struct kmr_spawn_option o )

Returns a print string of a single option, to check the bits are properly encoded in foreign language interfaces.

Definition at line 405 of file kmrutil.c.

void kmr_ckpt_create_context ( KMR * mr )

Initialize checkpoint context.

This function should be called only once when MapReduce data type is initialized.

Parameters

[in] mr MapReduce data type

Definition at line 119 of file kmrckpt.c.

void kmr_ckpt_free_context ( KMR * mr )

Free checkpoint context.

This function should be called only once when MapReduce data type is freed.

Parameters

[in] mr MapReduce data type

Definition at line 162 of file kmrckpt.c.

int kmr_ckpt_enabled ( KMR * mr )

Check if checkpoint/restart is enabled.

Parameters

[in] mr MapReduce data type

Returns: It returns 1 if checkpoint/restart is enabled. Otherwise it returns 0.

Definition at line 2478 of file kmrckpt.c.

int kmr_ckpt_disable_ckpt ( KMR * mr )

It temporally disables checkpoint/restart.

Parameters

[in] mr MapReduce data type

Returns: If it succeeds disabling, it returns a lock id. Otherwise it returns 0.

Definition at line 2494 of file kmrckpt.c.

int kmr_ckpt_enable_ckpt	(	KMR *	mr,
		int	lock_id
	)

It temporally enables checkpoint/restart which has been disabled by calling kmr_ckpt_disable_ckpt().

Parameters

[in]	mr	MapReduce data type
[in]	lock_id	ID of lock returned by kmr_ckpt_disable_ckpt()

Returns: If it succeeds enabling, it returns 1. Otherwise it returns 0.

Definition at line 2515 of file kmrckpt.c.

void kmr_ckpt_restore_ckpt ( KMR_KVS * kvs )

It restores checkpoint data to kvs.

Parameters

[out] kvs an KVS where the checkpoint data will be restored

Definition at line 2557 of file kmrckpt.c.

void kmr_ckpt_remove_ckpt ( KMR_KVS * kvs )

It removes checkpoint data file.

Parameters

[in] kvs KVS whose checkpoint data is removed

Definition at line 2612 of file kmrckpt.c.

void kmr_ckpt_save_kvo_whole	(	KMR *	mr,
		KMR_KVS *	kvo
	)

It saves all key-value pairs in the output KVS to a checkpoint data file.

Parameters

[in]	kvi	input KVS
[in]	kvo	output KVS

Definition at line 2638 of file kmrckpt.c.

void kmr_ckpt_save_kvo_block_init	(	KMR *	mr,
		KMR_KVS *	kvo
	)

It initializes saving blocks of key-value pairs of the output KVS to a checkpoint data file.

Parameters

[in]	mr	MapReduce data type
[in]	kvo	output KVS

Definition at line 2654 of file kmrckpt.c.

void kmr_ckpt_save_kvo_block_add	(	KMR *	mr,
		KMR_KVS *	kvo,
		long	nkvi
	)

It adds a new block of key-value pairs of the output KVS to the checkpoint data file.

Parameters

[in]	mr	MapReduce data type
[in]	kvo	output KVS
[in]	nkvi	number of processed kv in the input KVS

Definition at line 2670 of file kmrckpt.c.

void kmr_ckpt_save_kvo_block_fin	(	KMR *	mr,
		KMR_KVS *	kvo
	)

It finalizes saving block of key-value pairs of the output KVS to the checkpoint data file.

Parameters

[in]	mr	MapReduce data type
[in]	kvo	output KVS

Definition at line 2685 of file kmrckpt.c.

void kmr_ckpt_save_kvo_each_init	(	KMR *	mr,
		KMR_KVS *	kvo
	)

It initializes saving indexed key-value pairs of the output KVS to a checkpoint data file.

Parameters

[in]	mr	MapReduce data type
[in]	kvo	output KVS

Definition at line 2703 of file kmrckpt.c.

void kmr_ckpt_save_kvo_each_add	(	KMR *	mr,
		KMR_KVS *	kvo,
		long	ikv_index
	)

It adds new key-value pairs of the output KVS to the checkpoint data file.

Parameters

[in]	mr	MapReduce data type
[in]	kvo	output KVS
[in]	ikv_index	index of processed kv in the input KVS

Definition at line 2718 of file kmrckpt.c.

void kmr_ckpt_save_kvo_each_fin	(	KMR *	mr,
		KMR_KVS *	kvo
	)

It finalizes saving indexed key-value pairs of the output KVS to the checkpoint data file.

Parameters

[in]	mr	MapReduce data type
[in]	kvo	output KVS

Definition at line 2733 of file kmrckpt.c.

void kmr_ckpt_lock_start ( KMR * mr )

Define the start position of code region that is referred when restart.

If an execution is stopped due to an error in this region, restart with the different number of processes is not allowed.

Parameters

[in] mr MapReduce data type

Definition at line 1934 of file kmrckpt.c.

void kmr_ckpt_lock_finish ( KMR * mr )

Define the end position of code region that is referred when restart.

If an execution is stopped due to an error in this region, restart with the different number of processes is not allowed.

Parameters

[in] mr MapReduce data type

Definition at line 1945 of file kmrckpt.c.

int kmr_ckpt_progress_init	(	KMR_KVS *	kvi,
		KMR_KVS *	kvo,
		struct kmr_option	opt
	)

It initializes a progress of MapReduce checkpointing.

Parameters

[in]	kvi	input KVS to a MapReduce operation
[in]	kvo	output KVS to the MapReduce operation
[in]	opt	struct kmr_option

Returns: It returns 1 if operation can be skipped. Otherwise it returns 0.

Definition at line 2753 of file kmrckpt.c.

void kmr_ckpt_progress_fin ( KMR * mr )

It finalizes the progress of MapReduce checkpointing.

Parameters

[in] mr MapReduce data type

Definition at line 2845 of file kmrckpt.c.

long kmr_ckpt_first_unprocessed_kv ( KMR * mr )

It returns the index of the first unprocessed key-value in the input KVS.

Parameters

[in] mr MapReduce data type

Returns: It returns the index of the first unprocessed key-value in the input KVS.

Definition at line 2535 of file kmrckpt.c.

Macros

Typedefs

Functions

Variables

Detailed Description

Macro Definition Documentation

Typedef Documentation

Function Documentation