icu.h File Reference

ICU utilities. More...

#include <yaz/yconfig.h>
#include <yaz/xmltypes.h>
#include <unicode/utypes.h>

Go to the source code of this file.

Typedefs
typedef struct icu_chain *	yaz_icu_chain_t
	opaque ICU chain
typedef struct icu_iter *	yaz_icu_iter_t
	ICU tokenizer iterator type (opaque)

Functions
void	icu_chain_destroy (yaz_icu_chain_t chain)
	destroys ICU chain
yaz_icu_chain_t	icu_chain_xml_config (const xmlNode *xml_node, int sort, UErrorCode *status)
	constructs ICU chain from XML specification
int	icu_chain_assign_cstr (yaz_icu_chain_t chain, const char *src8cstr, UErrorCode *status)
	pass string to ICU for parsing/tokenization/etc
int	icu_chain_next_token (yaz_icu_chain_t chain, UErrorCode *status)
	returns one token (if any)
int	icu_chain_token_number (yaz_icu_chain_t chain)
	returns token number of last token processed
const char *	icu_chain_token_display (yaz_icu_chain_t chain)
	returns display token of last token processed
const char *	icu_chain_token_norm (yaz_icu_chain_t chain)
	returns normalized token of last token processed
const char *	icu_chain_token_sortkey (yaz_icu_chain_t chain)
	returns sortkey token of last token processed
void	icu_chain_get_org_info (yaz_icu_chain_t chain, size_t *start, size_t *len)
	returns token as it relates to original text (legacy)
void	icu_chain_get_org_info2 (yaz_icu_chain_t chain, size_t *start, size_t *len, const char **cstr)
	returns token as it relates to original text (2nd version)
yaz_icu_iter_t	icu_iter_create (struct icu_chain *chain)
	create ICU tokenizer iterator from chain
void	icu_iter_first (yaz_icu_iter_t iter, const char *src8cstr)
	starts iteration over string
int	icu_iter_next (yaz_icu_iter_t iter)
	iterates over one token
void	icu_iter_destroy (yaz_icu_iter_t iter)
	destroy ICU tokenizer iterator
const char *	icu_iter_get_norm (yaz_icu_iter_t iter)
	returns ICU normalized token
const char *	icu_iter_get_sortkey (yaz_icu_iter_t iter)
	returns ICU sortkey string
const char *	icu_iter_get_display (yaz_icu_iter_t iter)
	returns ICU display string
int	icu_iter_get_token_number (yaz_icu_iter_t iter)
	returns ICU token count for iterator
void	icu_iter_get_org_info (yaz_icu_iter_t iter, size_t *start, size_t *len)
	returns ICU original token start (offset) and length (legacy)
void	icu_iter_get_org_info2 (yaz_icu_iter_t iter, size_t *start, size_t *len, const char **cstr)
	returns ICU original token start (offset) and length

Detailed Description

ICU utilities.

Definition in file icu.h.

Typedef Documentation

yaz_icu_chain_t

typedef struct icu_chain* yaz_icu_chain_t

opaque ICU chain

Definition at line 45 of file icu.h.

yaz_icu_iter_t

typedef struct icu_iter* yaz_icu_iter_t

ICU tokenizer iterator type (opaque)

Definition at line 131 of file icu.h.

Function Documentation

icu_chain_assign_cstr()

int icu_chain_assign_cstr	(	yaz_icu_chain_t	chain,
		const char *	src8cstr,
		UErrorCode *	status
	)

pass string to ICU for parsing/tokenization/etc

Parameters

chain	ICU chain to be used for parsing
src8cstr	input C string (null-terminated)
status	may include ICU error on failure

Return values

0	failure
1	success

icu_chain_destroy()

void icu_chain_destroy

(

yaz_icu_chain_t

chain

)

destroys ICU chain

icu_chain_get_org_info()

void icu_chain_get_org_info	(	yaz_icu_chain_t	chain,
		size_t *	start,
		size_t *	len
	)

returns token as it relates to original text (legacy)

Parameters

chain	ICU chain
start	offset in original text
len	number of uchars in original text

icu_chain_get_org_info2()

void icu_chain_get_org_info2	(	yaz_icu_chain_t	chain,
		size_t *	start,
		size_t *	len,
		const char **	cstr
	)

returns token as it relates to original text (2nd version)

Parameters

chain	ICU chain
start	offset in original text
len	number of uchars in original text
cstr	if not-null, holds original string in there

icu_chain_next_token()

int icu_chain_next_token	(	yaz_icu_chain_t	chain,
		UErrorCode *	status
	)

returns one token (if any)

Parameters

chain	ICU chain
status	may include ICU error on failure

Return values

0	error or end-of-tokens (no more tokens)
>0	token number (1, 3, 3, ..)

This function tries to move to "next" token in assigned C-string .. Or returns 0 if no more is to be found

icu_chain_token_display()

const char * icu_chain_token_display

(

yaz_icu_chain_t

chain

)

returns display token of last token processed

Parameters

chain

ICU chain

Returns

display token string (C string) This function returns display string for last token returned by icu_chain_next_token.

icu_chain_token_norm()

const char * icu_chain_token_norm

(

yaz_icu_chain_t

chain

)

returns normalized token of last token processed

Parameters

chain

ICU chain

Returns

normalized token string (C string) This function returns normalized string for last token returned by icu_chain_next_token.

icu_chain_token_number()

int icu_chain_token_number

(

yaz_icu_chain_t

chain

)

returns token number of last token processed

chain ICU chain

Returns

token number (numbered from 1)

icu_chain_token_sortkey()

const char * icu_chain_token_sortkey

(

yaz_icu_chain_t

chain

)

returns sortkey token of last token processed

Parameters

chain

ICU chain

Returns

sortkey token string (C string) This function returns sortkey string for last token returned by icu_chain_next_token.

icu_chain_xml_config()

yaz_icu_chain_t icu_chain_xml_config	(	const xmlNode *	xml_node,
		int	sort,
		UErrorCode *	status
	)

constructs ICU chain from XML specification

Parameters

xml_node	icu_chain XML node - with attribute locale in it
sort	1 if ICU chain is to deal with sort keys; 0 otherwise
status	May include ICU error code on failure

Returns

chain ptr or NULL on failure in which case status may hold info

icu_iter_create()

yaz_icu_iter_t icu_iter_create

(

struct icu_chain *

chain

)

create ICU tokenizer iterator from chain

Parameters

chain

ICU chain

Returns

ICU iterator

icu_iter_destroy()

void icu_iter_destroy

(

yaz_icu_iter_t

iter

)

destroy ICU tokenizer iterator

Parameters

iter	ICU tokenizer iterator

icu_iter_first()

void icu_iter_first	(	yaz_icu_iter_t	iter,
		const char *	src8cstr
	)

starts iteration over string

Parameters

iter	ICU tokenizer iterator
src8cstr	input string (0-terminated)

Call icu_iter_next to iterate over each token.

icu_iter_get_display()

const char * icu_iter_get_display

(

yaz_icu_iter_t

iter

)

returns ICU display string

Parameters

iter	ICU tokenizer iterator

Returns

string (0-terminated)

icu_iter_get_norm()

const char * icu_iter_get_norm

(

yaz_icu_iter_t

iter

)

returns ICU normalized token

Parameters

iter	ICU tokenizer iterator

Returns

string (0-terminated)

icu_iter_get_org_info()

void icu_iter_get_org_info	(	yaz_icu_iter_t	iter,
		size_t *	start,
		size_t *	len
	)

returns ICU original token start (offset) and length (legacy)

Parameters

iter	ICU tokenizer iterator
start	offset of last token in original text
len	length of last token in original text

icu_iter_get_org_info2()

void icu_iter_get_org_info2	(	yaz_icu_iter_t	iter,
		size_t *	start,
		size_t *	len,
		const char **	cstr
	)

returns ICU original token start (offset) and length

Parameters

iter	ICU tokenizer iterator
start	offset of last token in original text
len	length of last token in original text
cstr	if non-null: original string

icu_iter_get_sortkey()

const char * icu_iter_get_sortkey

(

yaz_icu_iter_t

iter

)

returns ICU sortkey string

Parameters

iter	ICU tokenizer iterator

Returns

string (0-terminated)

icu_iter_get_token_number()

int icu_iter_get_token_number

(

yaz_icu_iter_t

iter

)

returns ICU token count for iterator

Parameters

iter	ICU tokenizer iterator

Returns

token count (1, 2, 3...)

icu_iter_next()

int icu_iter_next

(

yaz_icu_iter_t

iter

)

iterates over one token

Parameters

iter	ICU tokenizer iterator

Return values

0	no more tokens (EOF)
1	got one token (use icu_iter_get..-functions)