tokenizer.h File Reference

Header with public definitions about YAZ' tokenizer. More...

#include <yaz/nmem.h>

Go to the source code of this file.

Macros
#define	YAZ_TOK_EOF   0
#define	YAZ_TOK_ERROR   (-1)
#define	YAZ_TOK_STRING   (-2)
#define	YAZ_TOK_QSTRING   (-3)

Typedefs
typedef struct yaz_tok_cfg *	yaz_tok_cfg_t
typedef struct yaz_tok_parse *	yaz_tok_parse_t
typedef int(*	yaz_tok_get_byte_t) (void **vp)

Functions
yaz_tok_cfg_t	yaz_tok_cfg_create (void)
void	yaz_tok_cfg_destroy (yaz_tok_cfg_t t)
void	yaz_tok_cfg_single_tokens (yaz_tok_cfg_t t, const char *simple)
yaz_tok_parse_t	yaz_tok_parse_buf (yaz_tok_cfg_t t, const char *buf)
yaz_tok_parse_t	yaz_tok_parse_create (yaz_tok_cfg_t t, yaz_tok_get_byte_t h, void *vp)
void	yaz_tok_parse_destroy (yaz_tok_parse_t tp)
int	yaz_tok_move (yaz_tok_parse_t tp)
const char *	yaz_tok_parse_string (yaz_tok_parse_t tp)

Detailed Description

Header with public definitions about YAZ' tokenizer.

Definition in file tokenizer.h.

Macro Definition Documentation

YAZ_TOK_EOF

#define YAZ_TOK_EOF   0

Definition at line 38 of file tokenizer.h.

YAZ_TOK_ERROR

#define YAZ_TOK_ERROR   (-1)

Definition at line 39 of file tokenizer.h.

YAZ_TOK_QSTRING

#define YAZ_TOK_QSTRING   (-3)

Definition at line 41 of file tokenizer.h.

YAZ_TOK_STRING

#define YAZ_TOK_STRING   (-2)

Definition at line 40 of file tokenizer.h.

Typedef Documentation

yaz_tok_cfg_t

typedef struct yaz_tok_cfg* yaz_tok_cfg_t

Definition at line 43 of file tokenizer.h.

yaz_tok_get_byte_t

typedef int(* yaz_tok_get_byte_t) (void **vp)

Definition at line 46 of file tokenizer.h.

yaz_tok_parse_t

typedef struct yaz_tok_parse* yaz_tok_parse_t

Definition at line 44 of file tokenizer.h.

Function Documentation

yaz_tok_cfg_create()

yaz_tok_cfg_t yaz_tok_cfg_create

(

void

)

Definition at line 45 of file tokenizer.c.

References yaz_tok_cfg::comment, yaz_tok_cfg::quote_tokens_begin, yaz_tok_cfg::quote_tokens_end, yaz_tok_cfg::ref_count, yaz_tok_cfg::single_tokens, yaz_tok_cfg::white_space, xmalloc, and xstrdup.

Referenced by ccl_qual_add_special(), ccl_qual_field2(), and cql_transform_create().

yaz_tok_cfg_destroy()

void yaz_tok_cfg_destroy

(

yaz_tok_cfg_t

t

)

Definition at line 57 of file tokenizer.c.

References yaz_tok_cfg::comment, yaz_tok_cfg::quote_tokens_begin, yaz_tok_cfg::quote_tokens_end, yaz_tok_cfg::ref_count, yaz_tok_cfg::single_tokens, yaz_tok_cfg::white_space, and xfree.

Referenced by ccl_qual_add_special(), ccl_qual_field2(), cql_transform_close(), and yaz_tok_parse_destroy().

yaz_tok_cfg_single_tokens()

void yaz_tok_cfg_single_tokens	(	yaz_tok_cfg_t	t,
		const char *	simple
	)

Definition at line 39 of file tokenizer.c.

References yaz_tok_cfg::single_tokens, xfree, and xstrdup.

Referenced by ccl_qual_field2(), cql_transform_define_pattern(), and cql_transform_open_FILE().

yaz_tok_move()

int yaz_tok_move

(

yaz_tok_parse_t

tp

)

Definition at line 130 of file tokenizer.c.

References yaz_tok_parse::cfg, yaz_tok_cfg::comment, get_byte(), yaz_tok_parse::look, yaz_tok_cfg::quote_tokens_begin, yaz_tok_cfg::quote_tokens_end, yaz_tok_cfg::single_tokens, unget_byte(), yaz_tok_cfg::white_space, yaz_tok_parse::wr_string, wrbuf_putc, wrbuf_rewind(), YAZ_TOK_EOF, YAZ_TOK_ERROR, YAZ_TOK_QSTRING, and YAZ_TOK_STRING.

Referenced by ccl_qual_add_special(), ccl_qual_field2(), cql_transform_open_FILE(), and cql_transform_parse_tok_line().

yaz_tok_parse_buf()

yaz_tok_parse_t yaz_tok_parse_buf	(	yaz_tok_cfg_t	t,
		const char *	buf
	)

Definition at line 83 of file tokenizer.c.

References read_buf(), and yaz_tok_parse_create().

Referenced by ccl_qual_add_special(), ccl_qual_field2(), cql_transform_define_pattern(), and cql_transform_open_FILE().

yaz_tok_parse_create()

yaz_tok_parse_t yaz_tok_parse_create	(	yaz_tok_cfg_t	t,
		yaz_tok_get_byte_t	h,
		void *	vp
	)

Definition at line 104 of file tokenizer.c.

References yaz_tok_parse::cfg, yaz_tok_parse::get_byte_data, yaz_tok_parse::get_byte_func, yaz_tok_parse::look, yaz_tok_cfg::ref_count, yaz_tok_parse::unget_byte, yaz_tok_parse::wr_string, wrbuf_alloc(), xmalloc, and YAZ_TOK_ERROR.

Referenced by yaz_tok_parse_buf().

yaz_tok_parse_destroy()

void yaz_tok_parse_destroy

(

yaz_tok_parse_t

tp

)

Definition at line 123 of file tokenizer.c.

References yaz_tok_parse::cfg, yaz_tok_parse::wr_string, wrbuf_destroy(), xfree, and yaz_tok_cfg_destroy().

Referenced by ccl_qual_add_special(), ccl_qual_field2(), cql_transform_define_pattern(), and cql_transform_open_FILE().

yaz_tok_parse_string()

const char* yaz_tok_parse_string

(

yaz_tok_parse_t

tp

)

Definition at line 174 of file tokenizer.c.

References yaz_tok_parse::wr_string, and wrbuf_cstr().

Referenced by ccl_qual_add_special(), ccl_qual_field2(), cql_transform_open_FILE(), and cql_transform_parse_tok_line().