YAZ  5.25.0
Data Structures | Functions
tokenizer.c File Reference

Simple tokenizer system. More...

#include <assert.h>
#include <stdio.h>
#include <string.h>
#include <yaz/log.h>
#include <yaz/wrbuf.h>
#include <yaz/tokenizer.h>

Go to the source code of this file.

Data Structures

struct  yaz_tok_parse
struct  yaz_tok_cfg


void yaz_tok_cfg_single_tokens (yaz_tok_cfg_t t, const char *simple)
yaz_tok_cfg_t yaz_tok_cfg_create (void)
void yaz_tok_cfg_destroy (yaz_tok_cfg_t t)
static int read_buf (void **vp)
yaz_tok_parse_t yaz_tok_parse_buf (yaz_tok_cfg_t t, const char *buf)
static int get_byte (yaz_tok_parse_t tp)
static void unget_byte (yaz_tok_parse_t tp, int ch)
yaz_tok_parse_t yaz_tok_parse_create (yaz_tok_cfg_t t, yaz_tok_get_byte_t h, void *vp)
void yaz_tok_parse_destroy (yaz_tok_parse_t tp)
int yaz_tok_move (yaz_tok_parse_t tp)
const char * yaz_tok_parse_string (yaz_tok_parse_t tp)

Detailed Description

Simple tokenizer system.

Definition in file tokenizer.c.

Function Documentation

◆ get_byte()

static int get_byte ( yaz_tok_parse_t  tp)

◆ read_buf()

static int read_buf ( void **  vp)

Definition at line 71 of file tokenizer.c.

Referenced by yaz_tok_parse_buf().

◆ unget_byte()

static void unget_byte ( yaz_tok_parse_t  tp,
int  ch 

Definition at line 99 of file tokenizer.c.

References yaz_tok_parse::unget_byte.

◆ yaz_tok_cfg_create()

yaz_tok_cfg_t yaz_tok_cfg_create ( void  )

◆ yaz_tok_cfg_destroy()

void yaz_tok_cfg_destroy ( yaz_tok_cfg_t  t)

◆ yaz_tok_cfg_single_tokens()

void yaz_tok_cfg_single_tokens ( yaz_tok_cfg_t  t,
const char *  simple 

◆ yaz_tok_move()

int yaz_tok_move ( yaz_tok_parse_t  tp)

◆ yaz_tok_parse_buf()

yaz_tok_parse_t yaz_tok_parse_buf ( yaz_tok_cfg_t  t,
const char *  buf 

◆ yaz_tok_parse_create()

yaz_tok_parse_t yaz_tok_parse_create ( yaz_tok_cfg_t  t,
yaz_tok_get_byte_t  h,
void *  vp 

◆ yaz_tok_parse_destroy()

void yaz_tok_parse_destroy ( yaz_tok_parse_t  tp)

◆ yaz_tok_parse_string()

const char* yaz_tok_parse_string ( yaz_tok_parse_t  tp)