27 #include <unicode/ustring.h>
28 #include <unicode/uchar.h>
49 static void icu_tokenizer_reset(
struct icu_tokenizer *tokenizer,
52 tokenizer->action = action;
55 tokenizer->token_count = 0;
56 tokenizer->token_id = 0;
57 tokenizer->token_start = 0;
58 tokenizer->token_end = 0;
64 int32_t bufferSize = U_BRK_SAFECLONE_BUFFERSIZE;
65 UErrorCode status = U_ZERO_ERROR;
66 struct icu_tokenizer * tokenizer
67 = (
struct icu_tokenizer *)
xmalloc(
sizeof(
struct icu_tokenizer));
70 icu_tokenizer_reset(tokenizer, old->action);
72 tokenizer->bi = ubrk_safeClone(old->bi, NULL, &bufferSize, &status);
73 if (U_SUCCESS(status))
81 struct icu_tokenizer *tokenizer
82 = (
struct icu_tokenizer *)
xmalloc(
sizeof(
struct icu_tokenizer));
84 icu_tokenizer_reset(tokenizer, action);
85 switch (tokenizer->action)
89 tokenizer->bi = ubrk_open(UBRK_LINE, locale, 0, 0, status);
93 tokenizer->bi = ubrk_open(UBRK_SENTENCE, locale, 0, 0, status);
97 tokenizer->bi = ubrk_open(UBRK_WORD, locale, 0, 0, status);
101 tokenizer->bi = ubrk_open(UBRK_CHARACTER, locale, 0, 0, status);
105 tokenizer->bi = ubrk_open(UBRK_TITLE, locale, 0, 0, status);
108 *status = U_UNSUPPORTED_ERROR;
114 if (U_SUCCESS(*status))
128 ubrk_close(tokenizer->bi);
137 if (!tokenizer || !tokenizer->bi || !src16)
142 tokenizer->token_count = 0;
143 tokenizer->token_id = 0;
144 tokenizer->token_start = 0;
145 tokenizer->token_end = 0;
147 ubrk_setText(tokenizer->bi,
148 tokenizer->buf16->utf16, tokenizer->buf16->utf16_len, status);
150 if (U_FAILURE(*status))
159 size_t *start,
size_t *len)
161 int32_t tkn_start = 0;
165 if (!tokenizer || !tokenizer->bi
166 || !tokenizer->buf16 || !tokenizer->buf16->utf16_len)
176 if (0 == tokenizer->token_end)
177 tkn_start = ubrk_first(tokenizer->bi);
179 tkn_start = tokenizer->token_end;
182 tkn_end = ubrk_next(tokenizer->bi);
185 if (UBRK_DONE == tkn_end)
186 tkn_end = tokenizer->buf16->utf16_len;
189 if (U_FAILURE(*status))
193 tkn_len = tkn_end - tkn_start;
197 tokenizer->token_count++;
198 tokenizer->token_id++;
201 tokenizer->token_id = 0;
203 tokenizer->token_start = tkn_start;
204 tokenizer->token_end = tkn_end;
207 *len = tkn_end - tkn_start;
215 u_strncpy(tkn16->
utf16, &(tokenizer->buf16->utf16)[tkn_start],
226 return tokenizer->token_count;
Internal header for ICU utilities.
struct icu_buf_utf16 * icu_buf_utf16_copy(struct icu_buf_utf16 *dest16, const struct icu_buf_utf16 *src16)
struct icu_tokenizer * icu_tokenizer_create(const char *locale, char action, UErrorCode *status)
struct icu_tokenizer * icu_tokenizer_clone(struct icu_tokenizer *old)
int32_t icu_tokenizer_token_count(struct icu_tokenizer *tokenizer)
int icu_tokenizer_attach(struct icu_tokenizer *tokenizer, struct icu_buf_utf16 *src16, UErrorCode *status)
void icu_buf_utf16_destroy(struct icu_buf_utf16 *buf16)
void icu_tokenizer_destroy(struct icu_tokenizer *tokenizer)
struct icu_buf_utf16 * icu_buf_utf16_create(size_t capacity)
struct icu_buf_utf16 * icu_buf_utf16_resize(struct icu_buf_utf16 *buf16, size_t capacity)
int32_t icu_tokenizer_next_token(struct icu_tokenizer *tokenizer, struct icu_buf_utf16 *tkn16, UErrorCode *status, size_t *start, size_t *len)
Header for memory handling functions.
#define xfree(x)
utility macro which calls xfree_f
#define xmalloc(x)
utility macro which calls malloc_f