tokenizer.c
Go to the documentation of this file.
1 
15 #include <GKlib.h>
16 
17 
18 /************************************************************************
19 * This function tokenizes a string based on the user-supplied delimiters
20 * list. The resulting tokens are returned into an array of strings.
21 *************************************************************************/
22 void gk_strtokenize(char *str, char *delim, gk_Tokens_t *tokens)
23 {
24  int i, ntoks, slen;
25 
26  tokens->strbuf = gk_strdup(str);
27 
28  slen = strlen(str);
29  str = tokens->strbuf;
30 
31  /* Scan once to determine the number of tokens */
32  for (ntoks=0, i=0; i<slen;) {
33  /* Consume all the consecutive characters from the delimiters list */
34  while (i<slen && strchr(delim, str[i]))
35  i++;
36 
37  if (i == slen)
38  break;
39 
40  ntoks++;
41 
42  /* Consume all the consecutive characters from the token */
43  while (i<slen && !strchr(delim, str[i]))
44  i++;
45  }
46 
47 
48  tokens->ntoks = ntoks;
49  tokens->list = (char **)gk_malloc(ntoks*sizeof(char *), "strtokenize: tokens->list");
50 
51 
52  /* Scan a second time to mark and link the tokens */
53  for (ntoks=0, i=0; i<slen;) {
54  /* Consume all the consecutive characters from the delimiters list */
55  while (i<slen && strchr(delim, str[i]))
56  str[i++] = '\0';
57 
58  if (i == slen)
59  break;
60 
61  tokens->list[ntoks++] = str+i;
62 
63  /* Consume all the consecutive characters from the token */
64  while (i<slen && !strchr(delim, str[i]))
65  i++;
66  }
67 }
68 
69 
70 /************************************************************************
71 * This function frees the memory associated with a gk_Tokens_t
72 *************************************************************************/
74 {
75  gk_free((void *)&tokens->list, &tokens->strbuf, LTERM);
76 }
77 
char * strbuf
Definition: gk_struct.h:141
Definition: pytypes.h:1403
char * gk_strdup(char *orgstr)
Duplicates a string.
Definition: string.c:372
void * gk_malloc(size_t nbytes, char *msg)
Definition: memory.c:140
void gk_free(void **ptr1,...)
Definition: memory.c:202
char ** list
Definition: gk_struct.h:142
void gk_strtokenize(char *str, char *delim, gk_Tokens_t *tokens)
Definition: tokenizer.c:22
void gk_freetokenslist(gk_Tokens_t *tokens)
Definition: tokenizer.c:73
#define LTERM
Definition: gk_defs.h:14


gtsam
Author(s):
autogenerated on Tue Jul 4 2023 02:40:21