Perform "re lexing" on string list values rathern than on text.

Previously, we would pass original strings back to the original lexer
whenever we needed to re-lex something, (such as an expanded macro or
a macro argument). Now, we instead parse the macro or argument
originally to a string list, and then re-lex by simply returning each
string from this list in turn.

We do this in the recently added glcpp_parser_lex function that sits
on top of the lower-level glcpp_lex that only deals with text.

This doesn't change any behavior (at least according to the existing
test suite which all still passes) but it brings us much closer to
being able to "finalize" an unexpanded macro as required by the
specification.

3 files changed, 124 insertions, 133 deletions

diff --git a/glcpp-lex.l b/glcpp-lex.l
index 52be1b1..aec9679 100644
--- a/glcpp-lex.l
+++ b/glcpp-lex.l
@@ -84,12 +84,12 @@ TOKEN		[^[:space:](),]+
 
 <ST_DEFINE_OBJ_OR_FUNC>\n {
 	BEGIN INITIAL;
-	yylval.str = xtalloc_strdup (yyextra, "");
-	return REPLACEMENT;
+	return NEWLINE;
 }
 
 <ST_DEFINE_OBJ_OR_FUNC>{HSPACE}+ {
 	BEGIN ST_DEFINE_VALUE;
+	return SPACE;
 }
 
 <ST_DEFINE_OBJ_OR_FUNC>"(" {
@@ -113,10 +113,21 @@ TOKEN		[^[:space:](),]+
 
 <ST_DEFINE_PARAMETER>{HSPACE}+
 
-<ST_DEFINE_VALUE>.*\n {
+<ST_DEFINE_VALUE>{TOKEN} {
+	yylval.str = xtalloc_strdup (yyextra, yytext);
+	return TOKEN;
+}
+
+<ST_DEFINE_VALUE>[(),] {
+	yylval.str = xtalloc_strdup (yyextra, yytext);
+	return TOKEN;
+}
+
+<ST_DEFINE_VALUE>{HSPACE}+
+
+<ST_DEFINE_VALUE>\n {
 	BEGIN INITIAL;
-	yylval.str = xtalloc_strndup (yyextra, yytext, strlen (yytext) - 1);
-	return REPLACEMENT;
+	return NEWLINE;
 }
 
 {IDENTIFIER} {
@@ -161,54 +172,4 @@ TOKEN		[^[:space:](),]+
 
 {HSPACE}+
 
-<<EOF>> {
-	int done;
-
-	done = glcpp_lex_stack_pop (yyextra->lex_stack);
-
-	if (done)
-		yyterminate ();
-
-	glcpp_parser_pop_expansion (yyextra);
-}
-
 %%
-
-void
-glcpp_lex_stack_push (glcpp_lex_stack_t *stack, const char *string)
-{
-	struct yyguts_t *yyg = (struct yyguts_t*) stack->parser->scanner;
-	glcpp_lex_node_t *node;
-
-	/* Save the current buffer on the top of the stack. */
-	node = xtalloc (stack, glcpp_lex_node_t);
-	node->buffer = YY_CURRENT_BUFFER;
-
-	node->next = stack->head;
-	stack->head = node;
-
-	/* Then switch to a new scan buffer for string. */
-	yy_scan_string (string, stack->parser->scanner);
-}
-
-int
-glcpp_lex_stack_pop (glcpp_lex_stack_t *stack)
-{
-	struct yyguts_t *yyg = (struct yyguts_t*) stack->parser->scanner;
-	glcpp_lex_node_t *node;
-
-	node = stack->head;
-
-	if (node == NULL)
-		return 1;
-
-	stack->head = node->next;
-
-	yy_delete_buffer (YY_CURRENT_BUFFER, stack->parser->scanner);
-	yy_switch_to_buffer ((YY_BUFFER_STATE) node->buffer,
-			     stack->parser->scanner);
-
-	talloc_free (node);
-
-	return 0;
-}
diff --git a/glcpp-parse.y b/glcpp-parse.y
index bb57b30..2383c93 100644
--- a/glcpp-parse.y
+++ b/glcpp-parse.y
@@ -34,13 +34,13 @@ yyerror (void *scanner, const char *error);
 void
 _define_object_macro (glcpp_parser_t *parser,
 		      const char *macro,
-		      const char *replacement);
+		      string_list_t *replacements);
 
 void
 _define_function_macro (glcpp_parser_t *parser,
 			const char *macro,
 			string_list_t *parameters,
-			const char *replacement);
+			string_list_t *replacements);
 
 void
 _expand_object_macro (glcpp_parser_t *parser, const char *identifier);
@@ -80,6 +80,14 @@ _argument_list_length (argument_list_t *list);
 string_list_t *
 _argument_list_member_at (argument_list_t *list, int index);
 
+static void
+glcpp_parser_push_expansion_macro (glcpp_parser_t *parser,
+				   macro_t *macro,
+				   argument_list_t *arguments);
+
+static void
+glcpp_parser_pop_expansion (glcpp_parser_t *parser);
+
 #define yylex glcpp_parser_lex
 
 static int
@@ -96,9 +104,9 @@ glcpp_parser_lex (glcpp_parser_t *parser);
 %parse-param {glcpp_parser_t *parser}
 %lex-param {glcpp_parser_t *parser}
 
-%token DEFINE FUNC_MACRO IDENTIFIER OBJ_MACRO REPLACEMENT TOKEN UNDEF
-%type <str> argument_word FUNC_MACRO IDENTIFIER OBJ_MACRO REPLACEMENT TOKEN
-%type <string_list> argument macro parameter_list
+%token DEFINE FUNC_MACRO IDENTIFIER OBJ_MACRO NEWLINE SPACE TOKEN UNDEF
+%type <str> argument_word FUNC_MACRO IDENTIFIER OBJ_MACRO TOKEN
+%type <string_list> argument macro parameter_list replacement_list pp_tokens
 %type <argument_list> argument_list
 
 /* Hard to remove shift/reduce conflicts documented as follows:
@@ -194,10 +202,14 @@ argument_word:
 
 
 directive:
-	DEFINE IDENTIFIER REPLACEMENT {
-		_define_object_macro (parser, $2, $3);
+	DEFINE IDENTIFIER NEWLINE {
+		string_list_t *list = _string_list_create (parser);
+		_define_object_macro (parser, $2, list);
 	}
-|	DEFINE IDENTIFIER '(' parameter_list ')' REPLACEMENT {
+|	DEFINE IDENTIFIER SPACE replacement_list NEWLINE {
+		_define_object_macro (parser, $2, $4);
+	}
+|	DEFINE IDENTIFIER '(' parameter_list ')' replacement_list NEWLINE {
 		_define_function_macro (parser, $2, $4, $6);
 	}
 |	UNDEF IDENTIFIER {
@@ -229,6 +241,27 @@ parameter_list:
 	}
 ;
 
+replacement_list:
+	/* empty */ {
+		$$ = _string_list_create (parser);
+	}
+|	pp_tokens {
+		$$ = $1;
+	}
+;
+
+
+pp_tokens:
+	TOKEN {
+		$$ = _string_list_create (parser);
+		_string_list_append_item ($$, $1);
+	}
+|	pp_tokens TOKEN {
+		_string_list_append_item ($1, $2);
+		$$ = $1;
+	}
+;
+
 %%
 
 string_list_t *
@@ -413,10 +446,6 @@ glcpp_parser_create (void)
 					   hash_table_string_compare);
 	parser->expansions = NULL;
 
-	parser->lex_stack = xtalloc (parser, glcpp_lex_stack_t);
-	parser->lex_stack->parser = parser;
-	parser->lex_stack->head = NULL;
-
 	return parser;
 }
 
@@ -495,7 +524,7 @@ glcpp_parser_classify_token (glcpp_parser_t *parser,
 void
 _define_object_macro (glcpp_parser_t *parser,
 		      const char *identifier,
-		      const char *replacement)
+		      string_list_t *replacements)
 {
 	macro_t *macro;
 
@@ -504,7 +533,7 @@ _define_object_macro (glcpp_parser_t *parser,
 	macro->is_function = 0;
 	macro->parameters = NULL;
 	macro->identifier = talloc_strdup (macro, identifier);
-	macro->replacement = talloc_steal (macro, replacement);
+	macro->replacements = talloc_steal (macro, replacements);
 
 	hash_table_insert (parser->defines, macro, identifier);
 }
@@ -513,7 +542,7 @@ void
 _define_function_macro (glcpp_parser_t *parser,
 			const char *identifier,
 			string_list_t *parameters,
-			const char *replacement)
+			string_list_t *replacements)
 {
 	macro_t *macro;
 
@@ -522,7 +551,7 @@ _define_function_macro (glcpp_parser_t *parser,
 	macro->is_function = 1;
 	macro->parameters = talloc_steal (macro, parameters);
 	macro->identifier = talloc_strdup (macro, identifier);
-	macro->replacement = talloc_steal (macro, replacement);
+	macro->replacements = talloc_steal (macro, replacements);
 
 	hash_table_insert (parser->defines, macro, identifier);
 }
@@ -531,7 +560,7 @@ static void
 _glcpp_parser_push_expansion_internal (glcpp_parser_t *parser,
 				       macro_t *macro,
 				       argument_list_t *arguments,
-				       const char * replacement)
+				       string_node_t *replacements)
 {
 	expansion_node_t *node;
 
@@ -539,20 +568,19 @@ _glcpp_parser_push_expansion_internal (glcpp_parser_t *parser,
 
 	node->macro = macro;
 	node->arguments = arguments;
+	node->replacements = replacements;
 
 	node->next = parser->expansions;
 	parser->expansions = node;
-		
-	glcpp_lex_stack_push (parser->lex_stack, replacement);
 }
 
-void
+static void
 glcpp_parser_push_expansion_macro (glcpp_parser_t *parser,
 				   macro_t *macro,
 				   argument_list_t *arguments)
 {
 	_glcpp_parser_push_expansion_internal (parser, macro, arguments,
-					       macro->replacement);
+					       macro->replacements->head);
 }
 
 void
@@ -561,38 +589,16 @@ glcpp_parser_push_expansion_argument (glcpp_parser_t *parser,
 {
 	argument_list_t *arguments;
 	string_list_t *argument;
-	string_node_t *node;
-	char *argument_str, *s;
-	int length;
 
 	arguments = parser->expansions->arguments;
 
 	argument = _argument_list_member_at (arguments, argument_index);
 
-	length = 0;
-	for (node = argument->head; node; node = node->next)
-		length += strlen (node->str) + 1;
-
-	argument_str = xtalloc_size (parser, length);
-
-	*argument_str = '\0';
-	s = argument_str;
-	for (node = argument->head; node; node = node->next) {
-		strcpy (s, node->str);
-		s += strlen (node->str);
-		if (node->next) {
-			*s = ' ';
-			s++;
-			*s = '\0';
-		}
-	}
-
 	_glcpp_parser_push_expansion_internal (parser, NULL, NULL,
-					       argument_str);
+					       argument->head);
 }
 
-/* The lexer calls this when it exhausts a string. */
-void
+static void
 glcpp_parser_pop_expansion (glcpp_parser_t *parser)
 {
 	expansion_node_t *node;
@@ -649,5 +655,55 @@ _expand_function_macro (glcpp_parser_t *parser,
 static int
 glcpp_parser_lex (glcpp_parser_t *parser)
 {
-	return glcpp_lex (parser->scanner);
+	expansion_node_t *expansion;
+	string_node_t *replacements;
+	int parameter_index;
+
+    /* Who says C can't do efficient tail recursion? */
+    RECURSE:
+
+	expansion = parser->expansions;
+
+	if (expansion == NULL)
+		return glcpp_lex (parser->scanner);
+
+	replacements = expansion->replacements;
+
+	/* Pop expansion when replacements is exhausted. */
+	if (replacements == NULL) {
+		glcpp_parser_pop_expansion (parser);
+		goto RECURSE;
+	}
+
+	expansion->replacements = replacements->next;
+
+	if (strcmp (replacements->str, "(") == 0)
+		return '(';
+	else if (strcmp (replacements->str, ")") == 0)
+		return ')';
+	else if (strcmp (replacements->str, ",") == 0)
+		return ',';
+
+	yylval.str = xtalloc_strdup (parser, replacements->str);
+
+	switch (glcpp_parser_classify_token (parser, yylval.str,
+					     &parameter_index))
+	{
+	case TOKEN_CLASS_ARGUMENT:
+		talloc_free (yylval.str);
+		glcpp_parser_push_expansion_argument (parser,
+						      parameter_index);
+		goto RECURSE;
+		break;
+	case TOKEN_CLASS_IDENTIFIER:
+		return IDENTIFIER;
+		break;
+	case TOKEN_CLASS_FUNC_MACRO:
+		return FUNC_MACRO;
+		break;
+	default:
+	case TOKEN_CLASS_OBJ_MACRO:
+		return OBJ_MACRO;
+		break;
+	}
 }
diff --git a/glcpp.h b/glcpp.h
index 6aabf6f..ef821a7 100644
--- a/glcpp.h
+++ b/glcpp.h
@@ -54,24 +54,6 @@ typedef struct argument_list {
 
 typedef struct glcpp_parser glcpp_parser_t;
 
-/* Support for temporarily lexing/parsing tokens from a string. */
-
-typedef struct glcpp_lex_node {
-	void *buffer;
-	struct glcpp_lex_node *next;
-} glcpp_lex_node_t;
-
-typedef struct {
-	glcpp_parser_t *parser;
-	glcpp_lex_node_t *head;
-} glcpp_lex_stack_t;
-
-void
-glcpp_lex_stack_push (glcpp_lex_stack_t *stack, const char *string);
-
-int
-glcpp_lex_stack_pop (glcpp_lex_stack_t *stack);
-
 typedef enum {
 	TOKEN_CLASS_ARGUMENT,
 	TOKEN_CLASS_IDENTIFIER,
@@ -88,12 +70,13 @@ typedef struct {
 	int is_function;
 	string_list_t *parameters;
 	const char *identifier;
-	const char *replacement;
+	string_list_t *replacements;
 } macro_t;
 
 typedef struct expansion_node {
 	macro_t *macro;
 	argument_list_t *arguments;
+	string_node_t *replacements;
 	struct expansion_node *next;
 } expansion_node_t;
 
@@ -101,9 +84,12 @@ struct glcpp_parser {
 	yyscan_t scanner;
 	struct hash_table *defines;
 	expansion_node_t *expansions;
-	glcpp_lex_stack_t *lex_stack;
 };
 
+void
+glcpp_parser_push_expansion_argument (glcpp_parser_t *parser,
+				      int argument_index);
+
 glcpp_parser_t *
 glcpp_parser_create (void);
 
@@ -113,18 +99,6 @@ glcpp_parser_parse (glcpp_parser_t *parser);
 void
 glcpp_parser_destroy (glcpp_parser_t *parser);
 
-void
-glcpp_parser_push_expansion_macro (glcpp_parser_t *parser,
-				   macro_t *macro,
-				   argument_list_t *arguments);
-
-void
-glcpp_parser_push_expansion_argument (glcpp_parser_t *parser,
-				      int argument_index);
-
-void
-glcpp_parser_pop_expansion (glcpp_parser_t *parser);
-
 /* Generated by glcpp-lex.l to glcpp-lex.c */
 
 int