Implement token pasting of integers.

To do this correctly, we change the lexer to lex integers as string values,
(new token type of INTEGER_STRING), and only convert to integer values when
evaluating an expression value.

Add a new test case for this, (which does pass now).

4 files changed, 33 insertions, 17 deletions

diff --git a/Makefile b/Makefile
index 8811612..0c06aa8 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,7 @@ CFLAGS = -g
 override CFLAGS += -Wall -Wextra -Wwrite-strings -Wswitch-enum -Wno-unused
 
 glcpp: glcpp.o glcpp-lex.o glcpp-parse.o hash_table.o xtalloc.o
-	gcc -o $@ -ltalloc $^
+	gcc -o $@ -ltalloc -lm $^
 
 %.c %.h: %.y
 	bison --debug --defines=$*.h --output=$*.c $^
diff --git a/glcpp-lex.l b/glcpp-lex.l
index d6b7726..70d47d2 100644
--- a/glcpp-lex.l
+++ b/glcpp-lex.l
@@ -88,18 +88,18 @@ HEXADECIMAL_INTEGER	0[xX][0-9a-fA-F]+[uU]?
 }
 
 {DECIMAL_INTEGER} {
-	yylval.ival = strtoll (yytext, NULL, 10);
-	return INTEGER;
+	yylval.str = xtalloc_strdup (yyextra, yytext);
+	return INTEGER_STRING;
 }
 
 {OCTAL_INTEGER} {
-	yylval.ival = strtoll (yytext + 1, NULL, 8);
-	return INTEGER;
+	yylval.str = xtalloc_strdup (yyextra, yytext);
+	return INTEGER_STRING;
 }
 
 {HEXADECIMAL_INTEGER} {
-	yylval.ival = strtoll (yytext + 2, NULL, 16);
-	return INTEGER;
+	yylval.str = xtalloc_strdup (yyextra, yytext);
+	return INTEGER_STRING;
 }
 
 "<<"  {
diff --git a/glcpp-parse.y b/glcpp-parse.y
index d587a4b..5b2d0d3 100644
--- a/glcpp-parse.y
+++ b/glcpp-parse.y
@@ -132,10 +132,10 @@ glcpp_parser_lex_from (glcpp_parser_t *parser, token_list_t *list);
 %parse-param {glcpp_parser_t *parser}
 %lex-param {glcpp_parser_t *parser}
 
-%token COMMA_FINAL DEFINED ELIF_EXPANDED HASH HASH_DEFINE_FUNC HASH_DEFINE_OBJ HASH_ELIF HASH_ELSE HASH_ENDIF HASH_IF HASH_IFDEF HASH_IFNDEF HASH_UNDEF IDENTIFIER IF_EXPANDED INTEGER NEWLINE OTHER PLACEHOLDER SPACE
+%token COMMA_FINAL DEFINED ELIF_EXPANDED HASH HASH_DEFINE_FUNC HASH_DEFINE_OBJ HASH_ELIF HASH_ELSE HASH_ENDIF HASH_IF HASH_IFDEF HASH_IFNDEF HASH_UNDEF IDENTIFIER IF_EXPANDED INTEGER INTEGER_STRING NEWLINE OTHER PLACEHOLDER SPACE
 %token PASTE
 %type <ival> expression INTEGER operator SPACE
-%type <str> IDENTIFIER OTHER
+%type <str> IDENTIFIER INTEGER_STRING OTHER
 %type <string_list> identifier_list
 %type <token> preprocessing_token
 %type <token_list> pp_tokens replacement_list text_line
@@ -253,7 +253,16 @@ control_line:
 ;
 
 expression:
-	INTEGER {
+	INTEGER_STRING {
+		if (strlen ($1) >= 3 && strncmp ($1, "0x", 2) == 0) {
+			$$ = strtoll ($1 + 2, NULL, 16);
+		} else if ($1[0] == '0') {
+			$$ = strtoll ($1, NULL, 8);
+		} else {
+			$$ = strtoll ($1, NULL, 10);
+		}
+	}
+|	INTEGER {
 		$$ = $1;
 	}
 |	expression OR expression {
@@ -372,8 +381,8 @@ preprocessing_token:
 	IDENTIFIER {
 		$$ = _token_create_str (parser, IDENTIFIER, $1);
 	}
-|	INTEGER {
-		$$ = _token_create_ival (parser, INTEGER, $1);
+|	INTEGER_STRING {
+		$$ = _token_create_str (parser, INTEGER_STRING, $1);
 	}
 |	operator {
 		$$ = _token_create_ival (parser, $1, $1);
@@ -710,6 +719,7 @@ _token_print (token_t *token)
 		printf ("%" PRIxMAX, token->value.ival);
 		break;
 	case IDENTIFIER:
+	case INTEGER_STRING:
 	case OTHER:
 		printf ("%s", token->value.str);
 		break;
@@ -828,11 +838,13 @@ _token_paste (token_t *token, token_t *other)
 	/* Two string-valued tokens can usually just be mashed
 	 * together.
 	 *
-	 * XXX: Since our 'OTHER' case is currently so loose, this may
-	 * allow some things thruogh that should be treated as
-	 * errors. */
-	if ((token->type == IDENTIFIER || token->type == OTHER) &&
-	    (other->type == IDENTIFIER || other->type == OTHER))
+	 * XXX: This isn't actually legitimate. Several things here
+	 * should result in a diagnostic since the result cannot be a
+	 * valid, single pre-processing token. For example, pasting
+	 * "123" and "abc" is not legal, but we don't catch that
+	 * here. */
+	if ((token->type == IDENTIFIER || token->type == OTHER || token->type == INTEGER_STRING) &&
+	    (other->type == IDENTIFIER || other->type == OTHER || other->type == INTEGER_STRING))
 	{
 		token->value.str = talloc_strdup_append (token->value.str,
 							 other->value.str);
diff --git a/tests/059-token-pasting-integer.c b/tests/059-token-pasting-integer.c
new file mode 100644
index 0000000..37b895a
--- /dev/null
+++ b/tests/059-token-pasting-integer.c
@@ -0,0 +1,4 @@
+#define paste(x,y) x ## y
+paste(1,2)
+paste(1,000)
+paste(identifier,2)