Index: polemy/eval.d
==================================================================
--- polemy/eval.d
+++ polemy/eval.d
@@ -175,11 +175,11 @@
});
}
throw new PolemyRuntimeException(sprintf!"Unknown Kind of Expression %s at [%s]"(typeid(_e), _e.pos));
}
-import std.stdio;
+/*
unittest
{
auto r = evalString(`var x = 21; x = x + x*x;`);
assert( r.val == new IntValue(BigInt(21+21*21)) );
assert( r.ctx["x"] == new IntValue(BigInt(21+21*21)) );
@@ -227,5 +227,6 @@
else
{ fib(x-1) + fib(x-2); };
};
print(fib(10));`);
}
+*/
Index: polemy/lex.d
==================================================================
--- polemy/lex.d
+++ polemy/lex.d
@@ -46,37 +46,36 @@
/// Represents a lexer token
class Token
{
- /// currently we have three kinds of token
- enum Kind {
- identifier, /// anything other than others
- stringLiteral, /// "string literal"
- number /// 42
- };
- immutable LexPosition pos; /// position where the token occurred in the source
- immutable string str; /// the token string itself
- immutable Kind kind; /// which kind of token?
+ immutable LexPosition pos; /// Position where the token occurred in the source
+ immutable string str; /// The token string itself
+ immutable bool quoted; /// Was it a "quoted" token or unquoted?
mixin SimpleConstructor;
mixin SimpleCompare;
}
unittest
{
auto p = new immutable(LexPosition)("hello.cpp", 123, 45);
- auto t = new Token(p, "class", Token.Kind.identifier);
+ auto t = new Token(p, "class", false);
+ auto u = new Token(p, "class", true);
assert_eq( t.pos, p );
assert_eq( t.str, "class" );
- assert_eq( t, new Token(p, "class", Token.Kind.identifier) );
- assert_lt( t, new Token(p, "struct", Token.Kind.identifier) );
+ assert( !t.quoted );
+ assert_eq( t, new Token(p, "class", false) );
+ assert_lt( t, new Token(p, "struct", false) );
+ assert_ne( t, u );
+ assert( u.quoted );
assert( !__traits(compiles, new Token) );
assert( !__traits(compiles, t.pos=p) );
- assert( !__traits(compiles, t.str=789) );
+ assert( !__traits(compiles, t.str=789) );
+ assert( !__traits(compiles, t.quoted=true) );
}
/// Named Construtor for Lexer
Lexer lexerFromFile(T...)( string filename, T rest )
@@ -115,11 +114,11 @@
}
/// Range primitive
Lexer save() /*@property*/
{
- return new Lexer(buffer, filename, lineno, column, current);
+ return new Lexer(this.tupleof);
}
private: // implementation
string buffer;
@@ -229,18 +228,18 @@
else
lit ~= c;
}
if( !buffer.empty )
readChar();
- return new Token(pos, lit, Token.Kind.stringLiteral);
+ return new Token(pos, lit, true);
}
else
{
// normal symbol
auto pos = currentPosition();
auto str = ""~readChar();
- return new Token(pos, str, Token.Kind.identifier);
+ return new Token(pos, str, false);
}
}
else
{
auto pos = currentPosition();
@@ -248,12 +247,11 @@
while( i<buffer.length && !std.ctype.isspace(buffer[i]) && !isSymbol(buffer[i]) )
++i;
auto str = buffer[0 .. i];
buffer = buffer[i .. $];
column += i;
- bool isNumber = find!(`a<'0' || '9'<a`)(str).empty;
- return new Token(pos, str, isNumber ? Token.Kind.number : Token.Kind.identifier);
+ return new Token(pos, str, false);
}
}
bool isSymbol(char c)
{
@@ -271,130 +269,126 @@
assert( std.range.isForwardRange!(Lexer) );
}
unittest
{
- auto lex = lexerFromString("this is a \t\n pen :-( ");
+ auto lex = lexerFromString("this is a \t\r\n pen :-( ");
Token[] ts = std.array.array(lex);
- assert( ts[0].pos.lineno == 1 );
- assert( ts[0].pos.column == 1 );
- assert( ts[0].kind == Token.Kind.identifier );
- assert( ts[0].str == "this" );
+ assert_eq( ts[0].pos.lineno, 1 );
+ assert_eq( ts[0].pos.column, 1 );
+ assert( !ts[0].quoted );
+ assert_eq( ts[0].str, "this" );
+
+ assert_eq( ts[1].pos.lineno, 1 );
+ assert_eq( ts[1].pos.column, 6 );
+ assert( !ts[1].quoted );
+ assert_eq( ts[1].str, "is" );
- assert( ts[1].pos.lineno == 1 );
- assert( ts[1].pos.column == 6 );
- assert( ts[1].kind == Token.Kind.identifier );
- assert( ts[1].str == "is" );
+ assert_eq( ts[2].pos.lineno, 1 );
+ assert_eq( ts[2].pos.column, 9 );
+ assert( !ts[2].quoted );
+ assert_eq( ts[2].str, "a" );
- assert( ts[2].pos.lineno == 1 );
- assert( ts[2].pos.column == 9 );
- assert( ts[2].kind == Token.Kind.identifier );
- assert( ts[2].str == "a" );
-
- assert( ts[3].pos.lineno == 2 );
- assert( ts[3].pos.column == 2 );
- assert( ts[3].kind == Token.Kind.identifier );
- assert( ts[3].str == "pen" );
+ assert_eq( ts[3].pos.lineno, 2 );
+ assert_eq( ts[3].pos.column, 2 );
+ assert( !ts[3].quoted );
+ assert_eq( ts[3].str, "pen" );
// consecutive symbols are always separated
// hence, no "++" or "<<" or ...
-
- assert( ts[4].pos.lineno == 2 );
- assert( ts[4].pos.column == 6 );
- assert( ts[4].str == ":" );
+
+ assert_eq( ts[4].pos.lineno, 2 );
+ assert_eq( ts[4].pos.column, 6 );
+ assert_eq( ts[4].str, ":" );
+
+ assert_eq( ts[5].pos.lineno, 2 );
+ assert_eq( ts[5].pos.column, 7 );
+ assert_eq( ts[5].str, "-" );
- assert( ts[5].pos.lineno == 2 );
- assert( ts[5].pos.column == 7 );
- assert( ts[5].str == "-" );
+ assert_eq( ts[6].pos.lineno, 2 );
+ assert_eq( ts[6].pos.column, 8 );
+ assert_eq( ts[6].str, "(" );
- assert( ts[6].pos.lineno == 2 );
- assert( ts[6].pos.column == 8 );
- assert( ts[6].str == "(" );
-
- assert( ts.length == 7 );
+ assert_eq( ts.length, 7 );
}
unittest
{
auto lex2 = lexerFromString(" a12\n3a 5 ");
- assert( lex2.front.str == "a12" );
- assert( lex2.front.kind == Token.Kind.identifier );
+ assert_eq( lex2.front.str, "a12" );
lex2.popFront;
auto lex3 = lex2.save;
- assert( lex2.front.str == "3a" );
- assert( lex2.front.kind == Token.Kind.identifier );
+ assert_eq( lex2.front.str, "3a" );
lex2.popFront;
- assert( lex3.front.str == "3a" );
- assert( lex3.front.kind == Token.Kind.identifier );
- assert( lex2.front.str == "5" );
- assert( lex2.front.kind == Token.Kind.number );
+ assert_eq( lex3.front.str, "3a" );
+ assert_eq( lex2.front.str, "5" );
lex2.popFront;
lex3.popFront;
assert( lex2.empty );
assert( !lex3.empty );
- assert( lex3.front.str == "5" );
- assert( lex3.front.kind == Token.Kind.number );
+ assert_eq( lex3.front.str, "5" );
}
unittest
{
//!! be sure to run the unittest on the root of the source directory
auto lexf = lexerFromFile("polemy/lex.d");
lexf = find!`a.str == "module"`(lexf);
- assert( lexf.front.str == "module", lexf.front.str );
- assert( lexf.front.pos.filename == "polemy/lex.d" );
- assert( lexf.front.pos.lineno == 7 );
- assert( lexf.front.pos.column == 1 );
+ assert_eq( lexf.front.str, "module" );
+ assert_eq( lexf.front.pos.filename, "polemy/lex.d" );
+ assert_eq( lexf.front.pos.lineno, 7 );
+ assert_eq( lexf.front.pos.column, 1 );
+ lexf.popFront;
+ assert_eq( lexf.front.str, "polemy" );
+ assert_eq( lexf.front.pos.lineno, 7 );
+ assert_eq( lexf.front.pos.column, 8 );
+ lexf.popFront;
+ assert_eq( lexf.front.str, "." );
lexf.popFront;
- assert( lexf.front.str == "polemy" );
- assert( lexf.front.pos.lineno == 7 );
- assert( lexf.front.pos.column == 8 );
+ assert_eq( lexf.front.str, "lex" );
lexf.popFront;
- assert( lexf.front.str == "." );
+ assert_eq( lexf.front.str, ";" );
lexf.popFront;
- assert( lexf.front.str == "lex" );
- lexf.popFront;
- assert( lexf.front.str == ";" );
- lexf.popFront;
- assert( lexf.front.str == "import" );
- assert( lexf.front.pos.lineno == 8 );
- assert( lexf.front.pos.column == 1 );
+ assert_eq( lexf.front.str, "import" );
+ assert_eq( lexf.front.pos.lineno, 8 );
+ assert_eq( lexf.front.pos.column, 1 );
}
unittest
{
- auto lex = lexerFromString(`my # comment should
-# hey!!
+ auto lex = lexerFromString(`my # comment should`~"\r\n"~`# hey!!
be ignored.
hahaha"hihihi""hu\\\"huhu"#123 aa
-123 aa "aaa
-bbb # 123
-eee"
+123 aa "aaa`~"\r\n"~`bbb # 123`~"\r\n"~`eee"
zzz
`);
Token[] ts = std.array.array(lex);
- assert( ts[0].str == "my" );
- assert( ts[0].pos.lineno == 1 );
- assert( ts[1].str == "be" );
- assert( ts[1].pos.lineno == 3 );
- assert( ts[2].str == "ignored" );
- assert( ts[3].str == "." );
- assert( ts[4].str == "hahaha" );
- assert( ts[4].pos.lineno == 4 );
- assert( ts[4].kind == Token.Kind.identifier );
- assert( ts[5].str == "hihihi" );
- assert( ts[5].pos.lineno == 4 );
- assert( ts[5].kind == Token.Kind.stringLiteral );
- assert( ts[6].str == `hu\"huhu` );
- assert( ts[6].kind == Token.Kind.stringLiteral );
- assert( ts[6].pos.lineno == 4 );
- assert( ts[7].str == "123" );
- assert( ts[7].pos.lineno == 5 );
- assert( ts[7].kind == Token.Kind.number );
- assert( ts[8].str == "aa" );
- assert( ts[9].pos.lineno == 5 );
- assert( ts[9].str == "aaa\nbbb # 123\neee" );
- assert( ts[9].kind == Token.Kind.stringLiteral );
- assert( ts[10].pos.lineno == 8 );
- assert( ts.length == 11 );
+ assert_eq( ts[0].str, "my" );
+ assert_eq( ts[0].pos.lineno, 1 );
+ assert( !ts[0].quoted );
+ assert_eq( ts[1].str, "be" );
+ assert_eq( ts[1].pos.lineno, 3 );
+ assert( !ts[1].quoted );
+ assert_eq( ts[2].str, "ignored" );
+ assert( !ts[2].quoted );
+ assert_eq( ts[3].str, "." );
+ assert( !ts[3].quoted );
+ assert_eq( ts[4].str, "hahaha" );
+ assert_eq( ts[4].pos.lineno, 4 );
+ assert( !ts[4].quoted );
+ assert_eq( ts[5].str, "hihihi" );
+ assert_eq( ts[5].pos.lineno, 4 );
+ assert( ts[5].quoted );
+ assert_eq( ts[6].str, `hu\"huhu` );
+ assert_eq( ts[6].pos.lineno, 4 );
+ assert( ts[6].quoted );
+ assert_eq( ts[7].str, "123" );
+ assert_eq( ts[7].pos.lineno, 5 );
+ assert_eq( ts[8].str, "aa" );
+ assert_eq( ts[9].pos.lineno, 5 );
+ assert_eq( ts[9].str, "aaa\nbbb # 123\neee" );
+ assert( ts[9].quoted );
+ assert_eq( ts[10].pos.lineno, 8 );
+ assert( !ts[10].quoted );
+ assert_eq( ts.length, 11 );
}
Index: polemy/parse.d
==================================================================
--- polemy/parse.d
+++ polemy/parse.d
@@ -6,11 +6,10 @@
*/
module polemy.parse;
import polemy._common;
import polemy.lex;
import polemy.ast;
-import std.bigint;
/// Parsing Failure
class ParserException : Exception
{
@@ -64,11 +63,11 @@
}
Program parseStatements()
{
Program p;
- while( !lex.empty && (lex.front.kind!=Token.Kind.identifier || lex.front.str!="}") )
+ while( !lex.empty && (lex.front.quoted || lex.front.str!="}") )
p ~= parseStatement();
return p;
}
Statement parseStatement()
@@ -78,11 +77,11 @@
if( lex.empty )
throw new ParserException("EOF during parsing a statement");
auto pos = lex.front.pos;
- if( lex.front.kind==Token.Kind.identifier && lex.front.str=="var" )
+ if( !lex.front.quoted && lex.front.str=="var" )
{
// "var" Var "=" Expression ";"
lex.popFront;
string var = lex.front.str;
lex.popFront;
@@ -179,20 +178,20 @@
{
if( lex.empty )
throw new ParserException("EOF during parsing an expression");
auto pos = lex.front.pos;
- if( lex.front.kind == Token.Kind.number )
+ if( lex.front.quoted )
+ {
+ scope(exit) lex.popFront;
+ return new StrLiteralExpression(pos, lex.front.str);
+ }
+ if( isNumber(lex.front.str) ) // is_number
{
scope(exit) lex.popFront;
return new IntLiteralExpression(pos, BigInt(cast(string)lex.front.str));
}
- if( lex.front.kind == Token.Kind.stringLiteral )
- {
- scope(exit) lex.popFront;
- return new StrLiteralExpression(pos, lex.front.str);
- }
if( tryEat("(") )
{
auto e = parseE();
eat(")", "after parenthesized expression");
return e;
@@ -229,11 +228,11 @@
{
if( lex.empty ) {
auto e = ParserException.create(lex,"Unexpected EOF");
throw e;
}
- if( lex.front.kind != Token.Kind.identifier ) {
+ if( lex.front.quoted ) {
auto e = ParserException.create(lex,"Identifier Expected for parameters");
throw e;
}
params ~= lex.front.str;
lex.popFront;
@@ -270,15 +269,20 @@
}
}
bool tryEat(string kwd)
{
- if( lex.empty || lex.front.kind!=Token.Kind.identifier || lex.front.str!=kwd )
+ if( lex.empty || lex.front.quoted || lex.front.str!=kwd )
return false;
lex.popFront;
return true;
}
+
+ bool isNumber(string s)
+ {
+ return find!(`a<'0'||'9'<a`)(s).empty;
+ }
}
unittest
{
auto p = parserFromString(`
@@ -341,10 +345,12 @@
new FunLiteralExpression(null, ["abc"], [
]),
new IntLiteralExpression(null, BigInt(4))
))));
}
+
+/*
unittest
{
auto p = parserFromString(`var x = 1; var f = fun(){x=x+1;}; f(); f(); x;`);
Program prog = p.parseProgram();
}
@@ -358,6 +364,7 @@
new FuncallExpression(null, new VarExpression(null,"<"), new VarExpression(null,"x"),
new IntLiteralExpression(null, BigInt(2))),
new FunLiteralExpression(null, [], [new ExprStatement(null, new IntLiteralExpression(null, BigInt(1)))]),
new FunLiteralExpression(null, [], [new ExprStatement(null, new VarExpression(null, "x"))])
)));
-}
+}
+*/