4198578702 2010-11-07 kinaba: /** 4198578702 2010-11-07 kinaba: * Authors: k.inaba 4198578702 2010-11-07 kinaba: * License: NYSL 0.9982 http://www.kmonos.net/nysl/ 4198578702 2010-11-07 kinaba: * 4198578702 2010-11-07 kinaba: * Lexer for Polemy programming language. 4198578702 2010-11-07 kinaba: */ 423f308350 2010-11-07 kinaba: module polemy.lex; 423f308350 2010-11-07 kinaba: import polemy._common; 5e407d7cf8 2010-11-08 kinaba: import std.file : readText; 5e407d7cf8 2010-11-08 kinaba: import std.ctype : isspace, isalnum; 423f308350 2010-11-07 kinaba: 2459e9a821 2010-11-09 kinaba: /*mixin*/ 2459e9a821 2010-11-09 kinaba: template ExceptionWithPosition() 2459e9a821 2010-11-09 kinaba: { 2459e9a821 2010-11-09 kinaba: const LexPosition pos; 2459e9a821 2010-11-09 kinaba: this( const LexPosition pos, string msg, string file=null, size_t line=0, Throwable next=null ) 2459e9a821 2010-11-09 kinaba: { super(sprintf!"[%s] %s"(pos, msg), file, line, next); this.pos = pos; } 2459e9a821 2010-11-09 kinaba: } 2459e9a821 2010-11-09 kinaba: 1c01f44f52 2010-11-13 kinaba: /// Thrown when encountered an EOF in the middle of a lexical token 1c01f44f52 2010-11-13 kinaba: 2459e9a821 2010-11-09 kinaba: class UnexpectedEOF : Exception 2459e9a821 2010-11-09 kinaba: { 2459e9a821 2010-11-09 kinaba: mixin ExceptionWithPosition; 2459e9a821 2010-11-09 kinaba: } 38fcc662be 2010-11-10 kinaba: 1c01f44f52 2010-11-13 kinaba: /// Thrown when encountered a lexical error 1c01f44f52 2010-11-13 kinaba: 5e407d7cf8 2010-11-08 kinaba: class LexException : Exception 5e407d7cf8 2010-11-08 kinaba: { 2459e9a821 2010-11-09 kinaba: mixin ExceptionWithPosition; 5e407d7cf8 2010-11-08 kinaba: }; 5e407d7cf8 2010-11-08 kinaba: 1c01f44f52 2010-11-13 kinaba: /// Represents a position in source codes 423f308350 2010-11-07 kinaba: 423f308350 2010-11-07 kinaba: class LexPosition 423f308350 2010-11-07 kinaba: { 4198578702 2010-11-07 kinaba: immutable string filename; /// name of the source file b985f3bf91 2010-11-08 kinaba: immutable int lineno; /// 1-origin b985f3bf91 2010-11-08 kinaba: immutable int column; /// 1-origin 423f308350 2010-11-07 kinaba: 8de5b49cdf 2010-11-09 kinaba: mixin SimpleClass; 423f308350 2010-11-07 kinaba: override string toString() const 423f308350 2010-11-07 kinaba: { return sprintf!"%s:%d:%d"(filename, lineno, column); } 423f308350 2010-11-07 kinaba: 80ff567c75 2010-11-08 kinaba: static immutable LexPosition dummy; 80ff567c75 2010-11-08 kinaba: static this(){ dummy = new immutable(LexPosition)("<unnamed>",0,0); } 423f308350 2010-11-07 kinaba: } 423f308350 2010-11-07 kinaba: 423f308350 2010-11-07 kinaba: unittest 423f308350 2010-11-07 kinaba: { 423f308350 2010-11-07 kinaba: auto p = new LexPosition("hello.cpp", 123, 45); 423f308350 2010-11-07 kinaba: 61998c472a 2010-11-08 kinaba: assert_eq( p.filename, "hello.cpp" ); 61998c472a 2010-11-08 kinaba: assert_eq( p.lineno, 123 ); 61998c472a 2010-11-08 kinaba: assert_eq( p.column, 45 ); 515502e8d1 2010-11-20 kinaba: assert_eq( text(p), "hello.cpp:123:45" ); 423f308350 2010-11-07 kinaba: 423f308350 2010-11-07 kinaba: assert( !__traits(compiles, new LexPosition) ); 423f308350 2010-11-07 kinaba: assert( !__traits(compiles, p.filename="foo") ); 423f308350 2010-11-07 kinaba: assert( !__traits(compiles, p.lineno =789) ); 423f308350 2010-11-07 kinaba: assert( !__traits(compiles, p.column =222) ); 7de80acfb8 2010-11-09 kinaba: 7de80acfb8 2010-11-09 kinaba: auto q = new LexPosition("hello.cpp", 123, 46); 7de80acfb8 2010-11-09 kinaba: assert_lt( p, q ); 7de80acfb8 2010-11-09 kinaba: assert_ne( p, q ); 423f308350 2010-11-07 kinaba: } 423f308350 2010-11-07 kinaba: 423f308350 2010-11-07 kinaba: /// Represents a lexer token 423f308350 2010-11-07 kinaba: 423f308350 2010-11-07 kinaba: class Token 423f308350 2010-11-07 kinaba: { 8d297342aa 2010-11-08 kinaba: immutable LexPosition pos; /// Position where the token occurred in the source 8d297342aa 2010-11-08 kinaba: immutable string str; /// The token string itself 8d297342aa 2010-11-08 kinaba: immutable bool quoted; /// Was it a "quoted" token or unquoted? 423f308350 2010-11-07 kinaba: 8de5b49cdf 2010-11-09 kinaba: mixin SimpleClass; 423f308350 2010-11-07 kinaba: } 423f308350 2010-11-07 kinaba: 423f308350 2010-11-07 kinaba: unittest 423f308350 2010-11-07 kinaba: { 423f308350 2010-11-07 kinaba: auto p = new immutable(LexPosition)("hello.cpp", 123, 45); 8d297342aa 2010-11-08 kinaba: auto t = new Token(p, "class", false); 8d297342aa 2010-11-08 kinaba: auto u = new Token(p, "class", true); 423f308350 2010-11-07 kinaba: 61998c472a 2010-11-08 kinaba: assert_eq( t.pos, p ); 61998c472a 2010-11-08 kinaba: assert_eq( t.str, "class" ); 8d297342aa 2010-11-08 kinaba: assert( !t.quoted ); 8d297342aa 2010-11-08 kinaba: assert_eq( t, new Token(p, "class", false) ); 8d297342aa 2010-11-08 kinaba: assert_lt( t, new Token(p, "struct", false) ); 8d297342aa 2010-11-08 kinaba: assert_ne( t, u ); 8d297342aa 2010-11-08 kinaba: assert( u.quoted ); 423f308350 2010-11-07 kinaba: 423f308350 2010-11-07 kinaba: assert( !__traits(compiles, new Token) ); 423f308350 2010-11-07 kinaba: assert( !__traits(compiles, t.pos=p) ); 423f308350 2010-11-07 kinaba: assert( !__traits(compiles, t.str=789) ); 8d297342aa 2010-11-08 kinaba: assert( !__traits(compiles, t.quoted=true) ); 423f308350 2010-11-07 kinaba: } 423f308350 2010-11-07 kinaba: b985f3bf91 2010-11-08 kinaba: /// Named Construtors for Lexer 423f308350 2010-11-07 kinaba: 1c01f44f52 2010-11-13 kinaba: Lexer lexerFromFile(T...)( string filename, T ln_cn ) 423f308350 2010-11-07 kinaba: { 1c01f44f52 2010-11-13 kinaba: return lexerFromString( std.file.readText(filename), filename, ln_cn ); 423f308350 2010-11-07 kinaba: } 423f308350 2010-11-07 kinaba: 1c01f44f52 2010-11-13 kinaba: /// Named Construtor for Lexer 38fcc662be 2010-11-10 kinaba: 38fcc662be 2010-11-10 kinaba: LexerT!(PositionedReader!CharSeq) /* ddoc doesn't recognize auto return... bugzilla:2581 */ 38fcc662be 2010-11-10 kinaba: lexerFromString(CharSeq)( CharSeq str, string filename="<unnamed>", int lineno=1, int column=1 ) 423f308350 2010-11-07 kinaba: { 5e407d7cf8 2010-11-08 kinaba: return new LexerT!(PositionedReader!CharSeq)( 5e407d7cf8 2010-11-08 kinaba: PositionedReader!CharSeq(str, filename, lineno, column) 5e407d7cf8 2010-11-08 kinaba: ); 423f308350 2010-11-07 kinaba: } 423f308350 2010-11-07 kinaba: 1c01f44f52 2010-11-13 kinaba: /// Standard Lexer Type (all you have to know is that this is a forward range of Tokens!) 423f308350 2010-11-07 kinaba: 5e407d7cf8 2010-11-08 kinaba: alias LexerT!(PositionedReader!string) Lexer; 5e407d7cf8 2010-11-08 kinaba: 5e407d7cf8 2010-11-08 kinaba: /// Lexer Implementation 5e407d7cf8 2010-11-08 kinaba: 5e407d7cf8 2010-11-08 kinaba: class LexerT(Reader) 1c01f44f52 2010-11-13 kinaba: if( isForwardRange!(Reader) && is(ElementType!(Reader)==dchar) ) 423f308350 2010-11-07 kinaba: { 4198578702 2010-11-07 kinaba: /// Range primitive 423f308350 2010-11-07 kinaba: bool empty() /*@property*/ 423f308350 2010-11-07 kinaba: { 423f308350 2010-11-07 kinaba: return current is null; 423f308350 2010-11-07 kinaba: } 423f308350 2010-11-07 kinaba: 4198578702 2010-11-07 kinaba: /// Range primitive 423f308350 2010-11-07 kinaba: Token front() /*@property*/ 423f308350 2010-11-07 kinaba: { 423f308350 2010-11-07 kinaba: return std.exception.enforce(current, "Lexer has already reached the end"); 423f308350 2010-11-07 kinaba: } 423f308350 2010-11-07 kinaba: 4198578702 2010-11-07 kinaba: /// Range primitive 423f308350 2010-11-07 kinaba: void popFront() /*@property*/ 423f308350 2010-11-07 kinaba: { 423f308350 2010-11-07 kinaba: std.exception.enforce(current, "Lexer has already reached the end"); 423f308350 2010-11-07 kinaba: current = readNext(); 423f308350 2010-11-07 kinaba: } 423f308350 2010-11-07 kinaba: 4198578702 2010-11-07 kinaba: /// Range primitive 5e407d7cf8 2010-11-08 kinaba: typeof(this) save() /*@property*/ 423f308350 2010-11-07 kinaba: { 5e407d7cf8 2010-11-08 kinaba: return new typeof(this)(reader.save, current); 423f308350 2010-11-07 kinaba: } 423f308350 2010-11-07 kinaba: 423f308350 2010-11-07 kinaba: private: // implementation 423f308350 2010-11-07 kinaba: 5e407d7cf8 2010-11-08 kinaba: Reader reader; 423f308350 2010-11-07 kinaba: Token current; 423f308350 2010-11-07 kinaba: 423f308350 2010-11-07 kinaba: invariant() 423f308350 2010-11-07 kinaba: { 1c01f44f52 2010-11-13 kinaba: assert( reader.empty || !isSpace(reader.front) ); 5e407d7cf8 2010-11-08 kinaba: } 5e407d7cf8 2010-11-08 kinaba: 5e407d7cf8 2010-11-08 kinaba: this( Reader reader, Token current = null ) 5e407d7cf8 2010-11-08 kinaba: { 5e407d7cf8 2010-11-08 kinaba: this.reader = reader; 5e407d7cf8 2010-11-08 kinaba: readWhile!isSpace(); 5e407d7cf8 2010-11-08 kinaba: this.current = (current is null ? readNext() : current); 5e407d7cf8 2010-11-08 kinaba: } 5e407d7cf8 2010-11-08 kinaba: 1c01f44f52 2010-11-13 kinaba: public static 1c01f44f52 2010-11-13 kinaba: { 5e407d7cf8 2010-11-08 kinaba: bool isSpace (dchar c) { return std.ctype.isspace(c)!=0; } 5e407d7cf8 2010-11-08 kinaba: bool isSymbol (dchar c) { return 0x21<=c && c<=0x7f && !std.ctype.isalnum(c) && c!='_' && c!='\''; } 7465fcdd7f 2010-11-09 kinaba: bool isSSymbol (dchar c) { return "()[]{};@".canFind(c); } aa770610d3 2010-11-08 kinaba: bool isMSymbol (dchar c) { return isSymbol(c) && !isSSymbol(c) && c!='"' && c!='#'; } 5e407d7cf8 2010-11-08 kinaba: bool isLetter (dchar c) { return !isSpace(c) && !isSymbol(c); } 423f308350 2010-11-07 kinaba: } 423f308350 2010-11-07 kinaba: 5e407d7cf8 2010-11-08 kinaba: string readQuoted(const LexPosition pos){char[] buf; return readQuoted(pos,buf);} 5e407d7cf8 2010-11-08 kinaba: string readQuoted(const LexPosition pos, ref char[] buf) 423f308350 2010-11-07 kinaba: { 5e407d7cf8 2010-11-08 kinaba: if( reader.empty ) 2459e9a821 2010-11-09 kinaba: throw genex!UnexpectedEOF(pos, "Quoted string not terminated"); 5e407d7cf8 2010-11-08 kinaba: dchar c = reader.front; 5e407d7cf8 2010-11-08 kinaba: reader.popFront; 5e407d7cf8 2010-11-08 kinaba: if( c == '"' ) 5e407d7cf8 2010-11-08 kinaba: return assumeUnique(buf); 5e407d7cf8 2010-11-08 kinaba: if( c == '\\' && !reader.empty ) { 5e407d7cf8 2010-11-08 kinaba: if( reader.front=='"' ) { 5e407d7cf8 2010-11-08 kinaba: reader.popFront; 5e407d7cf8 2010-11-08 kinaba: return readQuoted(pos,buf ~= '\"'); 5e407d7cf8 2010-11-08 kinaba: } 5e407d7cf8 2010-11-08 kinaba: if( reader.front=='\\' ) { 5e407d7cf8 2010-11-08 kinaba: reader.popFront; 5e407d7cf8 2010-11-08 kinaba: return readQuoted(pos,buf ~= '\\'); 5e407d7cf8 2010-11-08 kinaba: } 5e407d7cf8 2010-11-08 kinaba: } 5e407d7cf8 2010-11-08 kinaba: return readQuoted(pos,buf ~= c); 423f308350 2010-11-07 kinaba: } 423f308350 2010-11-07 kinaba: 5e407d7cf8 2010-11-08 kinaba: string readWhile(alias fn)() 423f308350 2010-11-07 kinaba: { 5e407d7cf8 2010-11-08 kinaba: char[] buf; 5e407d7cf8 2010-11-08 kinaba: for(; !reader.empty && fn(reader.front); reader.popFront) 5e407d7cf8 2010-11-08 kinaba: buf ~= reader.front; 5e407d7cf8 2010-11-08 kinaba: return assumeUnique(buf); 423f308350 2010-11-07 kinaba: } 423f308350 2010-11-07 kinaba: 423f308350 2010-11-07 kinaba: Token readNext() 423f308350 2010-11-07 kinaba: { 5e407d7cf8 2010-11-08 kinaba: if( reader.empty ) 423f308350 2010-11-07 kinaba: return null; 5e407d7cf8 2010-11-08 kinaba: scope(success) 5e407d7cf8 2010-11-08 kinaba: readWhile!isSpace(); 5e407d7cf8 2010-11-08 kinaba: if( reader.front == '#' ) // comment 5e407d7cf8 2010-11-08 kinaba: { 5e407d7cf8 2010-11-08 kinaba: reader = find(reader, '\n'); 5e407d7cf8 2010-11-08 kinaba: readWhile!isSpace(); 5e407d7cf8 2010-11-08 kinaba: return readNext(); 5e407d7cf8 2010-11-08 kinaba: } 5e407d7cf8 2010-11-08 kinaba: else if( reader.front == '"' ) // quoted 5e407d7cf8 2010-11-08 kinaba: { 5e407d7cf8 2010-11-08 kinaba: auto pos = reader.currentPosition(); 5e407d7cf8 2010-11-08 kinaba: reader.popFront; 5e407d7cf8 2010-11-08 kinaba: return new Token(pos, readQuoted(pos), true); 5e407d7cf8 2010-11-08 kinaba: } 5e407d7cf8 2010-11-08 kinaba: else if( isSSymbol(reader.front) ) // paren 5e407d7cf8 2010-11-08 kinaba: { 5e407d7cf8 2010-11-08 kinaba: auto pos = reader.currentPosition(); 5e407d7cf8 2010-11-08 kinaba: string s; s~=reader.front; reader.popFront; 5e407d7cf8 2010-11-08 kinaba: return new Token(pos, s, false); 5e407d7cf8 2010-11-08 kinaba: } 5e407d7cf8 2010-11-08 kinaba: else if( isMSymbol(reader.front) ) // symbol 423f308350 2010-11-07 kinaba: { 5e407d7cf8 2010-11-08 kinaba: auto pos = reader.currentPosition(); 5e407d7cf8 2010-11-08 kinaba: return new Token(pos, readWhile!isMSymbol(), false); 423f308350 2010-11-07 kinaba: } 423f308350 2010-11-07 kinaba: else 423f308350 2010-11-07 kinaba: { 5e407d7cf8 2010-11-08 kinaba: auto pos = reader.currentPosition(); 5e407d7cf8 2010-11-08 kinaba: return new Token(pos, readWhile!isLetter(), false); 423f308350 2010-11-07 kinaba: } 423f308350 2010-11-07 kinaba: } 423f308350 2010-11-07 kinaba: } 423f308350 2010-11-07 kinaba: 423f308350 2010-11-07 kinaba: unittest 423f308350 2010-11-07 kinaba: { 423f308350 2010-11-07 kinaba: assert( std.range.isForwardRange!(Lexer) ); 7de80acfb8 2010-11-09 kinaba: assert( is(ElementType!(Lexer) == Token) ); 8d297342aa 2010-11-08 kinaba: } 8d297342aa 2010-11-08 kinaba: 8d297342aa 2010-11-08 kinaba: unittest 8d297342aa 2010-11-08 kinaba: { 5e407d7cf8 2010-11-08 kinaba: auto lex = lexerFromString("this is a \t\r\n pen :-( @@; "); 8d297342aa 2010-11-08 kinaba: Token[] ts = std.array.array(lex); 8d297342aa 2010-11-08 kinaba: 8d297342aa 2010-11-08 kinaba: assert_eq( ts[0].pos.lineno, 1 ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[0].pos.column, 1 ); 8d297342aa 2010-11-08 kinaba: assert( !ts[0].quoted ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[0].str, "this" ); 8d297342aa 2010-11-08 kinaba: 8d297342aa 2010-11-08 kinaba: assert_eq( ts[1].pos.lineno, 1 ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[1].pos.column, 6 ); 8d297342aa 2010-11-08 kinaba: assert( !ts[1].quoted ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[1].str, "is" ); 8d297342aa 2010-11-08 kinaba: 8d297342aa 2010-11-08 kinaba: assert_eq( ts[2].pos.lineno, 1 ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[2].pos.column, 9 ); 8d297342aa 2010-11-08 kinaba: assert( !ts[2].quoted ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[2].str, "a" ); 8d297342aa 2010-11-08 kinaba: 8d297342aa 2010-11-08 kinaba: assert_eq( ts[3].pos.lineno, 2 ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[3].pos.column, 2 ); 8d297342aa 2010-11-08 kinaba: assert( !ts[3].quoted ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[3].str, "pen" ); 8d297342aa 2010-11-08 kinaba: 8d297342aa 2010-11-08 kinaba: assert_eq( ts[4].pos.lineno, 2 ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[4].pos.column, 6 ); 5e407d7cf8 2010-11-08 kinaba: assert_eq( ts[4].str, ":-" ); 8d297342aa 2010-11-08 kinaba: 8d297342aa 2010-11-08 kinaba: assert_eq( ts[5].pos.lineno, 2 ); 5e407d7cf8 2010-11-08 kinaba: assert_eq( ts[5].pos.column, 8 ); 5e407d7cf8 2010-11-08 kinaba: assert_eq( ts[5].str, "(" ); 7465fcdd7f 2010-11-09 kinaba: assert_eq( ts[6].str, "@" ); 7465fcdd7f 2010-11-09 kinaba: assert_eq( ts[7].str, "@" ); 7465fcdd7f 2010-11-09 kinaba: assert_eq( ts[8].str, ";" ); // paren and simicolons, atmarks are split 8d297342aa 2010-11-08 kinaba: 7465fcdd7f 2010-11-09 kinaba: assert_eq( ts.length, 9 ); 8d297342aa 2010-11-08 kinaba: } 8d297342aa 2010-11-08 kinaba: 8d297342aa 2010-11-08 kinaba: unittest 8d297342aa 2010-11-08 kinaba: { 5e407d7cf8 2010-11-08 kinaba: // !! be sure to run the unittest on the root of the source directory 8d297342aa 2010-11-08 kinaba: auto lexf = lexerFromFile("polemy/lex.d"); 8d297342aa 2010-11-08 kinaba: lexf = find!`a.str == "module"`(lexf); 8d297342aa 2010-11-08 kinaba: assert_eq( lexf.front.str, "module" ); 8d297342aa 2010-11-08 kinaba: assert_eq( lexf.front.pos.filename, "polemy/lex.d" ); 8d297342aa 2010-11-08 kinaba: assert_eq( lexf.front.pos.lineno, 7 ); 8d297342aa 2010-11-08 kinaba: assert_eq( lexf.front.pos.column, 1 ); 8d297342aa 2010-11-08 kinaba: lexf.popFront; 8d297342aa 2010-11-08 kinaba: assert_eq( lexf.front.str, "polemy" ); 8d297342aa 2010-11-08 kinaba: assert_eq( lexf.front.pos.lineno, 7 ); 8d297342aa 2010-11-08 kinaba: assert_eq( lexf.front.pos.column, 8 ); 8d297342aa 2010-11-08 kinaba: lexf.popFront; 8d297342aa 2010-11-08 kinaba: lexf.popFront; 8d297342aa 2010-11-08 kinaba: lexf.popFront; 8d297342aa 2010-11-08 kinaba: lexf.popFront; 8d297342aa 2010-11-08 kinaba: assert_eq( lexf.front.str, "import" ); 8d297342aa 2010-11-08 kinaba: assert_eq( lexf.front.pos.lineno, 8 ); 8d297342aa 2010-11-08 kinaba: assert_eq( lexf.front.pos.column, 1 ); 8d297342aa 2010-11-08 kinaba: } 8d297342aa 2010-11-08 kinaba: 8d297342aa 2010-11-08 kinaba: unittest 8d297342aa 2010-11-08 kinaba: { 2459e9a821 2010-11-09 kinaba: assert_throw!UnexpectedEOF( lexerFromString(`"`) ); 5e407d7cf8 2010-11-08 kinaba: } 5e407d7cf8 2010-11-08 kinaba: 5e407d7cf8 2010-11-08 kinaba: unittest 5e407d7cf8 2010-11-08 kinaba: { 8d297342aa 2010-11-08 kinaba: auto lex = lexerFromString(`my # comment should`~"\r\n"~`# hey!! 8d297342aa 2010-11-08 kinaba: be ignored. 8d297342aa 2010-11-08 kinaba: hahaha"hihihi""hu\\\"huhu"#123 aa 5e407d7cf8 2010-11-08 kinaba: 123 aa "aaa`~"\n"~`bbb # 123`~"\r\n"~`eee" 8d297342aa 2010-11-08 kinaba: zzz 8d297342aa 2010-11-08 kinaba: `); 423f308350 2010-11-07 kinaba: Token[] ts = std.array.array(lex); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[0].str, "my" ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[0].pos.lineno, 1 ); 8d297342aa 2010-11-08 kinaba: assert( !ts[0].quoted ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[1].str, "be" ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[1].pos.lineno, 3 ); 8d297342aa 2010-11-08 kinaba: assert( !ts[1].quoted ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[2].str, "ignored" ); 8d297342aa 2010-11-08 kinaba: assert( !ts[2].quoted ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[3].str, "." ); 8d297342aa 2010-11-08 kinaba: assert( !ts[3].quoted ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[4].str, "hahaha" ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[4].pos.lineno, 4 ); 8d297342aa 2010-11-08 kinaba: assert( !ts[4].quoted ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[5].str, "hihihi" ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[5].pos.lineno, 4 ); 8d297342aa 2010-11-08 kinaba: assert( ts[5].quoted ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[6].str, `hu\"huhu` ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[6].pos.lineno, 4 ); 8d297342aa 2010-11-08 kinaba: assert( ts[6].quoted ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[7].str, "123" ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[7].pos.lineno, 5 ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[8].str, "aa" ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[9].pos.lineno, 5 ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[9].str, "aaa\nbbb # 123\neee" ); 8d297342aa 2010-11-08 kinaba: assert( ts[9].quoted ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts[10].pos.lineno, 8 ); 8d297342aa 2010-11-08 kinaba: assert( !ts[10].quoted ); 8d297342aa 2010-11-08 kinaba: assert_eq( ts.length, 11 ); 5e407d7cf8 2010-11-08 kinaba: } 5e407d7cf8 2010-11-08 kinaba: 5e407d7cf8 2010-11-08 kinaba: unittest 5e407d7cf8 2010-11-08 kinaba: { 5e407d7cf8 2010-11-08 kinaba: auto lex2 = lexerFromString(" a12\n3a 5 "); 5e407d7cf8 2010-11-08 kinaba: assert_eq( lex2.front.str, "a12" ); 5e407d7cf8 2010-11-08 kinaba: lex2.popFront; 5e407d7cf8 2010-11-08 kinaba: auto lex3 = lex2.save; 5e407d7cf8 2010-11-08 kinaba: assert_eq( lex2.front.str, "3a" ); 5e407d7cf8 2010-11-08 kinaba: lex2.popFront; 5e407d7cf8 2010-11-08 kinaba: assert_eq( lex3.front.str, "3a" ); 5e407d7cf8 2010-11-08 kinaba: assert_eq( lex2.front.str, "5" ); 5e407d7cf8 2010-11-08 kinaba: lex2.popFront; 5e407d7cf8 2010-11-08 kinaba: lex3.popFront; 5e407d7cf8 2010-11-08 kinaba: assert( lex2.empty ); 5e407d7cf8 2010-11-08 kinaba: assert( !lex3.empty ); 5e407d7cf8 2010-11-08 kinaba: assert_eq( lex3.front.str, "5" ); 5e407d7cf8 2010-11-08 kinaba: } 5e407d7cf8 2010-11-08 kinaba: aa770610d3 2010-11-08 kinaba: unittest aa770610d3 2010-11-08 kinaba: { aa770610d3 2010-11-08 kinaba: auto lex = lexerFromString(`=""`); aa770610d3 2010-11-08 kinaba: assert_eq(lex.front.str, "="); lex.popFront; aa770610d3 2010-11-08 kinaba: assert_eq(lex.front.str, ""); lex.popFront; aa770610d3 2010-11-08 kinaba: assert( lex.empty ); 7465fcdd7f 2010-11-09 kinaba: assert_eq( lexerFromString(`-@`).front.str, "-" ); aa770610d3 2010-11-08 kinaba: } aa770610d3 2010-11-08 kinaba: 5e407d7cf8 2010-11-08 kinaba: /// Forward range for reader character by character, 5e407d7cf8 2010-11-08 kinaba: /// keeping track of position information and caring \r\n -> \n conversion. 5e407d7cf8 2010-11-08 kinaba: 5e407d7cf8 2010-11-08 kinaba: struct PositionedReader(CharSeq) 1c01f44f52 2010-11-13 kinaba: if( isForwardRange!(CharSeq) && is(ElementType!(CharSeq)==dchar) ) 5e407d7cf8 2010-11-08 kinaba: { 5e407d7cf8 2010-11-08 kinaba: CharSeq buffer; 5e407d7cf8 2010-11-08 kinaba: string filename; 5e407d7cf8 2010-11-08 kinaba: int lineno; 5e407d7cf8 2010-11-08 kinaba: int column; 5e407d7cf8 2010-11-08 kinaba: 5e407d7cf8 2010-11-08 kinaba: /// Range primitive 5e407d7cf8 2010-11-08 kinaba: bool empty() /*@property*/ 5e407d7cf8 2010-11-08 kinaba: { 5e407d7cf8 2010-11-08 kinaba: return buffer.empty; 5e407d7cf8 2010-11-08 kinaba: } 5e407d7cf8 2010-11-08 kinaba: 5e407d7cf8 2010-11-08 kinaba: /// Range primitive 5e407d7cf8 2010-11-08 kinaba: dchar front() /*@property*/ 5e407d7cf8 2010-11-08 kinaba: { 5e407d7cf8 2010-11-08 kinaba: dchar c = buffer.front; 5e407d7cf8 2010-11-08 kinaba: return (c=='\r' ? '\n' : c); 5e407d7cf8 2010-11-08 kinaba: } 5e407d7cf8 2010-11-08 kinaba: 5e407d7cf8 2010-11-08 kinaba: /// Range primitive 5e407d7cf8 2010-11-08 kinaba: void popFront() /*@property*/ 5e407d7cf8 2010-11-08 kinaba: { 5e407d7cf8 2010-11-08 kinaba: dchar c = buffer.front; 5e407d7cf8 2010-11-08 kinaba: buffer.popFront; 5e407d7cf8 2010-11-08 kinaba: if( c=='\r' ) 5e407d7cf8 2010-11-08 kinaba: { 5e407d7cf8 2010-11-08 kinaba: if( !buffer.empty && buffer.front=='\n' ) 5e407d7cf8 2010-11-08 kinaba: buffer.popFront; 5e407d7cf8 2010-11-08 kinaba: c = '\n'; 5e407d7cf8 2010-11-08 kinaba: } 5e407d7cf8 2010-11-08 kinaba: if( c=='\n' ) 5e407d7cf8 2010-11-08 kinaba: { 5e407d7cf8 2010-11-08 kinaba: lineno ++; 5e407d7cf8 2010-11-08 kinaba: column = 1; 5e407d7cf8 2010-11-08 kinaba: } 5e407d7cf8 2010-11-08 kinaba: else 5e407d7cf8 2010-11-08 kinaba: column ++; 5e407d7cf8 2010-11-08 kinaba: } 5e407d7cf8 2010-11-08 kinaba: 5e407d7cf8 2010-11-08 kinaba: /// Range primitive 5e407d7cf8 2010-11-08 kinaba: typeof(this) save() /*@property*/ 5e407d7cf8 2010-11-08 kinaba: { 5e407d7cf8 2010-11-08 kinaba: return this; 5e407d7cf8 2010-11-08 kinaba: } 5e407d7cf8 2010-11-08 kinaba: 5e407d7cf8 2010-11-08 kinaba: /// Get the current position 5e407d7cf8 2010-11-08 kinaba: immutable(LexPosition) currentPosition() const 5e407d7cf8 2010-11-08 kinaba: { 5e407d7cf8 2010-11-08 kinaba: return new immutable(LexPosition)(filename, lineno, column); 5e407d7cf8 2010-11-08 kinaba: } 5e407d7cf8 2010-11-08 kinaba: } 5e407d7cf8 2010-11-08 kinaba: 5e407d7cf8 2010-11-08 kinaba: unittest 5e407d7cf8 2010-11-08 kinaba: { 5e407d7cf8 2010-11-08 kinaba: assert( isForwardRange!(PositionedReader!string) ); 5e407d7cf8 2010-11-08 kinaba: assert( is(ElementType!(PositionedReader!string) == dchar) ); 423f308350 2010-11-07 kinaba: }