dlex.test source code

1 module dlex.test;
2 
3 import dlex;
4 
5 unittest {
6     enum Type {
7 	Ident,
8 	Number,
9 	Space,
10 	Symbol,
11     }
12 
13     import std.uni;
14 
15     auto dlex = new DLex!(Type);
16     dlex.Rules([
17             dlex.RuleT(Type.Ident, Pred(&isAlpha) + Pred(&isAlphaNum).Repeat),
18             dlex.RuleT(Type.Number, Pred(&isNumber).Repeat),
19             dlex.RuleT(Type.Space, Pred(&isSpace).Skip),
20             dlex.RuleT(Type.Symbol, Any),
21     ]);
22     auto rs = dlex.Lex("Int 123 *");
23 
24     assert(rs.length == 3);
25     assert(rs[0].str == "Int");
26     assert(rs[0].pos.col == 1);
27     assert(rs[1].str == "123");
28     assert(rs[1].pos.col == 5);
29     assert(rs[2].str == "*");
30     assert(rs[2].pos.col == 9);
31 
32     dlex.Rules([
33 	    dlex.RuleT(Type.Symbol, Char('>')),
34 	    dlex.RuleT(Type.Symbol, Char('<')),
35 	    dlex.RuleT(Type.Symbol, String(">>")),
36 	    dlex.RuleT(Type.Symbol, String("<<")),
37 	    dlex.RuleT(Type.Space, Pred(&isSpace).Skip),
38     ]);
39     rs = dlex.Lex("> > >><>><>");
40     assert(rs.length == 7);
41     assert(rs[0].str == ">");
42     assert(rs[1].str == ">");
43     assert(rs[2].str == ">>");
44     assert(rs[3].str == "<");
45     assert(rs[4].str == ">>");
46     assert(rs[5].str == "<");
47     assert(rs[6].str == ">");
48 }
49 
50 
51 unittest {
52     import std.uni, std..string;
53     enum Type {
54 	Space,
55 	Newline,
56 	Number,
57 	Identifier,
58 	String,
59 	Symbol,
60     }
61     auto dlex = new DLex!(Type);
62     dlex.Rules([
63 	    dlex.RuleT(Type.Space, (Char(' ') | Char('\t')).Skip),
64 	    dlex.RuleT(Type.Newline, (Char('\n') | Char(';')).Repeat),
65 	    dlex.RuleT(Type.Number, Pred(&isNumber).Repeat),
66 	    dlex.RuleT(Type.Identifier, Pred((c) => (c == '_' || c.isAlpha))+Pred((c) => (c == '_' || c.isAlphaNum)).Repeat),
67 	    dlex.RuleT(Type.String, Between(Char('"'), Char('"'), String(`\"`).As((dstring s) => `"`d)|Any).As((dstring s) => s[1..$-1])),
68 	    dlex.RuleT(Type.Symbol, Char('=')),
69 	    dlex.RuleT(Type.Symbol, Char('+')),
70 	    dlex.RuleT(Type.Symbol, Char('<')),
71 	    dlex.RuleT(Type.Symbol, String("+=")),
72 	    dlex.RuleT(Type.Symbol, Char('{')),
73 	    dlex.RuleT(Type.Symbol, Char('}')),
74 	    dlex.RuleT(Type.Symbol, Char('(')),
75 	    dlex.RuleT(Type.Symbol, Char(')')),
76     ]);
77     auto rs = dlex.Lex(`
78 int main() {
79     int i = 1;
80     print("Start\"");
81     while (i < 10) {
82 	print(i);
83 	i += 1;
84     }
85 }
86     `d.strip);
87 
88     assert(rs[0].type == Type.Identifier);
89     assert(rs[0].str == "int");
90     assert(rs[2].type == Type.Symbol);
91     assert(rs[2].str == "(");
92     assert(rs[4].type == Type.Symbol);
93     assert(rs[4].str == "{");
94     assert(rs[5].type == Type.Newline);
95     assert(rs[5].str == "\n");
96     assert(rs[8].type == Type.Symbol);
97     assert(rs[8].str == "=");
98     assert(rs[9].type == Type.Number);
99     assert(rs[9].str == "1");
100     assert(rs[10].type == Type.Newline);
101     assert(rs[10].str == ";\n");
102     assert(rs[13].type == Type.String);
103     assert(rs[13].str == "Start\"");
104     assert(rs[13].pos.line == 3);
105     assert(rs[13].pos.col == 10);
106     assert(rs[30].type == Type.Symbol);
107     assert(rs[30].str == "+=");
108 }
109 
110 unittest {
111     enum Type {
112 	Ident,
113 	Number,
114 	Space,
115 	Symbol,
116     }
117 
118     import std.uni;
119     import std.exception;
120 
121     auto dlex = new DLex!(Type);
122     dlex.Rules([
123             dlex.RuleT(Type.Ident, Pred(&isAlpha) + Pred(&isAlphaNum).Repeat),
124             dlex.RuleT(Type.Number, Pred(&isNumber).Repeat),
125             dlex.RuleT(Type.Space, Pred(&isSpace).Skip),
126     ]);
127     assertThrown(dlex.Lex("Int 123 *"));
128 
129 }