1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
|
// Copyright Ferdinand Majerech 2011.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
module dyaml.test.tokens;
@safe unittest
{
import std.array : split;
import std.conv : text;
import std.file : readText;
import dyaml.test.common : run;
import dyaml.reader : Reader;
import dyaml.scanner : Scanner;
import dyaml.token : TokenID;
// Read and scan a YAML doc, returning a range of tokens.
static auto scanTestCommon(string filename) @safe
{
ubyte[] yamlData = cast(ubyte[])readText(filename).dup;
return Scanner(new Reader(yamlData, filename));
}
/**
Test tokens output by scanner.
Params:
dataFilename = File to scan.
tokensFilename = File containing expected tokens.
*/
static void testTokens(string dataFilename, string tokensFilename) @safe
{
//representations of YAML tokens in tokens file.
auto replace = [
TokenID.directive: "%",
TokenID.documentStart: "---",
TokenID.documentEnd: "...",
TokenID.alias_: "*",
TokenID.anchor: "&",
TokenID.tag: "!",
TokenID.scalar: "_",
TokenID.blockSequenceStart: "[[",
TokenID.blockMappingStart: "{{",
TokenID.blockEnd: "]}",
TokenID.flowSequenceStart: "[",
TokenID.flowSequenceEnd: "]",
TokenID.flowMappingStart: "{",
TokenID.flowMappingEnd: "}",
TokenID.blockEntry: ",",
TokenID.flowEntry: ",",
TokenID.key: "?",
TokenID.value: ":"
];
string[] tokens;
string[] expectedTokens = readText(tokensFilename).split();
foreach (token; scanTestCommon(dataFilename))
{
if (token.id != TokenID.streamStart && token.id != TokenID.streamEnd)
{
tokens ~= replace[token.id];
}
}
assert(tokens == expectedTokens,
text("In token test for '", tokensFilename, "', expected '", expectedTokens, "', got '", tokens, "'"));
}
/**
Test scanner by scanning a file, expecting no errors.
Params:
dataFilename = File to scan.
canonicalFilename = Another file to scan, in canonical YAML format.
*/
static void testScanner(string dataFilename, string canonicalFilename) @safe
{
foreach (filename; [dataFilename, canonicalFilename])
{
string[] tokens;
foreach (token; scanTestCommon(filename))
{
tokens ~= token.id.text;
}
}
}
run(&testTokens, ["data", "tokens"]);
run(&testScanner, ["data", "canonical"]);
}
|
