1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.pyyaml;
17
18 import java.io.File;
19 import java.io.FileInputStream;
20 import java.io.FileNotFoundException;
21 import java.io.IOException;
22 import java.io.InputStream;
23 import java.util.ArrayList;
24 import java.util.HashMap;
25 import java.util.List;
26 import java.util.Map;
27
28 import org.yaml.snakeyaml.reader.StreamReader;
29 import org.yaml.snakeyaml.reader.UnicodeReader;
30 import org.yaml.snakeyaml.scanner.Scanner;
31 import org.yaml.snakeyaml.scanner.ScannerImpl;
32 import org.yaml.snakeyaml.tokens.AliasToken;
33 import org.yaml.snakeyaml.tokens.AnchorToken;
34 import org.yaml.snakeyaml.tokens.BlockEndToken;
35 import org.yaml.snakeyaml.tokens.BlockEntryToken;
36 import org.yaml.snakeyaml.tokens.BlockMappingStartToken;
37 import org.yaml.snakeyaml.tokens.BlockSequenceStartToken;
38 import org.yaml.snakeyaml.tokens.DirectiveToken;
39 import org.yaml.snakeyaml.tokens.DocumentEndToken;
40 import org.yaml.snakeyaml.tokens.DocumentStartToken;
41 import org.yaml.snakeyaml.tokens.FlowEntryToken;
42 import org.yaml.snakeyaml.tokens.FlowMappingEndToken;
43 import org.yaml.snakeyaml.tokens.FlowMappingStartToken;
44 import org.yaml.snakeyaml.tokens.FlowSequenceEndToken;
45 import org.yaml.snakeyaml.tokens.FlowSequenceStartToken;
46 import org.yaml.snakeyaml.tokens.KeyToken;
47 import org.yaml.snakeyaml.tokens.ScalarToken;
48 import org.yaml.snakeyaml.tokens.StreamEndToken;
49 import org.yaml.snakeyaml.tokens.StreamStartToken;
50 import org.yaml.snakeyaml.tokens.TagToken;
51 import org.yaml.snakeyaml.tokens.Token;
52 import org.yaml.snakeyaml.tokens.ValueToken;
53
54
55
56
57 public class PyTokensTest extends PyImportTest {
58
59 public void testTokens() throws FileNotFoundException {
60 Map<Class<?>, String> replaces = new HashMap<Class<?>, String>();
61 replaces.put(DirectiveToken.class, "%");
62 replaces.put(DocumentStartToken.class, "---");
63 replaces.put(DocumentEndToken.class, "...");
64 replaces.put(AliasToken.class, "*");
65 replaces.put(AnchorToken.class, "&");
66 replaces.put(TagToken.class, "!");
67 replaces.put(ScalarToken.class, "_");
68 replaces.put(BlockSequenceStartToken.class, "[[");
69 replaces.put(BlockMappingStartToken.class, "{{");
70 replaces.put(BlockEndToken.class, "]}");
71 replaces.put(FlowSequenceStartToken.class, "[");
72 replaces.put(FlowSequenceEndToken.class, "]");
73 replaces.put(FlowMappingStartToken.class, "{");
74 replaces.put(FlowMappingEndToken.class, "}");
75 replaces.put(BlockEntryToken.class, ",");
76 replaces.put(FlowEntryToken.class, ",");
77 replaces.put(KeyToken.class, "?");
78 replaces.put(ValueToken.class, ":");
79
80 File[] tokensFiles = getStreamsByExtension(".tokens");
81 assertTrue("No test files found.", tokensFiles.length > 0);
82 for (int i = 0; i < tokensFiles.length; i++) {
83 String name = tokensFiles[i].getName();
84 int position = name.lastIndexOf('.');
85 String dataName = name.substring(0, position) + ".data";
86
87 String tokenFileData = getResource(name);
88 String[] split = tokenFileData.split("\\s+");
89 List<String> tokens2 = new ArrayList<String>();
90 for (int j = 0; j < split.length; j++) {
91 tokens2.add(split[j]);
92 }
93
94 List<String> tokens1 = new ArrayList<String>();
95 StreamReader reader = new StreamReader(new UnicodeReader(new FileInputStream(
96 getFileByName(dataName))));
97 Scanner scanner = new ScannerImpl(reader);
98 try {
99 while (scanner.checkToken(new Token.ID[0])) {
100 Token token = scanner.getToken();
101 if (!(token instanceof StreamStartToken || token instanceof StreamEndToken)) {
102 String replacement = replaces.get(token.getClass());
103 tokens1.add(replacement);
104 }
105 }
106 assertEquals(tokenFileData, tokens1.size(), tokens2.size());
107 assertEquals(tokens1, tokens2);
108 } catch (RuntimeException e) {
109 System.out.println("File name: \n" + tokensFiles[i].getName());
110 String data = getResource(tokensFiles[i].getName());
111 System.out.println("Data: \n" + data);
112 System.out.println("Tokens:");
113 for (String token : tokens1) {
114 System.out.println(token);
115 }
116 fail("Cannot scan: " + tokensFiles[i]);
117 }
118 }
119 }
120
121 public void testScanner() throws IOException {
122 File[] files = getStreamsByExtension(".data", true);
123 assertTrue("No test files found.", files.length > 0);
124 for (File file : files) {
125 List<String> tokens = new ArrayList<String>();
126 InputStream input = new FileInputStream(file);
127 StreamReader reader = new StreamReader(new UnicodeReader(input));
128 Scanner scanner = new ScannerImpl(reader);
129 try {
130 while (scanner.checkToken(new Token.ID[0])) {
131 Token token = scanner.getToken();
132 tokens.add(token.getClass().getName());
133 }
134 } catch (RuntimeException e) {
135 System.out.println("File name: \n" + file.getName());
136 String data = getResource(file.getName());
137 System.out.println("Data: \n" + data);
138 System.out.println("Tokens:");
139 for (String token : tokens) {
140 System.out.println(token);
141 }
142 fail("Cannot scan: " + file + "; " + e.getMessage());
143 } finally {
144 input.close();
145 }
146 }
147 }
148 }