1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.pyyaml;
18
19 import java.io.File;
20 import java.io.FileInputStream;
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.util.ArrayList;
25 import java.util.HashMap;
26 import java.util.List;
27 import java.util.Map;
28
29 import org.yaml.snakeyaml.reader.StreamReader;
30 import org.yaml.snakeyaml.reader.UnicodeReader;
31 import org.yaml.snakeyaml.scanner.Scanner;
32 import org.yaml.snakeyaml.scanner.ScannerImpl;
33 import org.yaml.snakeyaml.tokens.AliasToken;
34 import org.yaml.snakeyaml.tokens.AnchorToken;
35 import org.yaml.snakeyaml.tokens.BlockEndToken;
36 import org.yaml.snakeyaml.tokens.BlockEntryToken;
37 import org.yaml.snakeyaml.tokens.BlockMappingStartToken;
38 import org.yaml.snakeyaml.tokens.BlockSequenceStartToken;
39 import org.yaml.snakeyaml.tokens.DirectiveToken;
40 import org.yaml.snakeyaml.tokens.DocumentEndToken;
41 import org.yaml.snakeyaml.tokens.DocumentStartToken;
42 import org.yaml.snakeyaml.tokens.FlowEntryToken;
43 import org.yaml.snakeyaml.tokens.FlowMappingEndToken;
44 import org.yaml.snakeyaml.tokens.FlowMappingStartToken;
45 import org.yaml.snakeyaml.tokens.FlowSequenceEndToken;
46 import org.yaml.snakeyaml.tokens.FlowSequenceStartToken;
47 import org.yaml.snakeyaml.tokens.KeyToken;
48 import org.yaml.snakeyaml.tokens.ScalarToken;
49 import org.yaml.snakeyaml.tokens.StreamEndToken;
50 import org.yaml.snakeyaml.tokens.StreamStartToken;
51 import org.yaml.snakeyaml.tokens.TagToken;
52 import org.yaml.snakeyaml.tokens.Token;
53 import org.yaml.snakeyaml.tokens.ValueToken;
54
55
56
57
58 public class PyTokensTest extends PyImportTest {
59
60 public void testTokens() throws FileNotFoundException {
61 Map<Class<?>, String> replaces = new HashMap<Class<?>, String>();
62 replaces.put(DirectiveToken.class, "%");
63 replaces.put(DocumentStartToken.class, "---");
64 replaces.put(DocumentEndToken.class, "...");
65 replaces.put(AliasToken.class, "*");
66 replaces.put(AnchorToken.class, "&");
67 replaces.put(TagToken.class, "!");
68 replaces.put(ScalarToken.class, "_");
69 replaces.put(BlockSequenceStartToken.class, "[[");
70 replaces.put(BlockMappingStartToken.class, "{{");
71 replaces.put(BlockEndToken.class, "]}");
72 replaces.put(FlowSequenceStartToken.class, "[");
73 replaces.put(FlowSequenceEndToken.class, "]");
74 replaces.put(FlowMappingStartToken.class, "{");
75 replaces.put(FlowMappingEndToken.class, "}");
76 replaces.put(BlockEntryToken.class, ",");
77 replaces.put(FlowEntryToken.class, ",");
78 replaces.put(KeyToken.class, "?");
79 replaces.put(ValueToken.class, ":");
80
81 File[] tokensFiles = getStreamsByExtension(".tokens");
82 assertTrue("No test files found.", tokensFiles.length > 0);
83 for (int i = 0; i < tokensFiles.length; i++) {
84 String name = tokensFiles[i].getName();
85 int position = name.lastIndexOf('.');
86 String dataName = name.substring(0, position) + ".data";
87
88 String tokenFileData = getResource(name);
89 String[] split = tokenFileData.split("\\s+");
90 List<String> tokens2 = new ArrayList<String>();
91 for (int j = 0; j < split.length; j++) {
92 tokens2.add(split[j]);
93 }
94
95 List<String> tokens1 = new ArrayList<String>();
96 StreamReader reader = new StreamReader(new UnicodeReader(new FileInputStream(
97 getFileByName(dataName))));
98 Scanner scanner = new ScannerImpl(reader);
99 try {
100 while (scanner.checkToken(new Token.ID[0])) {
101 Token token = scanner.getToken();
102 if (!(token instanceof StreamStartToken || token instanceof StreamEndToken)) {
103 String replacement = replaces.get(token.getClass());
104 tokens1.add(replacement);
105 }
106 }
107 assertEquals(tokenFileData, tokens1.size(), tokens2.size());
108 assertEquals(tokens1, tokens2);
109 } catch (RuntimeException e) {
110 System.out.println("File name: \n" + tokensFiles[i].getName());
111 String data = getResource(tokensFiles[i].getName());
112 System.out.println("Data: \n" + data);
113 System.out.println("Tokens:");
114 for (String token : tokens1) {
115 System.out.println(token);
116 }
117 fail("Cannot scan: " + tokensFiles[i]);
118 }
119 }
120 }
121
122 public void testScanner() throws IOException {
123 File[] files = getStreamsByExtension(".data", true);
124 assertTrue("No test files found.", files.length > 0);
125 for (File file : files) {
126 List<String> tokens = new ArrayList<String>();
127 InputStream input = new FileInputStream(file);
128 StreamReader reader = new StreamReader(new UnicodeReader(input));
129 Scanner scanner = new ScannerImpl(reader);
130 try {
131 while (scanner.checkToken(new Token.ID[0])) {
132 Token token = scanner.getToken();
133 tokens.add(token.getClass().getName());
134 }
135 } catch (RuntimeException e) {
136 System.out.println("File name: \n" + file.getName());
137 String data = getResource(file.getName());
138 System.out.println("Data: \n" + data);
139 System.out.println("Tokens:");
140 for (String token : tokens) {
141 System.out.println(token);
142 }
143 fail("Cannot scan: " + file + "; " + e.getMessage());
144 } finally {
145 input.close();
146 }
147 }
148 }
149 }