View Javadoc

1   /**
2    * Copyright (c) 2008-2011, http://www.snakeyaml.org
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  
17  package org.pyyaml;
18  
19  import java.io.File;
20  import java.io.FileInputStream;
21  import java.io.FileNotFoundException;
22  import java.io.IOException;
23  import java.io.InputStream;
24  import java.util.ArrayList;
25  import java.util.HashMap;
26  import java.util.List;
27  import java.util.Map;
28  
29  import org.yaml.snakeyaml.reader.StreamReader;
30  import org.yaml.snakeyaml.reader.UnicodeReader;
31  import org.yaml.snakeyaml.scanner.Scanner;
32  import org.yaml.snakeyaml.scanner.ScannerImpl;
33  import org.yaml.snakeyaml.tokens.AliasToken;
34  import org.yaml.snakeyaml.tokens.AnchorToken;
35  import org.yaml.snakeyaml.tokens.BlockEndToken;
36  import org.yaml.snakeyaml.tokens.BlockEntryToken;
37  import org.yaml.snakeyaml.tokens.BlockMappingStartToken;
38  import org.yaml.snakeyaml.tokens.BlockSequenceStartToken;
39  import org.yaml.snakeyaml.tokens.DirectiveToken;
40  import org.yaml.snakeyaml.tokens.DocumentEndToken;
41  import org.yaml.snakeyaml.tokens.DocumentStartToken;
42  import org.yaml.snakeyaml.tokens.FlowEntryToken;
43  import org.yaml.snakeyaml.tokens.FlowMappingEndToken;
44  import org.yaml.snakeyaml.tokens.FlowMappingStartToken;
45  import org.yaml.snakeyaml.tokens.FlowSequenceEndToken;
46  import org.yaml.snakeyaml.tokens.FlowSequenceStartToken;
47  import org.yaml.snakeyaml.tokens.KeyToken;
48  import org.yaml.snakeyaml.tokens.ScalarToken;
49  import org.yaml.snakeyaml.tokens.StreamEndToken;
50  import org.yaml.snakeyaml.tokens.StreamStartToken;
51  import org.yaml.snakeyaml.tokens.TagToken;
52  import org.yaml.snakeyaml.tokens.Token;
53  import org.yaml.snakeyaml.tokens.ValueToken;
54  
55  /**
56   * @see imported from PyYAML
57   */
58  public class PyTokensTest extends PyImportTest {
59  
60      public void testTokens() throws FileNotFoundException {
61          Map<Class<?>, String> replaces = new HashMap<Class<?>, String>();
62          replaces.put(DirectiveToken.class, "%");
63          replaces.put(DocumentStartToken.class, "---");
64          replaces.put(DocumentEndToken.class, "...");
65          replaces.put(AliasToken.class, "*");
66          replaces.put(AnchorToken.class, "&");
67          replaces.put(TagToken.class, "!");
68          replaces.put(ScalarToken.class, "_");
69          replaces.put(BlockSequenceStartToken.class, "[[");
70          replaces.put(BlockMappingStartToken.class, "{{");
71          replaces.put(BlockEndToken.class, "]}");
72          replaces.put(FlowSequenceStartToken.class, "[");
73          replaces.put(FlowSequenceEndToken.class, "]");
74          replaces.put(FlowMappingStartToken.class, "{");
75          replaces.put(FlowMappingEndToken.class, "}");
76          replaces.put(BlockEntryToken.class, ",");
77          replaces.put(FlowEntryToken.class, ",");
78          replaces.put(KeyToken.class, "?");
79          replaces.put(ValueToken.class, ":");
80          //
81          File[] tokensFiles = getStreamsByExtension(".tokens");
82          assertTrue("No test files found.", tokensFiles.length > 0);
83          for (int i = 0; i < tokensFiles.length; i++) {
84              String name = tokensFiles[i].getName();
85              int position = name.lastIndexOf('.');
86              String dataName = name.substring(0, position) + ".data";
87              //
88              String tokenFileData = getResource(name);
89              String[] split = tokenFileData.split("\\s+");
90              List<String> tokens2 = new ArrayList<String>();
91              for (int j = 0; j < split.length; j++) {
92                  tokens2.add(split[j]);
93              }
94              //
95              List<String> tokens1 = new ArrayList<String>();
96              StreamReader reader = new StreamReader(new UnicodeReader(new FileInputStream(
97                      getFileByName(dataName))));
98              Scanner scanner = new ScannerImpl(reader);
99              try {
100                 while (scanner.checkToken(new Token.ID[0])) {
101                     Token token = scanner.getToken();
102                     if (!(token instanceof StreamStartToken || token instanceof StreamEndToken)) {
103                         String replacement = replaces.get(token.getClass());
104                         tokens1.add(replacement);
105                     }
106                 }
107                 assertEquals(tokenFileData, tokens1.size(), tokens2.size());
108                 assertEquals(tokens1, tokens2);
109             } catch (RuntimeException e) {
110                 System.out.println("File name: \n" + tokensFiles[i].getName());
111                 String data = getResource(tokensFiles[i].getName());
112                 System.out.println("Data: \n" + data);
113                 System.out.println("Tokens:");
114                 for (String token : tokens1) {
115                     System.out.println(token);
116                 }
117                 fail("Cannot scan: " + tokensFiles[i]);
118             }
119         }
120     }
121 
122     public void testScanner() throws IOException {
123         File[] files = getStreamsByExtension(".data", true);
124         assertTrue("No test files found.", files.length > 0);
125         for (File file : files) {
126             List<String> tokens = new ArrayList<String>();
127             InputStream input = new FileInputStream(file);
128             StreamReader reader = new StreamReader(new UnicodeReader(input));
129             Scanner scanner = new ScannerImpl(reader);
130             try {
131                 while (scanner.checkToken(new Token.ID[0])) {
132                     Token token = scanner.getToken();
133                     tokens.add(token.getClass().getName());
134                 }
135             } catch (RuntimeException e) {
136                 System.out.println("File name: \n" + file.getName());
137                 String data = getResource(file.getName());
138                 System.out.println("Data: \n" + data);
139                 System.out.println("Tokens:");
140                 for (String token : tokens) {
141                     System.out.println(token);
142                 }
143                 fail("Cannot scan: " + file + "; " + e.getMessage());
144             } finally {
145                 input.close();
146             }
147         }
148     }
149 }