1 /***
2 * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3 */
4 package net.sourceforge.pmd.cpd;
5
6 import net.sourceforge.pmd.cpd.cppast.CPPParser;
7 import net.sourceforge.pmd.cpd.cppast.CPPParserTokenManager;
8 import net.sourceforge.pmd.cpd.cppast.Token;
9 import net.sourceforge.pmd.cpd.cppast.TokenMgrError;
10
11 import java.io.StringReader;
12 import java.util.ArrayList;
13 import java.util.List;
14
15 public class CPPTokenizer implements Tokenizer {
16 protected String EOL = System.getProperty("line.separator", "\n");
17
18 private static boolean initted;
19
20 public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
21 StringBuffer sb = sourceCode.getCodeBuffer();
22 try {
23 if (!initted) {
24 new CPPParser(new StringReader(sb.toString()));
25 initted = true;
26 }
27 CPPParser.ReInit(new StringReader(sb.toString()));
28 Token currToken = CPPParserTokenManager.getNextToken();
29 while (currToken.image.length() > 0) {
30 tokenEntries.add(new TokenEntry(currToken.image, sourceCode.getFileName(), currToken.beginLine));
31 currToken = CPPParserTokenManager.getNextToken();
32 }
33 tokenEntries.add(TokenEntry.getEOF());
34 System.out.println("Added " + sourceCode.getFileName());
35 } catch (TokenMgrError err) {
36 System.out.println("Skipping " + sourceCode.getFileName() + " due to parse error");
37 List emptyCode = new ArrayList();
38 emptyCode.add("");
39 tokenEntries.add(TokenEntry.getEOF());
40 }
41 }
42 }