1 /** 2 * BSD-style license; for more info see http://pmd.sourceforge.net/license.html 3 */ 4 package net.sourceforge.pmd.cpd; 5 6 import net.sourceforge.pmd.cpd.cppast.CPPParserTokenManager; 7 import net.sourceforge.pmd.cpd.cppast.SimpleCharStream; 8 import net.sourceforge.pmd.cpd.cppast.Token; 9 import net.sourceforge.pmd.cpd.cppast.TokenMgrError; 10 11 import java.io.StringReader; 12 13 public class CPPTokenizer implements Tokenizer { 14 private static SimpleCharStream charStream; 15 16 public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { 17 StringBuffer sb = sourceCode.getCodeBuffer(); 18 try { 19 if (charStream == null) { 20 charStream = new SimpleCharStream(new StringReader(sb.toString())); 21 } else { 22 charStream.ReInit(new StringReader(sb.toString())); 23 } 24 CPPParserTokenManager.ReInit(charStream); 25 CPPParserTokenManager.setFileName(sourceCode.getFileName()); 26 Token currToken = CPPParserTokenManager.getNextToken(); 27 while (currToken.image.length() > 0) { 28 tokenEntries.add(new TokenEntry(currToken.image, sourceCode.getFileName(), currToken.beginLine)); 29 currToken = CPPParserTokenManager.getNextToken(); 30 } 31 tokenEntries.add(TokenEntry.getEOF()); 32 System.err.println("Added " + sourceCode.getFileName()); 33 } catch (TokenMgrError err) { 34 err.printStackTrace(); 35 System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error"); 36 tokenEntries.add(TokenEntry.getEOF()); 37 } 38 } 39 }