1 /**
2 * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3 */
4 package net.sourceforge.pmd.cpd;
5
6 import net.sourceforge.pmd.TargetJDK1_4;
7 import net.sourceforge.pmd.ast.JavaParserConstants;
8 import net.sourceforge.pmd.ast.JavaParserTokenManager;
9 import net.sourceforge.pmd.ast.Token;
10
11 import java.io.StringReader;
12 import java.util.Properties;
13
14 public class JavaTokenizer implements Tokenizer {
15
16 public static final String IGNORE_LITERALS = "ignore_literals";
17 public static final String IGNORE_IDENTIFIERS = "ignore_identifiers";
18
19 private boolean ignoreLiterals;
20 private boolean ignoreIdentifiers;
21
22 public void setProperties(Properties properties) {
23 ignoreLiterals = Boolean.parseBoolean(properties.getProperty(IGNORE_LITERALS, "false"));
24 ignoreIdentifiers = Boolean.parseBoolean(properties.getProperty(IGNORE_IDENTIFIERS, "false"));
25 }
26
27 public void tokenize(SourceCode tokens, Tokens tokenEntries) {
28 StringBuffer buffer = tokens.getCodeBuffer();
29
30
31
32
33
34
35
36
37 JavaParserTokenManager tokenMgr = new TargetJDK1_4().createJavaParserTokenManager(new StringReader(buffer.toString()));
38 Token currentToken = tokenMgr.getNextToken();
39 boolean inDiscardingState = false;
40 while (currentToken.image.length() > 0) {
41 if (currentToken.kind == JavaParserConstants.IMPORT || currentToken.kind == JavaParserConstants.PACKAGE) {
42 inDiscardingState = true;
43 currentToken = tokenMgr.getNextToken();
44 continue;
45 }
46
47 if (inDiscardingState && currentToken.kind == JavaParserConstants.SEMICOLON) {
48 inDiscardingState = false;
49 }
50
51 if (inDiscardingState) {
52 currentToken = tokenMgr.getNextToken();
53 continue;
54 }
55
56 if (currentToken.kind != JavaParserConstants.SEMICOLON) {
57 String image = currentToken.image;
58 if (ignoreLiterals && (currentToken.kind == JavaParserConstants.STRING_LITERAL || currentToken.kind == JavaParserConstants.CHARACTER_LITERAL
59 || currentToken.kind == JavaParserConstants.DECIMAL_LITERAL || currentToken.kind == JavaParserConstants.FLOATING_POINT_LITERAL)) {
60 image = String.valueOf(currentToken.kind);
61 }
62 if (ignoreIdentifiers && currentToken.kind == JavaParserConstants.IDENTIFIER) {
63 image = String.valueOf(currentToken.kind);
64 }
65 tokenEntries.add(new TokenEntry(image, tokens.getFileName(), currentToken.beginLine));
66 }
67
68 currentToken = tokenMgr.getNextToken();
69 }
70 tokenEntries.add(TokenEntry.getEOF());
71 }
72
73 public void setIgnoreLiterals(boolean ignore) {
74 this.ignoreLiterals = ignore;
75 }
76
77 public void setIgnoreIdentifiers(boolean ignore) {
78 this.ignoreIdentifiers = ignore;
79 }
80 }