1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package com.puppycrawl.tools.checkstyle.meta;
21
22 import java.util.ArrayDeque;
23 import java.util.Arrays;
24 import java.util.Collections;
25 import java.util.Deque;
26 import java.util.HashMap;
27 import java.util.HashSet;
28 import java.util.LinkedHashSet;
29 import java.util.Locale;
30 import java.util.Map;
31 import java.util.Optional;
32 import java.util.Set;
33 import java.util.regex.Matcher;
34 import java.util.regex.Pattern;
35 import java.util.stream.Collectors;
36
37 import javax.xml.parsers.ParserConfigurationException;
38 import javax.xml.transform.TransformerException;
39
40 import com.puppycrawl.tools.checkstyle.FileStatefulCheck;
41 import com.puppycrawl.tools.checkstyle.api.DetailAST;
42 import com.puppycrawl.tools.checkstyle.api.DetailNode;
43 import com.puppycrawl.tools.checkstyle.api.JavadocTokenTypes;
44 import com.puppycrawl.tools.checkstyle.api.TokenTypes;
45 import com.puppycrawl.tools.checkstyle.checks.javadoc.AbstractJavadocCheck;
46 import com.puppycrawl.tools.checkstyle.utils.TokenUtil;
47
48
49
50
51 @FileStatefulCheck
52 public class JavadocMetadataScraper extends AbstractJavadocCheck {
53
54
55
56
57
58 public static final String MSG_DESC_MISSING = "javadocmetadatascraper.description.missing";
59
60
61 private static final Map<String, ModuleDetails> MODULE_DETAILS_STORE = new HashMap<>();
62
63
64 private static final Pattern PROPERTY_TAG = Pattern.compile("\\s*Property\\s*");
65
66
67 private static final Pattern TYPE_TAG = Pattern.compile("^ Type is\\s.*");
68
69
70 private static final Pattern VALIDATION_TYPE_TAG =
71 Pattern.compile("\\s.*Validation type is\\s.*");
72
73
74 private static final Pattern DEFAULT_VALUE_TAG = Pattern.compile("^ Default value is:*.*");
75
76
77 private static final Pattern EXAMPLES_TAG =
78 Pattern.compile("\\s*To configure the (default )?check.*");
79
80
81 private static final Pattern PARENT_TAG = Pattern.compile("\\s*Parent is\\s*");
82
83
84 private static final Pattern VIOLATION_MESSAGES_TAG =
85 Pattern.compile("\\s*Violation Message Keys:\\s*");
86
87
88 private static final Pattern TOKEN_TEXT_PATTERN = Pattern.compile("([A-Z_]{2,})+");
89
90
91 private static final Pattern DESC_CLEAN = Pattern.compile("-\\s");
92
93
94 private static final Pattern FILE_SEPARATOR_PATTERN =
95 Pattern.compile(Pattern.quote(System.getProperty("file.separator")));
96
97
98 private static final Pattern QUOTE_PATTERN = Pattern.compile("\"");
99
100
101 private static final String JAVA_FILE_EXTENSION = ".java";
102
103
104
105
106
107 private static final Set<String> PROPERTIES_TO_NOT_WRITE = Set.of(
108 "null",
109 "the charset property of the parent <a href=https://checkstyle.org/"
110 + "config.html#Checker>Checker</a> module");
111
112
113
114
115 private static final String PROP_TYPE_MISSING = "Type for property '%s' is missing";
116
117
118
119
120 private static final String PROP_DEFAULT_VALUE_MISSING =
121 "Default value for property '%s' is missing";
122
123
124 private ModuleDetails moduleDetails;
125
126
127
128
129
130 private boolean scrapingViolationMessageList;
131
132
133
134
135
136
137 private boolean toScan;
138
139
140 private DetailNode rootNode;
141
142
143
144
145
146 private int propertySectionStartIdx;
147
148
149
150
151
152 private int exampleSectionStartIdx;
153
154
155
156
157
158 private int parentSectionStartIdx;
159
160
161
162
163 private boolean writeXmlOutput = true;
164
165
166
167
168
169
170 public final void setWriteXmlOutput(boolean writeXmlOutput) {
171 this.writeXmlOutput = writeXmlOutput;
172 }
173
174 @Override
175 public int[] getDefaultJavadocTokens() {
176 return new int[] {
177 JavadocTokenTypes.JAVADOC,
178 JavadocTokenTypes.PARAGRAPH,
179 JavadocTokenTypes.LI,
180 JavadocTokenTypes.SINCE_LITERAL,
181 };
182 }
183
184 @Override
185 public int[] getRequiredJavadocTokens() {
186 return getAcceptableJavadocTokens();
187 }
188
189 @Override
190 public void beginJavadocTree(DetailNode rootAst) {
191 if (isTopLevelClassJavadoc()) {
192 moduleDetails = new ModuleDetails();
193 toScan = false;
194 scrapingViolationMessageList = false;
195 propertySectionStartIdx = -1;
196 exampleSectionStartIdx = -1;
197 parentSectionStartIdx = -1;
198
199 String moduleName = getModuleSimpleName();
200 final String checkModuleExtension = "Check";
201 if (moduleName.endsWith(checkModuleExtension)) {
202 moduleName = moduleName
203 .substring(0, moduleName.length() - checkModuleExtension.length());
204 }
205 moduleDetails.setName(moduleName);
206 moduleDetails.setFullQualifiedName(getPackageName(getFilePath()));
207 moduleDetails.setModuleType(getModuleType());
208 }
209 }
210
211 @Override
212 public void visitJavadocToken(DetailNode ast) {
213 if (toScan) {
214 scrapeContent(ast);
215 }
216
217 if (ast.getType() == JavadocTokenTypes.JAVADOC) {
218 final DetailAST parent = getParent(getBlockCommentAst());
219 if (parent.getType() == TokenTypes.CLASS_DEF) {
220 rootNode = ast;
221 toScan = true;
222 }
223 }
224 else if (ast.getType() == JavadocTokenTypes.SINCE_LITERAL) {
225 toScan = false;
226 }
227 }
228
229 @Override
230 public void finishJavadocTree(DetailNode rootAst) {
231 moduleDetails.setDescription(getDescriptionText());
232 if (isTopLevelClassJavadoc()) {
233 if (moduleDetails.getDescription().isEmpty()) {
234 final String fullQualifiedName = moduleDetails.getFullQualifiedName();
235 log(rootAst.getLineNumber(), MSG_DESC_MISSING,
236 fullQualifiedName.substring(fullQualifiedName.lastIndexOf('.') + 1));
237 }
238 else if (writeXmlOutput) {
239 try {
240 XmlMetaWriter.write(moduleDetails);
241 }
242 catch (TransformerException | ParserConfigurationException ex) {
243 throw new IllegalStateException(
244 "Failed to write metadata into XML file for module: "
245 + getModuleSimpleName(), ex);
246 }
247 }
248 if (!writeXmlOutput) {
249 MODULE_DETAILS_STORE.put(moduleDetails.getFullQualifiedName(), moduleDetails);
250 }
251
252 }
253 }
254
255
256
257
258
259
260
261 private void scrapeContent(DetailNode ast) {
262 if (ast.getType() == JavadocTokenTypes.PARAGRAPH) {
263 if (isParentText(ast)) {
264 parentSectionStartIdx = getParentIndexOf(ast);
265 moduleDetails.setParent(getParentText(ast));
266 }
267 else if (isViolationMessagesText(ast)) {
268 scrapingViolationMessageList = true;
269 }
270 else if (exampleSectionStartIdx == -1
271 && isExamplesText(ast)) {
272 exampleSectionStartIdx = getParentIndexOf(ast);
273 }
274 }
275 else if (ast.getType() == JavadocTokenTypes.LI) {
276 if (isPropertyList(ast)) {
277 if (propertySectionStartIdx == -1) {
278 propertySectionStartIdx = getParentIndexOf(ast);
279 }
280 moduleDetails.addToProperties(createProperties(ast));
281 }
282 else if (scrapingViolationMessageList) {
283 moduleDetails.addToViolationMessages(getViolationMessages(ast));
284 }
285 }
286 }
287
288
289
290
291
292
293
294 private static ModulePropertyDetails createProperties(DetailNode nodeLi) {
295 final ModulePropertyDetails modulePropertyDetails = new ModulePropertyDetails();
296
297 final Optional<DetailNode> propertyNameNode = getFirstChildOfType(nodeLi,
298 JavadocTokenTypes.JAVADOC_INLINE_TAG, 0);
299 if (propertyNameNode.isPresent()) {
300 final DetailNode propertyNameTag = propertyNameNode.orElseThrow();
301 final String propertyName = getTextFromTag(propertyNameTag);
302
303 final DetailNode propertyType = getFirstChildOfMatchingText(nodeLi, TYPE_TAG)
304 .orElseThrow(() -> {
305 return new MetadataGenerationException(String.format(
306 Locale.ROOT, PROP_TYPE_MISSING, propertyName)
307 );
308 });
309 final String propertyDesc = DESC_CLEAN.matcher(
310 constructSubTreeText(nodeLi, propertyNameTag.getIndex() + 1,
311 propertyType.getIndex() - 1))
312 .replaceAll(Matcher.quoteReplacement(""));
313
314 modulePropertyDetails.setDescription(propertyDesc.trim());
315 modulePropertyDetails.setName(propertyName);
316 modulePropertyDetails.setType(getTagTextFromProperty(nodeLi, propertyType));
317
318 final Optional<DetailNode> validationTypeNodeOpt = getFirstChildOfMatchingText(nodeLi,
319 VALIDATION_TYPE_TAG);
320 if (validationTypeNodeOpt.isPresent()) {
321 final DetailNode validationTypeNode = validationTypeNodeOpt.orElseThrow();
322 modulePropertyDetails.setValidationType(getTagTextFromProperty(nodeLi,
323 validationTypeNode));
324 }
325
326 final String defaultValue = getFirstChildOfMatchingText(nodeLi, DEFAULT_VALUE_TAG)
327 .map(defaultValueNode -> getPropertyDefaultText(nodeLi, defaultValueNode))
328 .orElseThrow(() -> {
329 return new MetadataGenerationException(String.format(
330 Locale.ROOT, PROP_DEFAULT_VALUE_MISSING, propertyName)
331 );
332 });
333 if (!PROPERTIES_TO_NOT_WRITE.contains(defaultValue)) {
334 modulePropertyDetails.setDefaultValue(defaultValue);
335 }
336 }
337 return modulePropertyDetails;
338 }
339
340
341
342
343
344
345
346
347 private static String getTagTextFromProperty(DetailNode nodeLi, DetailNode propertyMeta) {
348 final Optional<DetailNode> tagNodeOpt = getFirstChildOfType(nodeLi,
349 JavadocTokenTypes.JAVADOC_INLINE_TAG, propertyMeta.getIndex() + 1);
350 DetailNode tagNode = null;
351 if (tagNodeOpt.isPresent()) {
352 tagNode = tagNodeOpt.orElseThrow();
353 }
354 return getTextFromTag(tagNode);
355 }
356
357
358
359
360
361
362
363 private static String cleanDefaultTokensText(String initialText) {
364 final Set<String> tokens = new LinkedHashSet<>();
365 final Matcher matcher = TOKEN_TEXT_PATTERN.matcher(initialText);
366 while (matcher.find()) {
367 tokens.add(matcher.group(0));
368 }
369 return String.join(",", tokens);
370 }
371
372
373
374
375
376
377
378
379
380
381 private static String constructSubTreeText(DetailNode node, int childLeftLimit,
382 int childRightLimit) {
383 DetailNode detailNode = node;
384
385 final Deque<DetailNode> stack = new ArrayDeque<>();
386 stack.addFirst(detailNode);
387 final Set<DetailNode> visited = new HashSet<>();
388 final StringBuilder result = new StringBuilder(1024);
389 while (!stack.isEmpty()) {
390 detailNode = stack.removeFirst();
391
392 if (visited.add(detailNode)) {
393 final String childText = detailNode.getText();
394 if (detailNode.getType() != JavadocTokenTypes.LEADING_ASTERISK
395 && !TOKEN_TEXT_PATTERN.matcher(childText).matches()) {
396 result.insert(0, childText);
397 }
398 }
399
400 for (DetailNode child : detailNode.getChildren()) {
401 if (child.getParent().equals(node)
402 && (child.getIndex() < childLeftLimit
403 || child.getIndex() > childRightLimit)) {
404 continue;
405 }
406 if (!visited.contains(child)) {
407 stack.addFirst(child);
408 }
409 }
410 }
411 return result.toString().trim();
412 }
413
414
415
416
417
418
419
420
421 private String getDescriptionText() {
422 final int descriptionEndIdx;
423 if (propertySectionStartIdx > -1) {
424 descriptionEndIdx = propertySectionStartIdx;
425 }
426 else if (exampleSectionStartIdx > -1) {
427 descriptionEndIdx = exampleSectionStartIdx;
428 }
429 else {
430 descriptionEndIdx = parentSectionStartIdx;
431 }
432 return constructSubTreeText(rootNode, 0, descriptionEndIdx - 1);
433 }
434
435
436
437
438
439
440
441
442 private static String getPropertyDefaultText(DetailNode nodeLi, DetailNode defaultValueNode) {
443 final Optional<DetailNode> propertyDefaultValueTag = getFirstChildOfType(nodeLi,
444 JavadocTokenTypes.JAVADOC_INLINE_TAG, defaultValueNode.getIndex() + 1);
445 final String result;
446 if (propertyDefaultValueTag.isPresent()) {
447 result = getTextFromTag(propertyDefaultValueTag.orElseThrow());
448 }
449 else {
450 final String tokenText = constructSubTreeText(nodeLi,
451 defaultValueNode.getIndex(), nodeLi.getChildren().length);
452 result = cleanDefaultTokensText(tokenText);
453 }
454 return result;
455 }
456
457
458
459
460
461
462
463 private static String getViolationMessages(DetailNode nodeLi) {
464 final Optional<DetailNode> resultNode = getFirstChildOfType(nodeLi,
465 JavadocTokenTypes.JAVADOC_INLINE_TAG, 0);
466 return resultNode.map(JavadocMetadataScraper::getTextFromTag).orElse("");
467 }
468
469
470
471
472
473
474
475 private static String getTextFromTag(DetailNode nodeTag) {
476 return Optional.ofNullable(nodeTag).map(JavadocMetadataScraper::getText).orElse("");
477 }
478
479
480
481
482
483
484
485
486
487
488 private static Optional<DetailNode> getFirstChildOfType(DetailNode node, int tokenType,
489 int offset) {
490 return Arrays.stream(node.getChildren())
491 .filter(child -> child.getIndex() >= offset && child.getType() == tokenType)
492 .findFirst();
493 }
494
495
496
497
498
499
500
501 private static String getText(DetailNode parentNode) {
502 return Arrays.stream(parentNode.getChildren())
503 .filter(child -> child.getType() == JavadocTokenTypes.TEXT)
504 .map(node -> QUOTE_PATTERN.matcher(node.getText().trim()).replaceAll(""))
505 .collect(Collectors.joining(" "));
506 }
507
508
509
510
511
512
513
514
515 private static Optional<DetailNode> getFirstChildOfMatchingText(DetailNode node,
516 Pattern pattern) {
517 return Arrays.stream(node.getChildren())
518 .filter(child -> pattern.matcher(child.getText()).matches())
519 .findFirst();
520 }
521
522
523
524
525
526
527
528 private static DetailAST getParent(DetailAST commentBlock) {
529 final DetailAST parentNode = commentBlock.getParent();
530 DetailAST result = parentNode;
531 if (result.getType() == TokenTypes.ANNOTATION) {
532 result = parentNode.getParent().getParent();
533 }
534 else if (result.getType() == TokenTypes.MODIFIERS) {
535 result = parentNode.getParent();
536 }
537 return result;
538 }
539
540
541
542
543
544
545
546
547 private static int getParentIndexOf(DetailNode node) {
548 DetailNode currNode = node;
549 while (currNode.getParent().getIndex() != -1) {
550 currNode = currNode.getParent();
551 }
552 return currNode.getIndex();
553 }
554
555
556
557
558
559
560
561 private static String getParentText(DetailNode nodeParagraph) {
562 return getFirstChildOfType(nodeParagraph, JavadocTokenTypes.JAVADOC_INLINE_TAG, 0)
563 .map(JavadocMetadataScraper::getTextFromTag)
564 .orElse(null);
565 }
566
567
568
569
570
571
572 private ModuleType getModuleType() {
573 final String simpleModuleName = getModuleSimpleName();
574 final ModuleType result;
575 if (simpleModuleName.endsWith("FileFilter")) {
576 result = ModuleType.FILEFILTER;
577 }
578 else if (simpleModuleName.endsWith("Filter")) {
579 result = ModuleType.FILTER;
580 }
581 else {
582 result = ModuleType.CHECK;
583 }
584 return result;
585 }
586
587
588
589
590
591
592 private String getModuleSimpleName() {
593 final String fullFileName = getFilePath();
594 final String[] pathTokens = FILE_SEPARATOR_PATTERN.split(fullFileName);
595 final String fileName = pathTokens[pathTokens.length - 1];
596 return fileName.substring(0, fileName.length() - JAVA_FILE_EXTENSION.length());
597 }
598
599
600
601
602
603
604
605 private static String getPackageName(String filePath) {
606 final Deque<String> result = new ArrayDeque<>();
607 final String[] filePathTokens = FILE_SEPARATOR_PATTERN.split(filePath);
608 for (int i = filePathTokens.length - 1; i >= 0; i--) {
609 if ("java".equals(filePathTokens[i]) || "resources".equals(filePathTokens[i])) {
610 break;
611 }
612 result.addFirst(filePathTokens[i]);
613 }
614 final String fileName = result.removeLast();
615 result.addLast(fileName.substring(0, fileName.length() - JAVA_FILE_EXTENSION.length()));
616 return String.join(".", result);
617 }
618
619
620
621
622
623
624 public static Map<String, ModuleDetails> getModuleDetailsStore() {
625 return Collections.unmodifiableMap(MODULE_DETAILS_STORE);
626 }
627
628
629 public static void resetModuleDetailsStore() {
630 MODULE_DETAILS_STORE.clear();
631 }
632
633
634
635
636
637
638
639 private boolean isTopLevelClassJavadoc() {
640 final DetailAST parent = getParent(getBlockCommentAst());
641 final Optional<DetailAST> className = TokenUtil
642 .findFirstTokenByPredicate(parent, child -> {
643 return parent.getType() == TokenTypes.CLASS_DEF
644 && child.getType() == TokenTypes.IDENT;
645 });
646 return className.isPresent()
647 && getModuleSimpleName().equals(className.orElseThrow().getText());
648 }
649
650
651
652
653
654
655
656 private static boolean isExamplesText(DetailNode ast) {
657 return isChildNodeTextMatches(ast, EXAMPLES_TAG);
658 }
659
660
661
662
663
664
665
666 private static boolean isPropertyList(DetailNode nodeLi) {
667 return isChildNodeTextMatches(nodeLi, PROPERTY_TAG);
668 }
669
670
671
672
673
674
675
676
677 private static boolean isViolationMessagesText(DetailNode nodeParagraph) {
678 return isChildNodeTextMatches(nodeParagraph, VIOLATION_MESSAGES_TAG);
679 }
680
681
682
683
684
685
686
687
688 private static boolean isParentText(DetailNode nodeParagraph) {
689 return isChildNodeTextMatches(nodeParagraph, PARENT_TAG);
690 }
691
692
693
694
695
696
697
698
699 private static boolean isChildNodeTextMatches(DetailNode ast, Pattern pattern) {
700 return getFirstChildOfType(ast, JavadocTokenTypes.TEXT, 0)
701 .map(DetailNode::getText)
702 .map(pattern::matcher)
703 .map(Matcher::matches)
704 .orElse(Boolean.FALSE);
705 }
706 }