1 module served.utils.ddoc;
2 
3 import served.lsp.protocol;
4 
5 import ddoc;
6 
7 import std.algorithm;
8 import std.array;
9 import std.format;
10 import std.range.primitives;
11 import std.string;
12 import std.uni : sicmp;
13 
14 public import ddoc : Comment;
15 
16 /**
17  * A test function for checking `DDoc` parsing
18  *
19  * Params:
20  *     hello = a string
21  *     world = an integer
22  *
23  * Author: Jonny
24  * Bugs: None
25  * ---
26  * import std.stdio;
27  *
28  * int main(string[] args) {
29  * 	   writeln("Testing inline code")
30  * }
31  * ---
32  */
33 private int testFunction(string foo, int bar)
34 {
35 	import std.stdio : writeln;
36 
37 	writeln(foo, bar);
38 	return 0;
39 }
40 
41 /**
42  * Parses a ddoc string into a divided comment.
43  * Returns: A comment if the ddoc could be parsed or Comment.init if it couldn't be parsed and throwError is false.
44  * Throws: Exception if comment has ddoc syntax errors.
45  * Params:
46  * 	ddoc = the documentation string as given by the user without any comment markers
47  * 	throwError = set to true to make parsing errors throw
48  */
49 Comment parseDdoc(string ddoc, bool throwError = false)
50 {
51 	if (ddoc.length == 0)
52 		return Comment.init;
53 
54 	if (throwError)
55 		return parseComment(prepareDDoc(ddoc), markdownMacros, false);
56 	else
57 	{
58 		try
59 		{
60 			return parseComment(prepareDDoc(ddoc), markdownMacros, false);
61 		}
62 		catch (Exception e)
63 		{
64 			return Comment.init;
65 		}
66 	}
67 }
68 
69 /**
70  * Convert a Ddoc comment string to markdown. Returns ddoc string back if it is
71  * not valid.
72  * Params:
73  *		ddoc = string of a valid Ddoc comment.
74  */
75 string ddocToMarkdown(string ddoc)
76 {
77 	// Parse ddoc. Return if exception.
78 	Comment comment;
79 	try
80 	{
81 		comment = parseComment(prepareDDoc(ddoc), markdownMacros, false);
82 	}
83 	catch (Exception e)
84 	{
85 		return ddoc;
86 	}
87 	return ddocToMarkdown(comment);
88 }
89 
90 /// ditto
91 string ddocToMarkdown(const Comment comment)
92 {
93 	auto output = "";
94 	foreach (section; comment.sections)
95 	{
96 		import std.uni : toLower;
97 
98 		string content = section.content.postProcessContent;
99 		switch (section.name.toLower)
100 		{
101 		case "":
102 		case "summary":
103 			output ~= content ~ "\n\n";
104 			break;
105 		case "params":
106 			output ~= "**Params**\n\n";
107 			foreach (parameter; section.mapping)
108 			{
109 				output ~= format!"`%s` %s\n\n"(parameter[0].postProcessContent,
110 						parameter[1].postProcessContent);
111 			}
112 			break;
113 		case "author":
114 		case "authors":
115 		case "bugs":
116 		case "date":
117 		case "deprecated":
118 		case "history":
119 		default:
120 			// Single line sections go on the same line as section titles. Multi
121 			// line sections go on the line below.
122 			import std.algorithm : canFind;
123 
124 			content = content.chomp();
125 			if (!content.canFind("\n"))
126 			{
127 				output ~= format!"**%s** — %s\n\n"(section.name, content);
128 			}
129 			else
130 			{
131 				output ~= format!"**%s**\n\n%s\n\n"(section.name, content);
132 			}
133 			break;
134 		}
135 	}
136 	return output.replace("$", "$");
137 }
138 
139 /// Removes leading */+ characters from each line per section if the entire section only consists of them. Sections are separated with lines starting with ---
140 private string preProcessContent(string content)
141 {
142 	bool hasLeadingStarOrPlus;
143 	foreach (chunk; content.lineSplitter!(KeepTerminator.yes)
144 			.chunkBy!(a => a.startsWith("---")))
145 	{
146 		foreach (line; chunk[1])
147 		{
148 			if (line.stripLeft.startsWith("*", "+"))
149 			{
150 				hasLeadingStarOrPlus = true;
151 				break;
152 			}
153 		}
154 
155 		if (hasLeadingStarOrPlus)
156 			break;
157 	}
158 
159 	if (!hasLeadingStarOrPlus)
160 		return content; // no leading * or + characters, no preprocessing needed.
161 
162 	auto newContent = appender!string();
163 	newContent.reserve(content.length);
164 	foreach (chunk; content.lineSplitter!(KeepTerminator.yes)
165 			.chunkBy!(a => a.startsWith("---")))
166 	{
167 		auto c = chunk[1].save;
168 
169 		bool isStrippable = true;
170 		foreach (line; c)
171 		{
172 			auto l = line.stripLeft;
173 			if (!l.length)
174 				continue;
175 			if (!l.startsWith("*", "+"))
176 			{
177 				isStrippable = false;
178 				break;
179 			}
180 		}
181 
182 		if (isStrippable)
183 		{
184 			foreach (line; chunk[1])
185 			{
186 				auto stripped = line.stripLeft;
187 				if (!stripped.length)
188 					stripped = line;
189 
190 				if (stripped.startsWith("* ", "+ ", "*\t", "+\t"))
191 					newContent.put(stripped[2 .. $]);
192 				else if (stripped.startsWith("*", "+"))
193 					newContent.put(stripped[1 .. $]);
194 				else
195 					newContent.put(line);
196 			}
197 		}
198 		else
199 			foreach (line; chunk[1])
200 				newContent.put(line);
201 	}
202 	return newContent.data;
203 }
204 
205 unittest
206 {
207 	string noChange = `Params:
208 	a = this does things
209 	b = this does too
210 
211 Examples:
212 ---
213 foo(a, b);
214 ---
215 `;
216 	assert(preProcessContent(noChange) is noChange);
217 
218 	assert(preProcessContent(`* Params:
219 *     a = this does things
220 *     b = this does too
221 *
222 * cool.`) == `Params:
223     a = this does things
224     b = this does too
225 
226 cool.`);
227 }
228 
229 /// Fixes code-d specific placeholders inserted during ddoc translation for better IDE integration.
230 private string postProcessContent(string content)
231 {
232 	while (true)
233 	{
234 		auto index = content.indexOf(inlineRefPrefix);
235 		if (index != -1)
236 		{
237 			auto end = content.indexOf('.', index + inlineRefPrefix.length);
238 			if (end == -1)
239 				break; // malformed
240 			content = content[0 .. index]
241 				~ content[index + inlineRefPrefix.length .. end].postProcessInlineRefPrefix
242 				~ content[end .. $];
243 		}
244 
245 		if (index == -1)
246 			break;
247 	}
248 	return content;
249 }
250 
251 private string postProcessInlineRefPrefix(string content)
252 {
253 	return content.splitter(',').map!strip.join('.');
254 }
255 
256 /**
257  * Convert a DDoc comment string to MarkedString (as defined in the language
258  * server spec)
259  * Params:
260  *		ddoc = A DDoc string to be converted to Markdown
261  */
262 MarkedString[] ddocToMarked(string ddoc)
263 {
264 	MarkedString[] ret;
265 	if (!ddoc.length)
266 		return ret;
267 	return markdownToMarked(ddoc.ddocToMarkdown);
268 }
269 
270 /// ditto
271 MarkedString[] ddocToMarked(const Comment comment)
272 {
273 	MarkedString[] ret;
274 	if (comment == Comment.init)
275 		return ret;
276 	return markdownToMarked(comment.ddocToMarkdown);
277 }
278 
279 /**
280  * Converts markdown code to MarkedString blocks as determined by D code blocks.
281  */
282 MarkedString[] markdownToMarked(string md)
283 {
284 	MarkedString[] ret;
285 	if (!md.length)
286 		return ret;
287 
288 	ret ~= MarkedString("");
289 
290 	foreach (line; md.lineSplitter!(KeepTerminator.yes))
291 	{
292 		if (line.strip == "```d")
293 			ret ~= MarkedString("", "d");
294 		else if (line.strip == "```")
295 			ret ~= MarkedString("", "text");
296 		else
297 			ret[$ - 1].value ~= line;
298 	}
299 
300 	if (ret.length >= 2 && !ret[$ - 1].value.strip.length)
301 		ret = ret[0 .. $ - 1];
302 
303 	return ret;
304 }
305 
306 /**
307  * Returns: the params section in a ddoc comment as key value pair. Or null if not found.
308  */
309 inout(KeyValuePair[]) getParams(inout Comment comment)
310 {
311 	foreach (section; comment.sections)
312 		if (section.name.sicmp("params") == 0)
313 			return section.mapping;
314 	return null;
315 }
316 
317 /**
318  * Returns: documentation for a given parameter in the params section of a documentation comment. Or null if not found.
319  */
320 string getParamDocumentation(const Comment comment, string searchParam)
321 {
322 	foreach (param; getParams(comment))
323 		if (param[0] == searchParam)
324 			return param[1];
325 	return null;
326 }
327 
328 /**
329  * Performs preprocessing of the document. Wraps code blocks in macros.
330  * Params:
331  * 		str = This is one of the params
332  */
333 private string prepareDDoc(string str)
334 {
335 	import ddoc.lexer : Lexer;
336 
337 	str = str.preProcessContent;
338 
339 	auto lex = Lexer(str, true);
340 	string output;
341 	foreach (tok; lex)
342 	{
343 		if (tok.type == Type.embedded || tok.type == Type.inlined)
344 		{
345 			// Add newlines before documentation
346 			if (tok.type == Type.embedded)
347 			{
348 				output ~= "\n\n";
349 			}
350 			output ~= tok.type == Type.embedded ? "$(D_CODE " : "$(DDOC_BACKQUOTED ";
351 			output ~= tok.text;
352 			output ~= ")";
353 		}
354 		else
355 		{
356 			output ~= tok.text;
357 		}
358 	}
359 	return output;
360 }
361 
362 static immutable inlineRefPrefix = "__CODED_INLINE_REF__:";
363 
364 string[string] markdownMacros;
365 static this()
366 {
367 	markdownMacros = [
368 		`B`: `**$0**`,
369 		`I`: `*$0*`,
370 		`U`: `<u>$0</u>`,
371 		`P`: `
372 
373 $0
374 
375 `,
376 		`BR`: "\n\n",
377 		`DL`: `$0`,
378 		`DT`: `**$0**`,
379 		`DD`: `
380 
381 * $0`,
382 		`TABLE`: `$0`,
383 		`TR`: `$0|`,
384 		`TH`: `| **$0** `,
385 		`TD`: `| $0 `,
386 		`OL`: `$0`,
387 		`UL`: `$0`,
388 		`LI`: `* $0`,
389 		`LINK`: `[$0]$(LPAREN)$0$(RPAREN)`,
390 		`LINK2`: `[$+]$(LPAREN)$1$(RPAREN)`,
391 		`LPAREN`: `(`,
392 		`RPAREN`: `)`,
393 		`DOLLAR`: `$`,
394 		`BACKTICK`: "`",
395 		`COLON`: ":",
396 		`DEPRECATED`: `$0`,
397 		`LREF`: `[$(BACKTICK)$0$(BACKTICK)](command$(COLON)code-d.navigateLocal?$0)`,
398 		`REF`: `[$(BACKTICK)` ~ inlineRefPrefix ~ `$+.$1$(BACKTICK)](command$(COLON)code-d.navigateGlobal?`
399 		~ inlineRefPrefix ~ `$+.$1)`,
400 		`RED`: `<font color=red>**$0**</font>`,
401 		`BLUE`: `<font color=blue>$0</font>`,
402 		`GREEN`: `<font color=green>$0</font>`,
403 		`YELLOW`: `<font color=yellow>$0</font>`,
404 		`BLACK`: `<font color=black>$0</font>`,
405 		`WHITE`: `<font color=white>$0</font>`,
406 		`D_CODE`: "$(BACKTICK)$(BACKTICK)$(BACKTICK)d
407 $0
408 $(BACKTICK)$(BACKTICK)$(BACKTICK)",
409 		`D_INLINECODE`: "$(BACKTICK)$0$(BACKTICK)",
410 		`D`: "$(BACKTICK)$0$(BACKTICK)",
411 		`D_COMMENT`: "$(BACKTICK)$0$(BACKTICK)",
412 		`D_STRING`: "$(BACKTICK)$0$(BACKTICK)",
413 		`D_KEYWORD`: "$(BACKTICK)$0$(BACKTICK)",
414 		`D_PSYMBOL`: "$(BACKTICK)$0$(BACKTICK)",
415 		`D_PARAM`: "$(BACKTICK)$0$(BACKTICK)",
416 		`DDOC`: `# $(TITLE)
417 
418 $(BODY)`,
419 		`DDOC_BACKQUOTED`: `$(D_INLINECODE $0)`,
420 		`DDOC_COMMENT`: ``,
421 		`DDOC_DECL`: `$(DT $(BIG $0))`,
422 		`DDOC_DECL_DD`: `$(DD $0)`,
423 		`DDOC_DITTO`: `$(BR)$0`,
424 		`DDOC_SECTIONS`: `$0`,
425 		`DDOC_SUMMARY`: `$0$(BR)$(BR)`,
426 		`DDOC_DESCRIPTION`: `$0$(BR)$(BR)`,
427 		`DDOC_AUTHORS`: "$(B Authors:)$(BR)\n$0$(BR)$(BR)",
428 		`DDOC_BUGS`: "$(RED BUGS:)$(BR)\n$0$(BR)$(BR)",
429 		`DDOC_COPYRIGHT`: "$(B Copyright:)$(BR)\n$0$(BR)$(BR)",
430 		`DDOC_DATE`: "$(B Date:)$(BR)\n$0$(BR)$(BR)",
431 		`DDOC_DEPRECATED`: "$(RED Deprecated:)$(BR)\n$0$(BR)$(BR)",
432 		`DDOC_EXAMPLES`: "$(B Examples:)$(BR)\n$0$(BR)$(BR)",
433 		`DDOC_HISTORY`: "$(B History:)$(BR)\n$0$(BR)$(BR)",
434 		`DDOC_LICENSE`: "$(B License:)$(BR)\n$0$(BR)$(BR)",
435 		`DDOC_RETURNS`: "$(B Returns:)$(BR)\n$0$(BR)$(BR)",
436 		`DDOC_SEE_ALSO`: "$(B See Also:)$(BR)\n$0$(BR)$(BR)",
437 		`DDOC_STANDARDS`: "$(B Standards:)$(BR)\n$0$(BR)$(BR)",
438 		`DDOC_THROWS`: "$(B Throws:)$(BR)\n$0$(BR)$(BR)",
439 		`DDOC_VERSION`: "$(B Version:)$(BR)\n$0$(BR)$(BR)",
440 		`DDOC_SECTION_H`: `$(B $0)$(BR)$(BR)`,
441 		`DDOC_SECTION`: `$0$(BR)$(BR)`,
442 		`DDOC_MEMBERS`: `$(DL $0)`,
443 		`DDOC_MODULE_MEMBERS`: `$(DDOC_MEMBERS $0)`,
444 		`DDOC_CLASS_MEMBERS`: `$(DDOC_MEMBERS $0)`,
445 		`DDOC_STRUCT_MEMBERS`: `$(DDOC_MEMBERS $0)`,
446 		`DDOC_ENUM_MEMBERS`: `$(DDOC_MEMBERS $0)`,
447 		`DDOC_TEMPLATE_MEMBERS`: `$(DDOC_MEMBERS $0)`,
448 		`DDOC_ENUM_BASETYPE`: `$0`,
449 		`DDOC_PARAMS`: "$(B Params:)$(BR)\n$(TABLE $0)$(BR)",
450 		`DDOC_PARAM_ROW`: `$(TR $0)`,
451 		`DDOC_PARAM_ID`: `$(TD $0)`,
452 		`DDOC_PARAM_DESC`: `$(TD $0)`,
453 		`DDOC_BLANKLINE`: `$(BR)$(BR)`,
454 
455 		`DDOC_ANCHOR`: `<a name="$1"></a>`,
456 		`DDOC_PSYMBOL`: `$(U $0)`,
457 		`DDOC_PSUPER_SYMBOL`: `$(U $0)`,
458 		`DDOC_KEYWORD`: `$(B $0)`,
459 		`DDOC_PARAM`: `$(I $0)`
460 	];
461 }
462 
463 unittest
464 {
465 	//dfmt off
466 	auto comment = "Quick example of a comment\n"
467 		~ "&#36;(D something, else) is *a\n"
468 		~ "------------\n"
469 		~ "test\n"
470 		~ "/** this is some test code */\n"
471 		~ "assert (whatever);\n"
472 		~ "---------\n"
473 		~ "Params:\n"
474 		~ "	a = $(B param)\n"
475 		~ "Returns:\n"
476 		~ "	nothing of consequence";
477 
478 	auto commentMarkdown = "Quick example of a comment\n"
479 		~ "$(D something, else) is *a\n"
480 		~ "\n"
481 		~ "```d\n"
482 		~ "test\n"
483 		~ "/** this is some test code */\n"
484 		~ "assert (whatever);\n"
485 		~ "```\n\n"
486 		~ "**Params**\n\n"
487 		~ "`a` **param**\n\n"
488 		~ "**Returns** — nothing of consequence\n\n";
489 	//dfmt on
490 
491 	assert(ddocToMarkdown(comment) == commentMarkdown);
492 }
493 
494 @("ddoc with inline references")
495 unittest
496 {
497 	//dfmt off
498 	auto comment = "creates a $(REF Exception,std, object) for this $(LREF error).";
499 
500 	auto commentMarkdown = "creates a [`std.object.Exception`](command:code-d.navigateGlobal?std.object.Exception) "
501 			~ "for this [`error`](command:code-d.navigateLocal?error).\n\n\n\n";
502 	//dfmt on
503 
504 	assert(ddocToMarkdown(comment) == commentMarkdown);
505 }
506 
507 @("messed up formatting")
508 unittest
509 {
510 	//dfmt off
511 	auto comment = ` * this documentation didn't have the stars stripped
512  * so we need to remove them.
513  * There is more content.
514 ---
515 // example code
516 ---`;
517 
518 	auto commentMarkdown = `this documentation didn't have the stars stripped
519 so we need to remove them.
520 There is more content.
521 
522 ` ~ "```" ~ `d
523 // example code
524 ` ~ "```\n\n";
525 	//dfmt on
526 
527 	assert(ddocToMarkdown(comment) == commentMarkdown);
528 }