1 module served.utils.ddoc;
2 
3 import served.lsp.protocol;
4 
5 import ddoc;
6 
7 import std.algorithm;
8 import std.array;
9 import std.format;
10 import std.range.primitives;
11 import std..string;
12 import std.uni : sicmp;
13 
14 public import ddoc : Comment;
15 
16 /**
17  * A test function for checking `DDoc` parsing
18  *
19  * Params:
20  *     hello = a string
21  *     world = an integer
22  *
23  * Author: Jonny
24  * Bugs: None
25  * ---
26  * import std.stdio;
27  *
28  * int main(string[] args) {
29  * 	   writeln("Testing inline code")
30  * }
31  * ---
32  */
33 private int testFunction(string foo, int bar)
34 {
35 	import std.stdio : writeln;
36 
37 	writeln(foo, bar);
38 	return 0;
39 }
40 
41 /**
42  * Parses a ddoc string into a divided comment.
43  * Returns: A comment if the ddoc could be parsed or Comment.init if it couldn't be parsed and throwError is false.
44  * Throws: Exception if comment has ddoc syntax errors.
45  * Params:
46  * 	ddoc = the documentation string as given by the user without any comment markers
47  * 	throwError = set to true to make parsing errors throw
48  */
49 Comment parseDdoc(string ddoc, bool throwError = false)
50 {
51 	if (ddoc.length == 0)
52 		return Comment.init;
53 
54 	if (throwError)
55 		return parseComment(prepareDDoc(ddoc), markdownMacros, false);
56 	else
57 	{
58 		try
59 		{
60 			return parseComment(prepareDDoc(ddoc), markdownMacros, false);
61 		}
62 		catch (Exception e)
63 		{
64 			return Comment.init;
65 		}
66 	}
67 }
68 
69 /**
70  * Convert a Ddoc comment string to markdown. Returns ddoc string back if it is
71  * not valid.
72  * Params:
73  *		ddoc = string of a valid Ddoc comment.
74  */
75 string ddocToMarkdown(string ddoc)
76 {
77 	// Parse ddoc. Return if exception.
78 	Comment comment;
79 	try
80 	{
81 		comment = parseComment(prepareDDoc(ddoc), markdownMacros, false);
82 	}
83 	catch (Exception e)
84 	{
85 		return ddoc;
86 	}
87 	return ddocToMarkdown(comment);
88 }
89 
90 /// ditto
91 string ddocToMarkdown(const Comment comment)
92 {
93 	auto output = "";
94 	foreach (section; comment.sections)
95 	{
96 		import std.uni : toLower;
97 
98 		string content = section.content.postProcessContent;
99 		switch (section.name.toLower)
100 		{
101 		case "":
102 		case "summary":
103 			output ~= content ~ "\n\n";
104 			break;
105 		case "params":
106 			output ~= "**Params**\n\n";
107 			foreach (parameter; section.mapping)
108 			{
109 				output ~= format!"`%s` %s\n\n"(parameter[0].postProcessContent,
110 						parameter[1].postProcessContent);
111 			}
112 			break;
113 		case "author":
114 		case "authors":
115 		case "bugs":
116 		case "date":
117 		case "deprecated":
118 		case "history":
119 		default:
120 			// Single line sections go on the same line as section titles. Multi
121 			// line sections go on the line below.
122 			import std.algorithm : canFind;
123 
124 			content = content.chomp();
125 			if (!content.canFind("\n"))
126 			{
127 				output ~= format!"**%s** — %s\n\n"(section.name, content);
128 			}
129 			else
130 			{
131 				output ~= format!"**%s**\n\n%s\n\n"(section.name, content);
132 			}
133 			break;
134 		}
135 	}
136 	return output.replace("$", "$");
137 }
138 
139 /// Removes leading */+ characters from each line per section if the entire section only consists of them. Sections are separated with lines starting with ---
140 private string preProcessContent(string content)
141 {
142 	auto newContent = appender!string();
143 	// TODO: optimize to not allocate when not changing content
144 	newContent.reserve(content.length);
145 	foreach (chunk; content.lineSplitter!(KeepTerminator.yes)
146 			.chunkBy!(a => a.startsWith("---")))
147 	{
148 		auto c = chunk[1].save;
149 
150 		bool isStrippable = true;
151 		foreach (line; c)
152 		{
153 			auto l = line.stripLeft;
154 			if (!l.length)
155 				continue;
156 			if (!l.startsWith("*", "+"))
157 			{
158 				isStrippable = false;
159 				break;
160 			}
161 		}
162 
163 		if (isStrippable)
164 		{
165 			foreach (line; chunk[1])
166 			{
167 				auto stripped = line.stripLeft;
168 				if (!stripped.length)
169 					stripped = line;
170 
171 				if (stripped.startsWith("* ", "+ ", "*\t", "+\t"))
172 					newContent.put(stripped[2 .. $]);
173 				else if (stripped.startsWith("*", "+"))
174 					newContent.put(stripped[1 .. $]);
175 				else
176 					newContent.put(line);
177 			}
178 		}
179 		else
180 			foreach (line; chunk[1])
181 				newContent.put(line);
182 	}
183 	return newContent.data;
184 }
185 
186 /// Fixes code-d specific placeholders inserted during ddoc translation for better IDE integration.
187 private string postProcessContent(string content)
188 {
189 	while (true)
190 	{
191 		auto index = content.indexOf(inlineRefPrefix);
192 		if (index != -1)
193 		{
194 			auto end = content.indexOf('.', index + inlineRefPrefix.length);
195 			if (end == -1)
196 				break; // malformed
197 			content = content[0 .. index]
198 				~ content[index + inlineRefPrefix.length .. end].postProcessInlineRefPrefix
199 				~ content[end .. $];
200 		}
201 
202 		if (index == -1)
203 			break;
204 	}
205 	return content;
206 }
207 
208 private string postProcessInlineRefPrefix(string content)
209 {
210 	return content.splitter(',').map!strip.join('.');
211 }
212 
213 /**
214  * Convert a DDoc comment string to MarkedString (as defined in the language
215  * server spec)
216  * Params:
217  *		ddoc = A DDoc string to be converted to Markdown
218  */
219 MarkedString[] ddocToMarked(string ddoc)
220 {
221 	MarkedString[] ret;
222 	if (!ddoc.length)
223 		return ret;
224 	return markdownToMarked(ddoc.ddocToMarkdown);
225 }
226 
227 /// ditto
228 MarkedString[] ddocToMarked(const Comment comment)
229 {
230 	MarkedString[] ret;
231 	if (comment == Comment.init)
232 		return ret;
233 	return markdownToMarked(comment.ddocToMarkdown);
234 }
235 
236 /**
237  * Converts markdown code to MarkedString blocks as determined by D code blocks.
238  */
239 MarkedString[] markdownToMarked(string md)
240 {
241 	MarkedString[] ret;
242 	if (!md.length)
243 		return ret;
244 
245 	ret ~= MarkedString("");
246 
247 	foreach (line; md.lineSplitter!(KeepTerminator.yes))
248 	{
249 		if (line.strip == "```d")
250 			ret ~= MarkedString("", "d");
251 		else if (line.strip == "```")
252 			ret ~= MarkedString("");
253 		else
254 			ret[$ - 1].value ~= line;
255 	}
256 
257 	if (ret.length >= 2 && !ret[$ - 1].value.strip.length)
258 		ret = ret[0 .. $ - 1];
259 
260 	return ret;
261 }
262 
263 /**
264  * Returns: the params section in a ddoc comment as key value pair. Or null if not found.
265  */
266 inout(KeyValuePair[]) getParams(inout Comment comment)
267 {
268 	foreach (section; comment.sections)
269 		if (section.name.sicmp("params") == 0)
270 			return section.mapping;
271 	return null;
272 }
273 
274 /**
275  * Returns: documentation for a given parameter in the params section of a documentation comment. Or null if not found.
276  */
277 string getParamDocumentation(const Comment comment, string searchParam)
278 {
279 	foreach (param; getParams(comment))
280 		if (param[0] == searchParam)
281 			return param[1];
282 	return null;
283 }
284 
285 /**
286  * Performs preprocessing of the document. Wraps code blocks in macros.
287  * Params:
288  * 		str = This is one of the params
289  */
290 private string prepareDDoc(string str)
291 {
292 	import ddoc.lexer : Lexer;
293 
294 	str = str.preProcessContent;
295 
296 	auto lex = Lexer(str, true);
297 	string output;
298 	foreach (tok; lex)
299 	{
300 		if (tok.type == Type.embedded || tok.type == Type.inlined)
301 		{
302 			// Add newlines before documentation
303 			if (tok.type == Type.embedded)
304 			{
305 				output ~= "\n\n";
306 			}
307 			output ~= tok.type == Type.embedded ? "$(D_CODE " : "$(DDOC_BACKQUOTED ";
308 			output ~= tok.text;
309 			output ~= ")";
310 		}
311 		else
312 		{
313 			output ~= tok.text;
314 		}
315 	}
316 	return output;
317 }
318 
319 static immutable inlineRefPrefix = "__CODED_INLINE_REF__:";
320 
321 string[string] markdownMacros;
322 static this()
323 {
324 	markdownMacros = [
325 		`B`: `**$0**`,
326 		`I`: `*$0*`,
327 		`U`: `<u>$0</u>`,
328 		`P`: `
329 
330 $0
331 
332 `,
333 		`BR`: "\n\n",
334 		`DL`: `$0`,
335 		`DT`: `**$0**`,
336 		`DD`: `
337 
338 * $0`,
339 		`TABLE`: `$0`,
340 		`TR`: `$0|`,
341 		`TH`: `| **$0** `,
342 		`TD`: `| $0 `,
343 		`OL`: `$0`,
344 		`UL`: `$0`,
345 		`LI`: `* $0`,
346 		`LINK`: `[$0]$(LPAREN)$0$(RPAREN)`,
347 		`LINK2`: `[$+]$(LPAREN)$1$(RPAREN)`,
348 		`LPAREN`: `(`,
349 		`RPAREN`: `)`,
350 		`DOLLAR`: `$`,
351 		`BACKTICK`: "`",
352 		`COLON`: ":",
353 		`DEPRECATED`: `$0`,
354 		`LREF`: `[$(BACKTICK)$0$(BACKTICK)](command$(COLON)code-d.navigateLocal?$0)`,
355 		`REF`: `[$(BACKTICK)` ~ inlineRefPrefix ~ `$+.$1$(BACKTICK)](command$(COLON)code-d.navigateGlobal?`
356 		~ inlineRefPrefix ~ `$+.$1)`,
357 		`RED`: `<font color=red>**$0**</font>`,
358 		`BLUE`: `<font color=blue>$0</font>`,
359 		`GREEN`: `<font color=green>$0</font>`,
360 		`YELLOW`: `<font color=yellow>$0</font>`,
361 		`BLACK`: `<font color=black>$0</font>`,
362 		`WHITE`: `<font color=white>$0</font>`,
363 		`D_CODE`: "$(BACKTICK)$(BACKTICK)$(BACKTICK)d
364 $0
365 $(BACKTICK)$(BACKTICK)$(BACKTICK)",
366 		`D_INLINECODE`: "$(BACKTICK)$0$(BACKTICK)",
367 		`D`: "$(BACKTICK)$0$(BACKTICK)",
368 		`D_COMMENT`: "$(BACKTICK)$0$(BACKTICK)",
369 		`D_STRING`: "$(BACKTICK)$0$(BACKTICK)",
370 		`D_KEYWORD`: "$(BACKTICK)$0$(BACKTICK)",
371 		`D_PSYMBOL`: "$(BACKTICK)$0$(BACKTICK)",
372 		`D_PARAM`: "$(BACKTICK)$0$(BACKTICK)",
373 		`DDOC`: `# $(TITLE)
374 
375 $(BODY)`,
376 		`DDOC_BACKQUOTED`: `$(D_INLINECODE $0)`,
377 		`DDOC_COMMENT`: ``,
378 		`DDOC_DECL`: `$(DT $(BIG $0))`,
379 		`DDOC_DECL_DD`: `$(DD $0)`,
380 		`DDOC_DITTO`: `$(BR)$0`,
381 		`DDOC_SECTIONS`: `$0`,
382 		`DDOC_SUMMARY`: `$0$(BR)$(BR)`,
383 		`DDOC_DESCRIPTION`: `$0$(BR)$(BR)`,
384 		`DDOC_AUTHORS`: "$(B Authors:)$(BR)\n$0$(BR)$(BR)",
385 		`DDOC_BUGS`: "$(RED BUGS:)$(BR)\n$0$(BR)$(BR)",
386 		`DDOC_COPYRIGHT`: "$(B Copyright:)$(BR)\n$0$(BR)$(BR)",
387 		`DDOC_DATE`: "$(B Date:)$(BR)\n$0$(BR)$(BR)",
388 		`DDOC_DEPRECATED`: "$(RED Deprecated:)$(BR)\n$0$(BR)$(BR)",
389 		`DDOC_EXAMPLES`: "$(B Examples:)$(BR)\n$0$(BR)$(BR)",
390 		`DDOC_HISTORY`: "$(B History:)$(BR)\n$0$(BR)$(BR)",
391 		`DDOC_LICENSE`: "$(B License:)$(BR)\n$0$(BR)$(BR)",
392 		`DDOC_RETURNS`: "$(B Returns:)$(BR)\n$0$(BR)$(BR)",
393 		`DDOC_SEE_ALSO`: "$(B See Also:)$(BR)\n$0$(BR)$(BR)",
394 		`DDOC_STANDARDS`: "$(B Standards:)$(BR)\n$0$(BR)$(BR)",
395 		`DDOC_THROWS`: "$(B Throws:)$(BR)\n$0$(BR)$(BR)",
396 		`DDOC_VERSION`: "$(B Version:)$(BR)\n$0$(BR)$(BR)",
397 		`DDOC_SECTION_H`: `$(B $0)$(BR)$(BR)`,
398 		`DDOC_SECTION`: `$0$(BR)$(BR)`,
399 		`DDOC_MEMBERS`: `$(DL $0)`,
400 		`DDOC_MODULE_MEMBERS`: `$(DDOC_MEMBERS $0)`,
401 		`DDOC_CLASS_MEMBERS`: `$(DDOC_MEMBERS $0)`,
402 		`DDOC_STRUCT_MEMBERS`: `$(DDOC_MEMBERS $0)`,
403 		`DDOC_ENUM_MEMBERS`: `$(DDOC_MEMBERS $0)`,
404 		`DDOC_TEMPLATE_MEMBERS`: `$(DDOC_MEMBERS $0)`,
405 		`DDOC_ENUM_BASETYPE`: `$0`,
406 		`DDOC_PARAMS`: "$(B Params:)$(BR)\n$(TABLE $0)$(BR)",
407 		`DDOC_PARAM_ROW`: `$(TR $0)`,
408 		`DDOC_PARAM_ID`: `$(TD $0)`,
409 		`DDOC_PARAM_DESC`: `$(TD $0)`,
410 		`DDOC_BLANKLINE`: `$(BR)$(BR)`,
411 
412 		`DDOC_ANCHOR`: `<a name="$1"></a>`,
413 		`DDOC_PSYMBOL`: `$(U $0)`,
414 		`DDOC_PSUPER_SYMBOL`: `$(U $0)`,
415 		`DDOC_KEYWORD`: `$(B $0)`,
416 		`DDOC_PARAM`: `$(I $0)`
417 	];
418 }
419 
420 unittest
421 {
422 	//dfmt off
423 	auto comment = "Quick example of a comment\n"
424 		~ "&#36;(D something, else) is *a\n"
425 		~ "------------\n"
426 		~ "test\n"
427 		~ "/** this is some test code */\n"
428 		~ "assert (whatever);\n"
429 		~ "---------\n"
430 		~ "Params:\n"
431 		~ "	a = $(B param)\n"
432 		~ "Returns:\n"
433 		~ "	nothing of consequence";
434 
435 	auto commentMarkdown = "Quick example of a comment\n"
436 		~ "$(D something, else) is *a\n"
437 		~ "\n"
438 		~ "```d\n"
439 		~ "test\n"
440 		~ "/** this is some test code */\n"
441 		~ "assert (whatever);\n"
442 		~ "```\n\n"
443 		~ "**Params**\n\n"
444 		~ "`a` **param**\n\n"
445 		~ "**Returns** — nothing of consequence\n\n";
446 	//dfmt on
447 
448 	assert(ddocToMarkdown(comment) == commentMarkdown);
449 }
450 
451 @("ddoc with inline references")
452 unittest
453 {
454 	//dfmt off
455 	auto comment = "creates a $(REF Exception,std, object) for this $(LREF error).";
456 
457 	auto commentMarkdown = "creates a [`std.object.Exception`](command:code-d.navigateGlobal?std.object.Exception) "
458 			~ "for this [`error`](command:code-d.navigateLocal?error).\n\n\n\n";
459 	//dfmt on
460 
461 	assert(ddocToMarkdown(comment) == commentMarkdown);
462 }
463 
464 @("messed up formatting")
465 unittest
466 {
467 	//dfmt off
468 	auto comment = ` * this documentation didn't have the stars stripped
469  * so we need to remove them.
470  * There is more content.
471 ---
472 // example code
473 ---`;
474 
475 	auto commentMarkdown = `this documentation didn't have the stars stripped
476 so we need to remove them.
477 There is more content.
478 
479 ` ~ "```" ~ `d
480 // example code
481 ` ~ "```\n\n";
482 	//dfmt on
483 
484 	assert(ddocToMarkdown(comment) == commentMarkdown);
485 }