Code blocks support for all formats. Single line break support (#123)

*  HTML-ize code blocks inside a markdown table. Single line break.

 Add support for embedding markdown code blocks inside markdown tables

 Add support for single-line breaks: when the line ends with 2 spaces, place
 a single <br>.

* Fix the testsuite, todo improve the escape code for document

* Add the WithIndent testdata

examples/variables.tf

@@ -57,3 +57,18 @@ variable "input-with-pipe" {
   description = "It includes v1 | v2 | v3"
   default     = "v1"
 }
+
+variable "input-with-code-block" {
+  description = <<EOD
+This is a complicated one. We need a newline.  
+And an example in a code block
+```
+default     = [
+  "machine rack01:neptune"
+]
+```
+EOD
+  default     = [
+    "name rack:location"
+  ]
+}

internal/pkg/doc/doc_test.go

@@ -120,6 +120,18 @@ var inputWithPipe = doc.Input{
 	Type: "string",
 }
 
+var inputWithCodeBlock = doc.Input{
+	Name:        "input-with-code-block",
+	Description: "This is a complicated one. We need a newline.  \nAnd an example in a code block\n```\ndefault     = [\n  \"machine rack01:neptune\"\n]\n```\n",
+	Default: &doc.Value{
+		Type: "list",
+		Value: []interface{}{
+			"name rack:location",
+		},
+	},
+	Type: "list",
+}
+
 var output1 = doc.Output{
 	Name:        "output-1",
 	Description: "It's output number one.",
@@ -215,6 +227,7 @@ func TestInputs(t *testing.T) {
 		inputList1,
 		inputWithUnderscores,
 		inputWithPipe,
+		inputWithCodeBlock,
 	}
 
 	assert.Equal(t, expected, actual)
@@ -246,6 +259,7 @@ func TestInputsFromVariablesTf(t *testing.T) {
 		inputList1,
 		inputWithUnderscores,
 		inputWithPipe,
+		inputWithCodeBlock,
 	}
 
 	assert.Equal(t, expected, actual)
@@ -257,6 +271,7 @@ func TestInputsSortedByName(t *testing.T) {
 	doc.SortInputsByName(actual)
 
 	expected := []doc.Input{
+		inputWithCodeBlock,
 		inputWithPipe,
 		inputWithUnderscores,
 		inputList1,
@@ -285,6 +300,7 @@ func TestInputsSortedByRequired(t *testing.T) {
 		inputMap2,
 		inputString2,
 		inputUnquoted,
+		inputWithCodeBlock,
 		inputWithPipe,
 		inputList1,
 		inputList3,

internal/pkg/print/json/testdata/json-WithSortByName.golden

@@ -1,6 +1,17 @@
 {
   "Comment": "Usage:\n\nmodule \"foo\" {\n  source = \"github.com/foo/bar\"\n\n  id   = \"1234567890\"\n  name = \"baz\"\n\n  zones = [\"us-east-1\", \"us-west-1\"]\n\n  tags = {\n    Name         = \"baz\"\n    Created-By   = \"first.last@email.com\"\n    Date-Created = \"20180101\"\n  }\n}\n",
   "Inputs": [
+    {
+      "Name": "input-with-code-block",
+      "Description": "This is a complicated one. We need a newline.  \nAnd an example in a code block\n```\ndefault     = [\n  \"machine rack01:neptune\"\n]\n```\n",
+      "Default": {
+        "Type": "list",
+        "Value": [
+          "name rack:location"
+        ]
+      },
+      "Type": "list"
+    },
     {
       "Name": "input-with-pipe",
       "Description": "It includes v1 | v2 | v3",

internal/pkg/print/json/testdata/json-WithSortInputsByRequired.golden

@@ -31,6 +31,17 @@
       "Default": null,
       "Type": "string"
     },
+    {
+      "Name": "input-with-code-block",
+      "Description": "This is a complicated one. We need a newline.  \nAnd an example in a code block\n```\ndefault     = [\n  \"machine rack01:neptune\"\n]\n```\n",
+      "Default": {
+        "Type": "list",
+        "Value": [
+          "name rack:location"
+        ]
+      },
+      "Type": "list"
+    },
     {
       "Name": "input-with-pipe",
       "Description": "It includes v1 | v2 | v3",

internal/pkg/print/json/testdata/json.golden

@@ -101,6 +101,17 @@
         "Value": "v1"
       },
       "Type": "string"
+    },
+    {
+      "Name": "input-with-code-block",
+      "Description": "This is a complicated one. We need a newline.  \nAnd an example in a code block\n```\ndefault     = [\n  \"machine rack01:neptune\"\n]\n```\n",
+      "Default": {
+        "Type": "list",
+        "Value": [
+          "name rack:location"
+        ]
+      },
+      "Type": "list"
     }
   ],
   "Outputs": [

internal/pkg/print/markdown/document/document.go

@@ -3,6 +3,7 @@ package document
 import (
 	"bytes"
 	"fmt"
+	"strings"
 
 	"github.com/segmentio/terraform-docs/internal/pkg/doc"
 	"github.com/segmentio/terraform-docs/internal/pkg/print"
@@ -42,7 +43,31 @@ func Print(document *doc.Doc, settings *settings.Settings) (string, error) {
 		printOutputs(&buffer, document.Outputs, settings)
 	}
 
-	return markdown.Sanitize(buffer.String()), nil
+	// out := markdown.Sanitize(buffer.String())
+
+	out := strings.Replace(buffer.String(), "<br>```<br>", "\n```\n", -1)
+
+	// the left over <br> or either inside or outside a code block:
+	segments := strings.Split(out, "\n```\n")
+	buf := bytes.NewBufferString("")
+	nextIsInCodeBlock := strings.HasPrefix(out, "```\n")
+	for i, segment := range segments {
+		if !nextIsInCodeBlock {
+			if i > 0 && len(segment) > 0 {
+				buf.WriteString("\n```\n")
+			}
+			segment = markdown.Sanitize(segment)
+			segment = strings.Replace(segment, "<br><br>", "\n\n", -1)
+			segment = strings.Replace(segment, "<br>", "  \n", -1)
+			buf.WriteString(segment)
+			nextIsInCodeBlock = true
+		} else {
+			buf.WriteString("\n```\n")
+			buf.WriteString(strings.Replace(segment, "<br>", "\n", -1))
+			nextIsInCodeBlock = false
+		}
+	}
+	return strings.Replace(buf.String(), " \n\n", "\n\n", -1), nil
 }
 
 func getInputDefaultValue(input *doc.Input, settings *settings.Settings) string {

internal/pkg/print/markdown/document/testdata/document-WithAggregateTypeDefaults.golden

@@ -139,6 +139,26 @@ Type: `string`
 
 Default: `"v1"`
 
+### input-with-code-block
+
+Description: This is a complicated one. We need a newline.  
+And an example in a code block
+```
+default     = [
+  "machine rack01:neptune"
+]
+```
+
+Type: `list`
+
+Default:
+
+```json
+[
+  "name rack:location"
+]
+```
+
 ## Outputs
 
 The following outputs are exported:

internal/pkg/print/markdown/document/testdata/document-WithEscapeName.golden

@@ -115,6 +115,20 @@ Type: `string`
 
 Default: `"v1"`
 
+### input-with-code-block
+
+Description: This is a complicated one. We need a newline.  
+And an example in a code block
+```
+default     = [
+  "machine rack01:neptune"
+]
+```
+
+Type: `list`
+
+Default: `<list>`
+
 ## Outputs
 
 The following outputs are exported:

internal/pkg/print/markdown/document/testdata/document-WithIndentationAboveAllowed.golden

@@ -115,6 +115,20 @@ Type: `string`
 
 Default: `"v1"`
 
+### input-with-code-block
+
+Description: This is a complicated one. We need a newline.  
+And an example in a code block
+```
+default     = [
+  "machine rack01:neptune"
+]
+```
+
+Type: `list`
+
+Default: `<list>`
+
 ## Outputs
 
 The following outputs are exported:

internal/pkg/print/markdown/document/testdata/document-WithIndentationBellowAllowed.golden

@@ -115,6 +115,20 @@ Type: `string`
 
 Default: `"v1"`
 
+### input-with-code-block
+
+Description: This is a complicated one. We need a newline.  
+And an example in a code block
+```
+default     = [
+  "machine rack01:neptune"
+]
+```
+
+Type: `list`
+
+Default: `<list>`
+
 ## Outputs
 
 The following outputs are exported:

internal/pkg/print/markdown/document/testdata/document-WithIndentationOfFour.golden

@@ -115,6 +115,20 @@ Type: `string`
 
 Default: `"v1"`
 
+##### input-with-code-block
+
+Description: This is a complicated one. We need a newline.  
+And an example in a code block
+```
+default     = [
+  "machine rack01:neptune"
+]
+```
+
+Type: `list`
+
+Default: `<list>`
+
 #### Outputs
 
 The following outputs are exported:

internal/pkg/print/markdown/document/testdata/document-WithRequired.golden

@@ -109,6 +109,20 @@ Type: `string`
 
 Default: `"v1"`
 
+### input-with-code-block
+
+Description: This is a complicated one. We need a newline.  
+And an example in a code block
+```
+default     = [
+  "machine rack01:neptune"
+]
+```
+
+Type: `list`
+
+Default: `<list>`
+
 ## Outputs
 
 The following outputs are exported:

internal/pkg/print/markdown/document/testdata/document-WithSortByName.golden

@@ -19,6 +19,20 @@ module "foo" {
 
 The following input variables are supported:
 
+### input-with-code-block
+
+Description: This is a complicated one. We need a newline.  
+And an example in a code block
+```
+default     = [
+  "machine rack01:neptune"
+]
+```
+
+Type: `list`
+
+Default: `<list>`
+
 ### input-with-pipe
 
 Description: It includes v1 \| v2 \| v3

internal/pkg/print/markdown/document/testdata/document-WithSortInputsByRequired.golden

@@ -59,6 +59,20 @@ Type: `string`
 
 Default: n/a
 
+### input-with-code-block
+
+Description: This is a complicated one. We need a newline.  
+And an example in a code block
+```
+default     = [
+  "machine rack01:neptune"
+]
+```
+
+Type: `list`
+
+Default: `<list>`
+
 ### input-with-pipe
 
 Description: It includes v1 \| v2 \| v3

internal/pkg/print/markdown/document/testdata/document.golden

@@ -115,6 +115,20 @@ Type: `string`
 
 Default: `"v1"`
 
+### input-with-code-block
+
+Description: This is a complicated one. We need a newline.  
+And an example in a code block
+```
+default     = [
+  "machine rack01:neptune"
+]
+```
+
+Type: `list`
+
+Default: `<list>`
+
 ## Outputs
 
 The following outputs are exported:

internal/pkg/print/markdown/markdown.go

@@ -1,6 +1,7 @@
 package markdown
 
 import (
+	"bytes"
 	"regexp"
 	"strings"
 
@@ -17,16 +18,59 @@ func SanitizeName(s string, settings *settings.Settings) string {
 	return s
 }
 
-// SanitizeDescription converts description to suitable Markdown representation. (including line-break, illegal characters, etc)
 func SanitizeDescription(s string, settings *settings.Settings) string {
-	s = ConvertMultiLineText(s)
-	s = EscapeIllegalCharacters(s, settings)
+	// s = ConvertMultiLineText(s)
+	// s = EscapeIllegalCharacters(s, settings)
+	// return s
+	// Isolate blocks of code. Dont escape anything inside them
+	nextIsInCodeBlock := strings.HasPrefix(s, "```\n")
+	segments := strings.Split(s, "\n```\n")
+	buf := bytes.NewBufferString("")
+	for i, segment := range segments {
+		if !nextIsInCodeBlock {
+			segment = ConvertMultiLineText(segment)
+			segment = EscapeIllegalCharacters(segment, settings)
+			if i > 0 && len(segment) > 0 {
+				buf.WriteString("<br>```<br>")
+			}
+			buf.WriteString(segment)
+			nextIsInCodeBlock = true
+		} else {
+			buf.WriteString("<br>```<br>")
+			buf.WriteString(segment)
+			buf.WriteString("<br>```")
+			nextIsInCodeBlock = false
+		}
+	}
+	return buf.String()
+}
 
-	return s
+// SanitizeDescription converts description to suitable Markdown representation for a table. (including line-break, illegal characters, code blocks etc)
+func SanitizeDescriptionForTable(s string, settings *settings.Settings) string {
+	// Isolate blocks of code. Dont escape anything inside them
+	nextIsInCodeBlock := strings.HasPrefix(s, "```\n")
+	segments := strings.Split(s, "```\n")
+	buf := bytes.NewBufferString("")
+	for _, segment := range segments {
+		if !nextIsInCodeBlock {
+			segment = ConvertMultiLineText(segment)
+			segment = EscapeIllegalCharacters(segment, settings)
+			buf.WriteString(segment)
+			nextIsInCodeBlock = true
+		} else {
+			buf.WriteString("<code><pre>")
+			buf.WriteString(strings.Replace(strings.Replace(segment, "\n", "<br>", -1), "\r", "", -1))
+			buf.WriteString("</pre></code>")
+			nextIsInCodeBlock = false
+		}
+	}
+
+	return buf.String()
 }
 
 // ConvertMultiLineText converts a multi-line text into a suitable Markdown representation.
 func ConvertMultiLineText(s string) string {
+
 	// Convert double newlines to <br><br>.
 	s = strings.Replace(
 		strings.TrimSpace(s),
@@ -34,6 +78,9 @@ func ConvertMultiLineText(s string) string {
 		"<br><br>",
 		-1)
 
+	// Convert space-space-newline to <br>
+	s = strings.Replace(s, "  \n", "<br>", -1)
+
 	// Convert single newline to space.
 	return strings.Replace(s, "\n", " ", -1)
 }
@@ -50,7 +97,7 @@ func EscapeIllegalCharacters(s string, settings *settings.Settings) string {
 		// Escape asterisk
 		s = strings.Replace(s, "*", "\\*", -1)
 
-		// Escape paranthesis
+		// Escape parenthesis
 		s = strings.Replace(s, "(", "\\(", -1)
 		s = strings.Replace(s, ")", "\\)", -1)
 

internal/pkg/print/markdown/table/table.go

@@ -81,7 +81,7 @@ func printInputs(buffer *bytes.Buffer, inputs []doc.Input, settings *settings.Se
 		buffer.WriteString(
 			fmt.Sprintf("| %s | %s | %s | %s |",
 				markdown.SanitizeName(input.Name, settings),
-				markdown.SanitizeDescription(input.Description, settings),
+				markdown.SanitizeDescriptionForTable(input.Description, settings),
 				input.Type,
 				getInputDefaultValue(&input, settings)))
 
@@ -110,6 +110,6 @@ func printOutputs(buffer *bytes.Buffer, outputs []doc.Output, settings *settings
 		buffer.WriteString(
 			fmt.Sprintf("| %s | %s |\n",
 				markdown.SanitizeName(output.Name, settings),
-				markdown.SanitizeDescription(output.Description, settings)))
+				markdown.SanitizeDescriptionForTable(output.Description, settings)))
 	}
 }

internal/pkg/print/markdown/table/testdata/table-WithAggregateTypeDefaults.golden

@@ -31,6 +31,7 @@ module "foo" {
 | list-1 | It's list number one. | list | `[ "a", "b", "c" ]` |
 | input_with_underscores | A variable with underscores. | string | n/a |
 | input-with-pipe | It includes v1 \| v2 \| v3 | string | `"v1"` |
+| input-with-code-block | This is a complicated one. We need a newline.<br>And an example in a code block<code><pre>default     = [<br>  "machine rack01:neptune"<br>]<br></pre></code> | list | `[ "name rack:location" ]` |
 
 ## Outputs
 

internal/pkg/print/markdown/table/testdata/table-WithEscapeName.golden

@@ -31,6 +31,7 @@ module "foo" {
 | list-1 | It's list number one. | list | `<list>` |
 | input\_with\_underscores | A variable with underscores. | string | n/a |
 | input-with-pipe | It includes v1 \| v2 \| v3 | string | `"v1"` |
+| input-with-code-block | This is a complicated one. We need a newline.<br>And an example in a code block<code><pre>default     = [<br>  "machine rack01:neptune"<br>]<br></pre></code> | list | `<list>` |
 
 ## Outputs
 

internal/pkg/print/markdown/table/testdata/table-WithIndentationAboveAllowed.golden

@@ -31,6 +31,7 @@ module "foo" {
 | list-1 | It's list number one. | list | `<list>` |
 | input_with_underscores | A variable with underscores. | string | n/a |
 | input-with-pipe | It includes v1 \| v2 \| v3 | string | `"v1"` |
+| input-with-code-block | This is a complicated one. We need a newline.<br>And an example in a code block<code><pre>default     = [<br>  "machine rack01:neptune"<br>]<br></pre></code> | list | `<list>` |
 
 ## Outputs
 

internal/pkg/print/markdown/table/testdata/table-WithIndentationBellowAllowed.golden

@@ -31,6 +31,7 @@ module "foo" {
 | list-1 | It's list number one. | list | `<list>` |
 | input_with_underscores | A variable with underscores. | string | n/a |
 | input-with-pipe | It includes v1 \| v2 \| v3 | string | `"v1"` |
+| input-with-code-block | This is a complicated one. We need a newline.<br>And an example in a code block<code><pre>default     = [<br>  "machine rack01:neptune"<br>]<br></pre></code> | list | `<list>` |
 
 ## Outputs
 

internal/pkg/print/markdown/table/testdata/table-WithIndentationOfFour.golden

@@ -31,6 +31,7 @@ module "foo" {
 | list-1 | It's list number one. | list | `<list>` |
 | input_with_underscores | A variable with underscores. | string | n/a |
 | input-with-pipe | It includes v1 \| v2 \| v3 | string | `"v1"` |
+| input-with-code-block | This is a complicated one. We need a newline.<br>And an example in a code block<code><pre>default     = [<br>  "machine rack01:neptune"<br>]<br></pre></code> | list | `<list>` |
 
 #### Outputs
 

internal/pkg/print/markdown/table/testdata/table-WithRequired.golden

@@ -31,6 +31,7 @@ module "foo" {
 | list-1 | It's list number one. | list | `<list>` | no |
 | input_with_underscores | A variable with underscores. | string | n/a | yes |
 | input-with-pipe | It includes v1 \| v2 \| v3 | string | `"v1"` | no |
+| input-with-code-block | This is a complicated one. We need a newline.<br>And an example in a code block<code><pre>default     = [<br>  "machine rack01:neptune"<br>]<br></pre></code> | list | `<list>` | no |
 
 ## Outputs
 

internal/pkg/print/markdown/table/testdata/table-WithSortByName.golden

@@ -19,6 +19,7 @@ module "foo" {
 
 | Name | Description | Type | Default |
 |------|-------------|:----:|:-----:|
+| input-with-code-block | This is a complicated one. We need a newline.<br>And an example in a code block<code><pre>default     = [<br>  "machine rack01:neptune"<br>]<br></pre></code> | list | `<list>` |
 | input-with-pipe | It includes v1 \| v2 \| v3 | string | `"v1"` |
 | input_with_underscores | A variable with underscores. | string | n/a |
 | list-1 | It's list number one. | list | `<list>` |

internal/pkg/print/markdown/table/testdata/table-WithSortInputsByRequired.golden

@@ -24,6 +24,7 @@ module "foo" {
 | map-2 | It's map number two. | map | n/a |
 | string-2 | It's string number two. | string | n/a |
 | unquoted |  | string | n/a |
+| input-with-code-block | This is a complicated one. We need a newline.<br>And an example in a code block<code><pre>default     = [<br>  "machine rack01:neptune"<br>]<br></pre></code> | list | `<list>` |
 | input-with-pipe | It includes v1 \| v2 \| v3 | string | `"v1"` |
 | list-1 | It's list number one. | list | `<list>` |
 | list-3 |  | list | `<list>` |

internal/pkg/print/markdown/table/testdata/table.golden

@@ -31,6 +31,7 @@ module "foo" {
 | list-1 | It's list number one. | list | `<list>` |
 | input_with_underscores | A variable with underscores. | string | n/a |
 | input-with-pipe | It includes v1 \| v2 \| v3 | string | `"v1"` |
+| input-with-code-block | This is a complicated one. We need a newline.<br>And an example in a code block<code><pre>default     = [<br>  "machine rack01:neptune"<br>]<br></pre></code> | list | `<list>` |
 
 ## Outputs
 

internal/pkg/print/pretty/pretty_test.go

@@ -78,6 +78,9 @@ func TestPretty(t *testing.T) {
 			"  " + sgr_color_1 + "var.input-with-pipe" + sgr_reset + " (\"v1\")\n" +
 			"  " + sgr_color_2 + "It includes v1 | v2 | v3" + sgr_reset + "\n" +
 			"\n" +
+			"  " + sgr_color_1 + "var.input-with-code-block" + sgr_reset + " (<list>)\n" +
+			"  " + sgr_color_2 + "This is a complicated one. We need a newline.  \nAnd an example in a code block\n```\ndefault     = [\n  \"machine rack01:neptune\"\n]\n```\n" + sgr_reset + "\n" +
+			"\n" +
 			"\n" +
 			"\n" +
 			"  " + sgr_color_1 + "output.unquoted" + sgr_reset + "\n" +
@@ -165,6 +168,9 @@ func TestPrettyWithWithAggregateTypeDefaults(t *testing.T) {
 			"  " + sgr_color_1 + "var.input-with-pipe" + sgr_reset + " (\"v1\")\n" +
 			"  " + sgr_color_2 + "It includes v1 | v2 | v3" + sgr_reset + "\n" +
 			"\n" +
+			"  " + sgr_color_1 + "var.input-with-code-block" + sgr_reset + " ([ \"name rack:location\" ])\n" +
+			"  " + sgr_color_2 + "This is a complicated one. We need a newline.  \nAnd an example in a code block\n```\ndefault     = [\n  \"machine rack01:neptune\"\n]\n```\n" + sgr_reset + "\n" +
+			"\n" +
 			"\n" +
 			"\n" +
 			"  " + sgr_color_1 + "output.unquoted" + sgr_reset + "\n" +
@@ -216,6 +222,9 @@ func TestPrettyWithSortByName(t *testing.T) {
 			"}\n" +
 			"\n" +
 			"\n" +
+			"  " + sgr_color_1 + "var.input-with-code-block" + sgr_reset + " (<list>)\n" +
+			"  " + sgr_color_2 + "This is a complicated one. We need a newline.  \nAnd an example in a code block\n```\ndefault     = [\n  \"machine rack01:neptune\"\n]\n```\n" + sgr_reset + "\n" +
+			"\n" +
 			"  " + sgr_color_1 + "var.input-with-pipe" + sgr_reset + " (\"v1\")\n" +
 			"  " + sgr_color_2 + "It includes v1 | v2 | v3" + sgr_reset + "\n" +
 			"\n" +
@@ -319,6 +328,9 @@ func TestPrettyWithSortInputsByRequired(t *testing.T) {
 			"  " + sgr_color_1 + "var.unquoted" + sgr_reset + " (required)\n" +
 			"  " + sgr_color_2 + "" + sgr_reset + "\n" +
 			"\n" +
+			"  " + sgr_color_1 + "var.input-with-code-block" + sgr_reset + " (<list>)\n" +
+			"  " + sgr_color_2 + "This is a complicated one. We need a newline.  \nAnd an example in a code block\n```\ndefault     = [\n  \"machine rack01:neptune\"\n]\n```\n" + sgr_reset + "\n" +
+			"\n" +
 			"  " + sgr_color_1 + "var.input-with-pipe" + sgr_reset + " (\"v1\")\n" +
 			"  " + sgr_color_2 + "It includes v1 | v2 | v3" + sgr_reset + "\n" +
 			"\n" +