diff --git a/README.md b/README.md index 4ef65cb..e73e80a 100644 --- a/README.md +++ b/README.md @@ -750,6 +750,7 @@ GLOBAL OPTIONS: --dry-run resolve page and ancestry, show resulting HTML and exit. (default: false) [$MARK_DRY_RUN] --edit-lock, -k lock page editing to current user only to prevent accidental manual edits over Confluence Web UI. (default: false) [$MARK_EDIT_LOCK] --drop-h1, --h1_drop don't include the first H1 heading in Confluence output. (default: false) [$MARK_H1_DROP] + --strip-linebreak remove linebreaks inside of tags, to accomodate Confluence non-standard behavior (default: false) --title-from-h1, --h1_title extract page title from a leading H1 heading. If no H1 heading on a page exists, then title must be set in the page metadata. (default: false) [$MARK_H1_TITLE] --minor-edit don't send notifications while updating Confluence page. (default: false) [$MARK_MINOR_EDIT] --color value display logs in color. Possible values: auto, never. (default: "auto") [$MARK_COLOR] diff --git a/docker-compose.yaml b/docker-compose.yaml index 7928355..0689b9b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -23,7 +23,7 @@ services: # Linux 32-bit # - GOOS=linux # - GOARCH=386 - + # Windows 64-bit # - GOOS=windows # - GOARCH=amd64 diff --git a/main.go b/main.go index ff9b5f7..8f8f7f1 100644 --- a/main.go +++ b/main.go @@ -63,6 +63,13 @@ var flags = []cli.Flag{ Usage: "don't include the first H1 heading in Confluence output.", EnvVars: []string{"MARK_H1_DROP"}, }), + altsrc.NewBoolFlag(&cli.BoolFlag{ + Name: "strip-linebreaks", + Value: false, + Aliases: []string{"L"}, + Usage: "remove linebreaks inside of tags, to accomodate non-standard Confluence behavior", + EnvVars: []string{"MARK_STRIP_LINEBREAK"}, + }), altsrc.NewBoolFlag(&cli.BoolFlag{ Name: "title-from-h1", Value: false, @@ -383,7 +390,7 @@ func processFile( ) } - html, _ := mark.CompileMarkdown(markdown, stdlib, file, cCtx.String("mermaid-provider"), cCtx.Float64("mermaid-scale"), cCtx.Bool("drop-h1")) + html, _ := mark.CompileMarkdown(markdown, stdlib, file, cCtx.String("mermaid-provider"), cCtx.Float64("mermaid-scale"), cCtx.Bool("drop-h1"), cCtx.Bool("strip-linebreaks")) fmt.Println(html) os.Exit(0) } @@ -459,7 +466,7 @@ func processFile( ) } - html, inlineAttachments := mark.CompileMarkdown(markdown, stdlib, file, cCtx.String("mermaid-provider"), cCtx.Float64("mermaid-scale"), cCtx.Bool("drop-h1")) + html, inlineAttachments := mark.CompileMarkdown(markdown, stdlib, file, cCtx.String("mermaid-provider"), cCtx.Float64("mermaid-scale"), cCtx.Bool("drop-h1"), cCtx.Bool("strip-linebreaks")) // Resolve attachements detected from markdown _, err = attachment.ResolveAttachments( diff --git a/pkg/mark/markdown.go b/pkg/mark/markdown.go index f970257..996100d 100644 --- a/pkg/mark/markdown.go +++ b/pkg/mark/markdown.go @@ -26,11 +26,12 @@ type ConfluenceExtension struct { MermaidProvider string MermaidScale float64 DropFirstH1 bool + StripNewlines bool Attachments []attachment.Attachment } // NewConfluenceRenderer creates a new instance of the ConfluenceRenderer -func NewConfluenceExtension(stdlib *stdlib.Lib, path string, mermaidProvider string, mermaidScale float64, dropFirstH1 bool) *ConfluenceExtension { +func NewConfluenceExtension(stdlib *stdlib.Lib, path string, mermaidProvider string, mermaidScale float64, dropFirstH1 bool, stripNewlines bool) *ConfluenceExtension { return &ConfluenceExtension{ Config: html.NewConfig(), Stdlib: stdlib, @@ -38,6 +39,7 @@ func NewConfluenceExtension(stdlib *stdlib.Lib, path string, mermaidProvider str MermaidProvider: mermaidProvider, MermaidScale: mermaidScale, DropFirstH1: dropFirstH1, + StripNewlines: stripNewlines, Attachments: []attachment.Attachment{}, } } @@ -45,6 +47,7 @@ func NewConfluenceExtension(stdlib *stdlib.Lib, path string, mermaidProvider str func (c *ConfluenceExtension) Extend(m goldmark.Markdown) { m.Renderer().AddOptions(renderer.WithNodeRenderers( + util.Prioritized(crenderer.NewConfluenceTextRenderer(c.StripNewlines), 100), util.Prioritized(crenderer.NewConfluenceBlockQuoteRenderer(), 100), util.Prioritized(crenderer.NewConfluenceCodeBlockRenderer(c.Stdlib, c.Path), 100), util.Prioritized(crenderer.NewConfluenceFencedCodeBlockRenderer(c.Stdlib, &c.Attachments, c.MermaidProvider, c.MermaidScale), 100), @@ -61,10 +64,10 @@ func (c *ConfluenceExtension) Extend(m goldmark.Markdown) { )) } -func CompileMarkdown(markdown []byte, stdlib *stdlib.Lib, path string, mermaidProvider string, mermaidScale float64, dropFirstH1 bool) (string, []attachment.Attachment) { +func CompileMarkdown(markdown []byte, stdlib *stdlib.Lib, path string, mermaidProvider string, mermaidScale float64, dropFirstH1 bool, stripNewlines bool) (string, []attachment.Attachment) { log.Tracef(nil, "rendering markdown:\n%s", string(markdown)) - confluenceExtension := NewConfluenceExtension(stdlib, path, mermaidProvider, mermaidScale, dropFirstH1) + confluenceExtension := NewConfluenceExtension(stdlib, path, mermaidProvider, mermaidScale, dropFirstH1, stripNewlines) converter := goldmark.New( goldmark.WithExtensions( diff --git a/pkg/mark/markdown_test.go b/pkg/mark/markdown_test.go index 79edc82..eaf18cd 100644 --- a/pkg/mark/markdown_test.go +++ b/pkg/mark/markdown_test.go @@ -10,6 +10,24 @@ import ( "github.com/stretchr/testify/assert" ) +func loadData(t *testing.T, filename, variant string) ([]byte, string, []byte) { + t.Helper() + basename := filepath.Base(filename) + testname := strings.TrimSuffix(basename, ".md") + htmlname := filepath.Join(filepath.Dir(filename), testname+variant+".html") + + markdown, err := os.ReadFile(filename) + if err != nil { + panic(err) + } + html, err := os.ReadFile(htmlname) + if err != nil { + panic(err) + } + + return markdown, htmlname, html +} + func TestCompileMarkdown(t *testing.T) { test := assert.New(t) @@ -19,24 +37,50 @@ func TestCompileMarkdown(t *testing.T) { } for _, filename := range testcases { - basename := filepath.Base(filename) - testname := strings.TrimSuffix(basename, ".md") - htmlname := filepath.Join(filepath.Dir(filename), testname+".html") - - markdown, err := os.ReadFile(filename) - if err != nil { - panic(err) - } - html, err := os.ReadFile(htmlname) - if err != nil { - panic(err) - } - lib, err := stdlib.New(nil) if err != nil { panic(err) } - actual, _ := CompileMarkdown(markdown, lib, filename, "", 1.0, false) + markdown, htmlname, html := loadData(t, filename, "") + actual, _ := CompileMarkdown(markdown, lib, filename, "", 1.0, false, false) + test.EqualValues(string(html), actual, filename+" vs "+htmlname) + } +} + +func TestCompileMarkdownDropH1(t *testing.T) { + test := assert.New(t) + + testcases, err := filepath.Glob("testdata/*.md") + if err != nil { + panic(err) + } + + for _, filename := range testcases { + lib, err := stdlib.New(nil) + if err != nil { + panic(err) + } + markdown, htmlname, html := loadData(t, filename, "-droph1") + actual, _ := CompileMarkdown(markdown, lib, filename, "", 1.0, true, false) + test.EqualValues(string(html), actual, filename+" vs "+htmlname) + } +} + +func TestCompileMarkdownStripNewlines(t *testing.T) { + test := assert.New(t) + + testcases, err := filepath.Glob("testdata/*.md") + if err != nil { + panic(err) + } + + for _, filename := range testcases { + lib, err := stdlib.New(nil) + if err != nil { + panic(err) + } + markdown, htmlname, html := loadData(t, filename, "-stripnewlines") + actual, _ := CompileMarkdown(markdown, lib, filename, "", 1.0, false, true) test.EqualValues(string(html), actual, filename+" vs "+htmlname) } } diff --git a/pkg/mark/renderer/text.go b/pkg/mark/renderer/text.go new file mode 100644 index 0000000..e7db208 --- /dev/null +++ b/pkg/mark/renderer/text.go @@ -0,0 +1,71 @@ +package renderer + +import ( + "unicode/utf8" + + //"github.com/kovetskiy/mark/pkg/mark/stdlib" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/renderer" + "github.com/yuin/goldmark/renderer/html" + "github.com/yuin/goldmark/util" +) + +type ConfluenceTextRenderer struct { + html.Config + softBreak rune +} + +// NewConfluenceRenderer creates a new instance of the ConfluenceRenderer +func NewConfluenceTextRenderer(stripNL bool, opts ...html.Option) renderer.NodeRenderer { + sb := '\n' + if stripNL { + sb = ' ' + } + return &ConfluenceTextRenderer{ + Config: html.NewConfig(), + softBreak: sb, + } +} + +// RegisterFuncs implements NodeRenderer.RegisterFuncs . +func (r *ConfluenceTextRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { + reg.Register(ast.KindText, r.renderText) +} + +func (r *ConfluenceTextRenderer) renderText(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + n := node.(*ast.Text) + segment := n.Segment + if n.IsRaw() { + r.Writer.RawWrite(w, segment.Value(source)) + } else { + value := segment.Value(source) + r.Writer.Write(w, value) + if n.HardLineBreak() || (n.SoftLineBreak() && r.HardWraps) { + if r.XHTML { + _, _ = w.WriteString("
\n") + } else { + _, _ = w.WriteString("
\n") + } + } else if n.SoftLineBreak() { + if r.EastAsianLineBreaks && len(value) != 0 { + sibling := node.NextSibling() + if sibling != nil && sibling.Kind() == ast.KindText { + if siblingText := sibling.(*ast.Text).Text(source); len(siblingText) != 0 { + thisLastRune := util.ToRune(value, len(value)-1) + siblingFirstRune, _ := utf8.DecodeRune(siblingText) + if !(util.IsEastAsianWideRune(thisLastRune) && + util.IsEastAsianWideRune(siblingFirstRune)) { + _ = w.WriteByte(byte(r.softBreak)) + } + } + } + } else { + _ = w.WriteByte(byte(r.softBreak)) + } + } + } + return ast.WalkContinue, nil +} diff --git a/pkg/mark/testdata/codes-droph1.html b/pkg/mark/testdata/codes-droph1.html new file mode 100644 index 0000000..733410b --- /dev/null +++ b/pkg/mark/testdata/codes-droph1.html @@ -0,0 +1,20 @@ +

inline

+falsebashfalsebashfalseunknownfalse

text +text 2

+unknownfalseshfalseA b cbashtrueA b cctruenestedfalsefalsetruefalseB; + A-->C; + B-->D; + C-->D;]]>truetruemy mermaid graphB; + A-->C; + B-->D; + C-->D;]]>truefalsemy mermaid graphB; + A-->C; + B-->D; + C-->D;]]> \ No newline at end of file diff --git a/pkg/mark/testdata/codes-stripnewlines.html b/pkg/mark/testdata/codes-stripnewlines.html new file mode 100644 index 0000000..63eff28 --- /dev/null +++ b/pkg/mark/testdata/codes-stripnewlines.html @@ -0,0 +1,19 @@ +

inline

+falsebashfalsebashfalseunknownfalse

text text 2

+unknownfalseshfalseA b cbashtrueA b cctruenestedfalsefalsetruefalseB; + A-->C; + B-->D; + C-->D;]]>truetruemy mermaid graphB; + A-->C; + B-->D; + C-->D;]]>truefalsemy mermaid graphB; + A-->C; + B-->D; + C-->D;]]> \ No newline at end of file diff --git a/pkg/mark/testdata/header-droph1.html b/pkg/mark/testdata/header-droph1.html new file mode 100644 index 0000000..c63a140 --- /dev/null +++ b/pkg/mark/testdata/header-droph1.html @@ -0,0 +1,6 @@ +

b

+

c

+

d

+
e
+

f

+

g

diff --git a/pkg/mark/testdata/header-stripnewlines.html b/pkg/mark/testdata/header-stripnewlines.html new file mode 100644 index 0000000..566c4ae --- /dev/null +++ b/pkg/mark/testdata/header-stripnewlines.html @@ -0,0 +1,7 @@ +

a

+

b

+

c

+

d

+
e
+

f

+

g

diff --git a/pkg/mark/testdata/issue-64-broken-link-droph1.html b/pkg/mark/testdata/issue-64-broken-link-droph1.html new file mode 100644 index 0000000..0d39ce8 --- /dev/null +++ b/pkg/mark/testdata/issue-64-broken-link-droph1.html @@ -0,0 +1 @@ +

v71

diff --git a/pkg/mark/testdata/issue-64-broken-link-stripnewlines.html b/pkg/mark/testdata/issue-64-broken-link-stripnewlines.html new file mode 100644 index 0000000..0d39ce8 --- /dev/null +++ b/pkg/mark/testdata/issue-64-broken-link-stripnewlines.html @@ -0,0 +1 @@ +

v71

diff --git a/pkg/mark/testdata/links-droph1.html b/pkg/mark/testdata/links-droph1.html new file mode 100644 index 0000000..82fd040 --- /dev/null +++ b/pkg/mark/testdata/links-droph1.html @@ -0,0 +1,17 @@ +

Use https://example.com

+

Use aaa

+

Use

+

Use

+

Use

+

Use

+

+

+

Use footnotes link 1

+
+
+
    +
  1. +

    a footnote link ↩︎

    +
    2. +
+
diff --git a/pkg/mark/testdata/links-stripnewlines.html b/pkg/mark/testdata/links-stripnewlines.html new file mode 100644 index 0000000..82fd040 --- /dev/null +++ b/pkg/mark/testdata/links-stripnewlines.html @@ -0,0 +1,17 @@ +

Use https://example.com

+

Use aaa

+

Use

+

Use

+

Use

+

Use

+

+

+

Use footnotes link 1

+
+
+
    +
  1. +

    a footnote link ↩︎

    +
    2. +
+
diff --git a/pkg/mark/testdata/lists-droph1.html b/pkg/mark/testdata/lists-droph1.html new file mode 100644 index 0000000..af27c54 --- /dev/null +++ b/pkg/mark/testdata/lists-droph1.html @@ -0,0 +1,21 @@ + +

text

+ diff --git a/pkg/mark/testdata/lists-stripnewlines.html b/pkg/mark/testdata/lists-stripnewlines.html new file mode 100644 index 0000000..af27c54 --- /dev/null +++ b/pkg/mark/testdata/lists-stripnewlines.html @@ -0,0 +1,21 @@ + +

text

+ diff --git a/pkg/mark/testdata/macro-include-droph1.html b/pkg/mark/testdata/macro-include-droph1.html new file mode 100644 index 0000000..8d8ba05 --- /dev/null +++ b/pkg/mark/testdata/macro-include-droph1.html @@ -0,0 +1,6 @@ +

bar

+

+true +Attention +This is an info! +

diff --git a/pkg/mark/testdata/macro-include-stripnewlines.html b/pkg/mark/testdata/macro-include-stripnewlines.html new file mode 100644 index 0000000..52e4af7 --- /dev/null +++ b/pkg/mark/testdata/macro-include-stripnewlines.html @@ -0,0 +1,2 @@ +

bar

+

true Attention This is an info!

diff --git a/pkg/mark/testdata/newlines-droph1.html b/pkg/mark/testdata/newlines-droph1.html new file mode 100644 index 0000000..b97e510 --- /dev/null +++ b/pkg/mark/testdata/newlines-droph1.html @@ -0,0 +1,10 @@ +

one-1 +one-2

+

two-1

+

two-2

+

three-1

+

three-2

+

space-1 +space-2

+

2space-1
+2space-2

diff --git a/pkg/mark/testdata/newlines-stripnewlines.html b/pkg/mark/testdata/newlines-stripnewlines.html new file mode 100644 index 0000000..2fd4dc0 --- /dev/null +++ b/pkg/mark/testdata/newlines-stripnewlines.html @@ -0,0 +1,8 @@ +

one-1 one-2

+

two-1

+

two-2

+

three-1

+

three-2

+

space-1 space-2

+

2space-1
+2space-2

diff --git a/pkg/mark/testdata/pagelayout-droph1.html b/pkg/mark/testdata/pagelayout-droph1.html new file mode 100644 index 0000000..66f5783 --- /dev/null +++ b/pkg/mark/testdata/pagelayout-droph1.html @@ -0,0 +1,18 @@ + + + +

More Content

+ + +

More Content

+ + +

Even More Content

+ + + + +

Still More Content

+ + + diff --git a/pkg/mark/testdata/pagelayout-stripnewlines.html b/pkg/mark/testdata/pagelayout-stripnewlines.html new file mode 100644 index 0000000..66f5783 --- /dev/null +++ b/pkg/mark/testdata/pagelayout-stripnewlines.html @@ -0,0 +1,18 @@ + + + +

More Content

+ + +

More Content

+ + +

Even More Content

+ + + + +

Still More Content

+ + + diff --git a/pkg/mark/testdata/quotes-droph1.html b/pkg/mark/testdata/quotes-droph1.html new file mode 100644 index 0000000..d8fdcdf --- /dev/null +++ b/pkg/mark/testdata/quotes-droph1.html @@ -0,0 +1,34 @@ +

First Heading

+true +

NOTES:

+
    +
  1. Note number one
    2. +
  2. Note number two
    3. +
+
+

a +b

+
+

Warn (Should not be picked as blockquote type)

+ +

Second Heading

+true +

Warn

+ + + +

Third Heading

+true + +

Test

+ +

Simple Blockquote

+
+

This paragraph is a simple blockquote

+
diff --git a/pkg/mark/testdata/quotes-stripnewlines.html b/pkg/mark/testdata/quotes-stripnewlines.html new file mode 100644 index 0000000..7a43019 --- /dev/null +++ b/pkg/mark/testdata/quotes-stripnewlines.html @@ -0,0 +1,33 @@ +

Main Heading

+

First Heading

+true +

NOTES:

+
    +
  1. Note number one
    2. +
  2. Note number two
    3. +
+
+

a b

+
+

Warn (Should not be picked as blockquote type)

+ +

Second Heading

+true +

Warn

+ + + +

Third Heading

+true + +

Test

+ +

Simple Blockquote

+
+

This paragraph is a simple blockquote

+
diff --git a/pkg/mark/testdata/quotes.html b/pkg/mark/testdata/quotes.html index acb61d6..c1519c7 100644 --- a/pkg/mark/testdata/quotes.html +++ b/pkg/mark/testdata/quotes.html @@ -20,6 +20,10 @@ b

  • Warn bullet 2
    • +

    Third Heading

    true diff --git a/pkg/mark/testdata/quotes.md b/pkg/mark/testdata/quotes.md index f6b9958..995f67f 100644 --- a/pkg/mark/testdata/quotes.md +++ b/pkg/mark/testdata/quotes.md @@ -19,6 +19,8 @@ > * Warn bullet 1 > * Warn bullet 2 +* Regular list + that runs long ## Third Heading > diff --git a/pkg/mark/testdata/table-droph1.html b/pkg/mark/testdata/table-droph1.html new file mode 100644 index 0000000..9b4f80a --- /dev/null +++ b/pkg/mark/testdata/table-droph1.html @@ -0,0 +1,14 @@ + + + + + + + + + + + + + +
    HEADER1HEADER2
    row1row2
    diff --git a/pkg/mark/testdata/table-stripnewlines.html b/pkg/mark/testdata/table-stripnewlines.html new file mode 100644 index 0000000..9b4f80a --- /dev/null +++ b/pkg/mark/testdata/table-stripnewlines.html @@ -0,0 +1,14 @@ + + + + + + + + + + + + + +
    HEADER1HEADER2
    row1row2
    diff --git a/pkg/mark/testdata/tags-droph1.html b/pkg/mark/testdata/tags-droph1.html new file mode 100644 index 0000000..a5161db --- /dev/null +++ b/pkg/mark/testdata/tags-droph1.html @@ -0,0 +1,6 @@ +

    bold +bold

    +

    vitalik +vitalik

    +

    strikethrough +strikethrough

    diff --git a/pkg/mark/testdata/tags-stripnewlines.html b/pkg/mark/testdata/tags-stripnewlines.html new file mode 100644 index 0000000..cc0698b --- /dev/null +++ b/pkg/mark/testdata/tags-stripnewlines.html @@ -0,0 +1,3 @@ +

    bold bold

    +

    vitalik vitalik

    +

    strikethrough strikethrough