Refactor renders (#15175)

* Refactor renders

* Some performance optimization

* Fix comment

* Transform reader

* Fix csv test

* Fix test

* Fix tests

* Improve optimaziation

* Fix test

* Fix test

* Detect file encoding with reader

* Improve optimaziation

* reduce memory usage

* improve code

* fix build

* Fix test

* Fix for go1.15

* Fix render

* Fix comment

* Fix lint

* Fix test

* Don't use NormalEOF when unnecessary

* revert change on util.go

* Apply suggestions from code review

Co-authored-by: zeripath <art27@cantab.net>

* rename function

* Take NormalEOF back

Co-authored-by: zeripath <art27@cantab.net>

contrib/pr/checkout.go

@@ -114,7 +114,7 @@ func runPR() {
 
 	log.Printf("[PR] Setting up router\n")
 	//routers.GlobalInit()
-	external.RegisterParsers()
+	external.RegisterRenderers()
 	markup.Init()
 	c := routes.NormalRoutes()
 

models/issue_comment.go

@@ -16,6 +16,7 @@ import (
 
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/markdown"
 	"code.gitea.io/gitea/modules/references"
 	"code.gitea.io/gitea/modules/structs"
@@ -1178,8 +1179,13 @@ func findCodeComments(e Engine, opts FindCommentsOptions, issue *Issue, currentU
 			return nil, err
 		}
 
-		comment.RenderedContent = string(markdown.Render([]byte(comment.Content), issue.Repo.Link(),
-			issue.Repo.ComposeMetas()))
+		var err error
+		if comment.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
+			URLPrefix: issue.Repo.Link(),
+			Metas:     issue.Repo.ComposeMetas(),
+		}, comment.Content); err != nil {
+			return nil, err
+		}
 	}
 	return comments[:n], nil
 }

models/repo.go

@@ -863,7 +863,10 @@ func (repo *Repository) getUsersWithAccessMode(e Engine, mode AccessMode) (_ []*
 
 // DescriptionHTML does special handles to description and return HTML string.
 func (repo *Repository) DescriptionHTML() template.HTML {
-	desc, err := markup.RenderDescriptionHTML([]byte(repo.Description), repo.HTMLURL(), repo.ComposeMetas())
+	desc, err := markup.RenderDescriptionHTML(&markup.RenderContext{
+		URLPrefix: repo.HTMLURL(),
+		Metas:     repo.ComposeMetas(),
+	}, repo.Description)
 	if err != nil {
 		log.Error("Failed to render description for %s (ID: %d): %v", repo.Name, repo.ID, err)
 		return template.HTML(markup.Sanitize(repo.Description))

models/repo_generate.go

@@ -5,13 +5,14 @@
 package models
 
 import (
+	"bufio"
+	"bytes"
 	"strconv"
 	"strings"
 
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/storage"
-	"code.gitea.io/gitea/modules/util"
 
 	"github.com/gobwas/glob"
 )
@@ -49,9 +50,9 @@ func (gt GiteaTemplate) Globs() []glob.Glob {
 	}
 
 	gt.globs = make([]glob.Glob, 0)
-	lines := strings.Split(string(util.NormalizeEOL(gt.Content)), "\n")
-	for _, line := range lines {
-		line = strings.TrimSpace(line)
+	scanner := bufio.NewScanner(bytes.NewReader(gt.Content))
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
 		if line == "" || strings.HasPrefix(line, "#") {
 			continue
 		}

modules/charset/charset.go

@@ -7,6 +7,8 @@ package charset
 import (
 	"bytes"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"strings"
 	"unicode/utf8"
 
@@ -21,6 +23,33 @@ import (
 // UTF8BOM is the utf-8 byte-order marker
 var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'}
 
+// ToUTF8WithFallbackReader detects the encoding of content and coverts to UTF-8 reader if possible
+func ToUTF8WithFallbackReader(rd io.Reader) io.Reader {
+	var buf = make([]byte, 2048)
+	n, err := rd.Read(buf)
+	if err != nil {
+		return rd
+	}
+
+	charsetLabel, err := DetectEncoding(buf[:n])
+	if err != nil || charsetLabel == "UTF-8" {
+		return io.MultiReader(bytes.NewReader(RemoveBOMIfPresent(buf[:n])), rd)
+	}
+
+	encoding, _ := charset.Lookup(charsetLabel)
+	if encoding == nil {
+		return io.MultiReader(bytes.NewReader(buf[:n]), rd)
+	}
+
+	return transform.NewReader(
+		io.MultiReader(
+			bytes.NewReader(RemoveBOMIfPresent(buf[:n])),
+			rd,
+		),
+		encoding.NewDecoder(),
+	)
+}
+
 // ToUTF8WithErr converts content to UTF8 encoding
 func ToUTF8WithErr(content []byte) (string, error) {
 	charsetLabel, err := DetectEncoding(content)
@@ -49,24 +78,8 @@ func ToUTF8WithErr(content []byte) (string, error) {
 
 // ToUTF8WithFallback detects the encoding of content and coverts to UTF-8 if possible
 func ToUTF8WithFallback(content []byte) []byte {
-	charsetLabel, err := DetectEncoding(content)
-	if err != nil || charsetLabel == "UTF-8" {
-		return RemoveBOMIfPresent(content)
-	}
-
-	encoding, _ := charset.Lookup(charsetLabel)
-	if encoding == nil {
-		return content
-	}
-
-	// If there is an error, we concatenate the nicely decoded part and the
-	// original left over. This way we won't lose data.
-	result, n, err := transform.Bytes(encoding.NewDecoder(), content)
-	if err != nil {
-		return append(result, content[n:]...)
-	}
-
-	return RemoveBOMIfPresent(result)
+	bs, _ := ioutil.ReadAll(ToUTF8WithFallbackReader(bytes.NewReader(content)))
+	return bs
 }
 
 // ToUTF8 converts content to UTF8 encoding and ignore error

modules/csv/csv.go

@@ -7,7 +7,9 @@ package csv
 import (
 	"bytes"
 	"encoding/csv"
+	stdcsv "encoding/csv"
 	"errors"
+	"io"
 	"regexp"
 	"strings"
 
@@ -18,17 +20,31 @@ import (
 var quoteRegexp = regexp.MustCompile(`["'][\s\S]+?["']`)
 
 // CreateReader creates a csv.Reader with the given delimiter.
-func CreateReader(rawBytes []byte, delimiter rune) *csv.Reader {
-	rd := csv.NewReader(bytes.NewReader(rawBytes))
+func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader {
+	rd := stdcsv.NewReader(input)
 	rd.Comma = delimiter
 	rd.TrimLeadingSpace = true
 	return rd
 }
 
 // CreateReaderAndGuessDelimiter tries to guess the field delimiter from the content and creates a csv.Reader.
-func CreateReaderAndGuessDelimiter(rawBytes []byte) *csv.Reader {
-	delimiter := guessDelimiter(rawBytes)
-	return CreateReader(rawBytes, delimiter)
+func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) {
+	var data = make([]byte, 1e4)
+	size, err := rd.Read(data)
+	if err != nil {
+		return nil, err
+	}
+
+	delimiter := guessDelimiter(data[:size])
+
+	var newInput io.Reader
+	if size < 1e4 {
+		newInput = bytes.NewReader(data[:size])
+	} else {
+		newInput = io.MultiReader(bytes.NewReader(data), rd)
+	}
+
+	return CreateReader(newInput, delimiter), nil
 }
 
 // guessDelimiter scores the input CSV data against delimiters, and returns the best match.

modules/csv/csv_test.go

@@ -5,20 +5,23 @@
 package csv
 
 import (
+	"bytes"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
 func TestCreateReader(t *testing.T) {
-	rd := CreateReader([]byte{}, ',')
+	rd := CreateReader(bytes.NewReader([]byte{}), ',')
 	assert.Equal(t, ',', rd.Comma)
 }
 
 func TestCreateReaderAndGuessDelimiter(t *testing.T) {
 	input := "a;b;c\n1;2;3\n4;5;6"
 
-	rd := CreateReaderAndGuessDelimiter([]byte(input))
+	rd, err := CreateReaderAndGuessDelimiter(strings.NewReader(input))
+	assert.NoError(t, err)
 	assert.Equal(t, ';', rd.Comma)
 }
 

modules/markup/csv/csv.go

@@ -5,9 +5,11 @@
 package markup
 
 import (
+	"bufio"
 	"bytes"
 	"html"
 	"io"
+	"io/ioutil"
 	"strconv"
 
 	"code.gitea.io/gitea/modules/csv"
@@ -16,55 +18,89 @@ import (
 )
 
 func init() {
-	markup.RegisterParser(Parser{})
+	markup.RegisterRenderer(Renderer{})
 }
 
-// Parser implements markup.Parser for csv files
-type Parser struct {
+// Renderer implements markup.Renderer for csv files
+type Renderer struct {
 }
 
-// Name implements markup.Parser
-func (Parser) Name() string {
+// Name implements markup.Renderer
+func (Renderer) Name() string {
 	return "csv"
 }
 
-// NeedPostProcess implements markup.Parser
-func (Parser) NeedPostProcess() bool { return false }
+// NeedPostProcess implements markup.Renderer
+func (Renderer) NeedPostProcess() bool { return false }
 
-// Extensions implements markup.Parser
-func (Parser) Extensions() []string {
+// Extensions implements markup.Renderer
+func (Renderer) Extensions() []string {
 	return []string{".csv", ".tsv"}
 }
 
-// Render implements markup.Parser
-func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
-	var tmpBlock bytes.Buffer
+func writeField(w io.Writer, element, class, field string) error {
+	if _, err := io.WriteString(w, "<"); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, element); err != nil {
+		return err
+	}
+	if len(class) > 0 {
+		if _, err := io.WriteString(w, " class=\""); err != nil {
+			return err
+		}
+		if _, err := io.WriteString(w, class); err != nil {
+			return err
+		}
+		if _, err := io.WriteString(w, "\""); err != nil {
+			return err
+		}
+	}
+	if _, err := io.WriteString(w, ">"); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, html.EscapeString(field)); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, "</"); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, element); err != nil {
+		return err
+	}
+	_, err := io.WriteString(w, ">")
+	return err
+}
+
+// Render implements markup.Renderer
+func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
+	var tmpBlock = bufio.NewWriter(output)
+
+	// FIXME: don't read all to memory
+	rawBytes, err := ioutil.ReadAll(input)
+	if err != nil {
+		return err
+	}
 
 	if setting.UI.CSV.MaxFileSize != 0 && setting.UI.CSV.MaxFileSize < int64(len(rawBytes)) {
-		tmpBlock.WriteString("<pre>")
-		tmpBlock.WriteString(html.EscapeString(string(rawBytes)))
-		tmpBlock.WriteString("</pre>")
-		return tmpBlock.Bytes()
-	}
-
-	rd := csv.CreateReaderAndGuessDelimiter(rawBytes)
-
-	writeField := func(element, class, field string) {
-		tmpBlock.WriteString("<")
-		tmpBlock.WriteString(element)
-		if len(class) > 0 {
-			tmpBlock.WriteString(" class=\"")
-			tmpBlock.WriteString(class)
-			tmpBlock.WriteString("\"")
+		if _, err := tmpBlock.WriteString("<pre>"); err != nil {
+			return err
 		}
-		tmpBlock.WriteString(">")
-		tmpBlock.WriteString(html.EscapeString(field))
-		tmpBlock.WriteString("</")
-		tmpBlock.WriteString(element)
-		tmpBlock.WriteString(">")
+		if _, err := tmpBlock.WriteString(html.EscapeString(string(rawBytes))); err != nil {
+			return err
+		}
+		_, err = tmpBlock.WriteString("</pre>")
+		return err
 	}
 
-	tmpBlock.WriteString(`<table class="data-table">`)
+	rd, err := csv.CreateReaderAndGuessDelimiter(bytes.NewReader(rawBytes))
+	if err != nil {
+		return err
+	}
+
+	if _, err := tmpBlock.WriteString(`<table class="data-table">`); err != nil {
+		return err
+	}
 	row := 1
 	for {
 		fields, err := rd.Read()
@@ -74,20 +110,29 @@ func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string,
 		if err != nil {
 			continue
 		}
-		tmpBlock.WriteString("<tr>")
+		if _, err := tmpBlock.WriteString("<tr>"); err != nil {
+			return err
+		}
 		element := "td"
 		if row == 1 {
 			element = "th"
 		}
-		writeField(element, "line-num", strconv.Itoa(row))
-		for _, field := range fields {
-			writeField(element, "", field)
+		if err := writeField(tmpBlock, element, "line-num", strconv.Itoa(row)); err != nil {
+			return err
+		}
+		for _, field := range fields {
+			if err := writeField(tmpBlock, element, "", field); err != nil {
+				return err
+			}
+		}
+		if _, err := tmpBlock.WriteString("</tr>"); err != nil {
+			return err
 		}
-		tmpBlock.WriteString("</tr>")
 
 		row++
 	}
-	tmpBlock.WriteString("</table>")
-
-	return tmpBlock.Bytes()
+	if _, err = tmpBlock.WriteString("</table>"); err != nil {
+		return err
+	}
+	return tmpBlock.Flush()
 }

modules/markup/csv/csv_test.go

@@ -5,13 +5,16 @@
 package markup
 
 import (
+	"strings"
 	"testing"
 
+	"code.gitea.io/gitea/modules/markup"
+
 	"github.com/stretchr/testify/assert"
 )
 
 func TestRenderCSV(t *testing.T) {
-	var parser Parser
+	var render Renderer
 	var kases = map[string]string{
 		"a":        "<table class=\"data-table\"><tr><th class=\"line-num\">1</th><th>a</th></tr></table>",
 		"1,2":      "<table class=\"data-table\"><tr><th class=\"line-num\">1</th><th>1</th><th>2</th></tr></table>",
@@ -20,7 +23,9 @@ func TestRenderCSV(t *testing.T) {
 	}
 
 	for k, v := range kases {
-		res := parser.Render([]byte(k), "", nil, false)
-		assert.EqualValues(t, v, string(res))
+		var buf strings.Builder
+		err := render.Render(&markup.RenderContext{}, strings.NewReader(k), &buf)
+		assert.NoError(t, err)
+		assert.EqualValues(t, v, buf.String())
 	}
 }

modules/markup/external/external.go

@@ -5,7 +5,7 @@
 package external
 
 import (
-	"bytes"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
@@ -19,32 +19,32 @@ import (
 	"code.gitea.io/gitea/modules/util"
 )
 
-// RegisterParsers registers all supported third part parsers according settings
-func RegisterParsers() {
-	for _, parser := range setting.ExternalMarkupParsers {
-		if parser.Enabled && parser.Command != "" && len(parser.FileExtensions) > 0 {
-			markup.RegisterParser(&Parser{parser})
+// RegisterRenderers registers all supported third part renderers according settings
+func RegisterRenderers() {
+	for _, renderer := range setting.ExternalMarkupRenderers {
+		if renderer.Enabled && renderer.Command != "" && len(renderer.FileExtensions) > 0 {
+			markup.RegisterRenderer(&Renderer{renderer})
 		}
 	}
 }
 
-// Parser implements markup.Parser for external tools
-type Parser struct {
-	setting.MarkupParser
+// Renderer implements markup.Renderer for external tools
+type Renderer struct {
+	setting.MarkupRenderer
 }
 
 // Name returns the external tool name
-func (p *Parser) Name() string {
+func (p *Renderer) Name() string {
 	return p.MarkupName
 }
 
-// NeedPostProcess implements markup.Parser
-func (p *Parser) NeedPostProcess() bool {
-	return p.MarkupParser.NeedPostProcess
+// NeedPostProcess implements markup.Renderer
+func (p *Renderer) NeedPostProcess() bool {
+	return p.MarkupRenderer.NeedPostProcess
 }
 
 // Extensions returns the supported extensions of the tool
-func (p *Parser) Extensions() []string {
+func (p *Renderer) Extensions() []string {
 	return p.FileExtensions
 }
 
@@ -56,14 +56,10 @@ func envMark(envName string) string {
 }
 
 // Render renders the data of the document to HTML via the external tool.
-func (p *Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
+func (p *Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
 	var (
-		bs           []byte
-		buf          = bytes.NewBuffer(bs)
-		rd           = bytes.NewReader(rawBytes)
-		urlRawPrefix = strings.Replace(urlPrefix, "/src/", "/raw/", 1)
-
-		command = strings.NewReplacer(envMark("GITEA_PREFIX_SRC"), urlPrefix,
+		urlRawPrefix = strings.Replace(ctx.URLPrefix, "/src/", "/raw/", 1)
+		command      = strings.NewReplacer(envMark("GITEA_PREFIX_SRC"), ctx.URLPrefix,
 			envMark("GITEA_PREFIX_RAW"), urlRawPrefix).Replace(p.Command)
 		commands = strings.Fields(command)
 		args     = commands[1:]
@@ -73,8 +69,7 @@ func (p *Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]stri
 		// write to temp file
 		f, err := ioutil.TempFile("", "gitea_input")
 		if err != nil {
-			log.Error("%s create temp file when rendering %s failed: %v", p.Name(), p.Command, err)
-			return []byte("")
+			return fmt.Errorf("%s create temp file when rendering %s failed: %v", p.Name(), p.Command, err)
 		}
 		tmpPath := f.Name()
 		defer func() {
@@ -83,17 +78,15 @@ func (p *Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]stri
 			}
 		}()
 
-		_, err = io.Copy(f, rd)
+		_, err = io.Copy(f, input)
 		if err != nil {
 			f.Close()
-			log.Error("%s write data to temp file when rendering %s failed: %v", p.Name(), p.Command, err)
-			return []byte("")
+			return fmt.Errorf("%s write data to temp file when rendering %s failed: %v", p.Name(), p.Command, err)
 		}
 
 		err = f.Close()
 		if err != nil {
-			log.Error("%s close temp file when rendering %s failed: %v", p.Name(), p.Command, err)
-			return []byte("")
+			return fmt.Errorf("%s close temp file when rendering %s failed: %v", p.Name(), p.Command, err)
 		}
 		args = append(args, f.Name())
 	}
@@ -101,16 +94,15 @@ func (p *Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]stri
 	cmd := exec.Command(commands[0], args...)
 	cmd.Env = append(
 		os.Environ(),
-		"GITEA_PREFIX_SRC="+urlPrefix,
+		"GITEA_PREFIX_SRC="+ctx.URLPrefix,
 		"GITEA_PREFIX_RAW="+urlRawPrefix,
 	)
 	if !p.IsInputFile {
-		cmd.Stdin = rd
+		cmd.Stdin = input
 	}
-	cmd.Stdout = buf
+	cmd.Stdout = output
 	if err := cmd.Run(); err != nil {
-		log.Error("%s render run command %s %v failed: %v", p.Name(), commands[0], args, err)
-		return []byte("")
+		return fmt.Errorf("%s render run command %s %v failed: %v", p.Name(), commands[0], args, err)
 	}
-	return buf.Bytes()
+	return nil
 }

modules/markup/html.go

@@ -7,6 +7,8 @@ package markup
 import (
 	"bytes"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"net/url"
 	"path"
 	"path/filepath"
@@ -144,7 +146,7 @@ func (p *postProcessError) Error() string {
 	return "PostProcess: " + p.context + ", " + p.err.Error()
 }
 
-type processor func(ctx *postProcessCtx, node *html.Node)
+type processor func(ctx *RenderContext, node *html.Node)
 
 var defaultProcessors = []processor{
 	fullIssuePatternProcessor,
@@ -159,34 +161,17 @@ var defaultProcessors = []processor{
 	emojiShortCodeProcessor,
 }
 
-type postProcessCtx struct {
-	metas          map[string]string
-	urlPrefix      string
-	isWikiMarkdown bool
-
-	// processors used by this context.
-	procs []processor
-}
-
 // PostProcess does the final required transformations to the passed raw HTML
 // data, and ensures its validity. Transformations include: replacing links and
 // emails with HTML links, parsing shortlinks in the format of [[Link]], like
 // MediaWiki, linking issues in the format #ID, and mentions in the format
 // @user, and others.
 func PostProcess(
-	rawHTML []byte,
-	urlPrefix string,
-	metas map[string]string,
-	isWikiMarkdown bool,
-) ([]byte, error) {
-	// create the context from the parameters
-	ctx := &postProcessCtx{
-		metas:          metas,
-		urlPrefix:      urlPrefix,
-		isWikiMarkdown: isWikiMarkdown,
-		procs:          defaultProcessors,
-	}
-	return ctx.postProcess(rawHTML)
+	ctx *RenderContext,
+	input io.Reader,
+	output io.Writer,
+) error {
+	return postProcess(ctx, defaultProcessors, input, output)
 }
 
 var commitMessageProcessors = []processor{
@@ -205,23 +190,18 @@ var commitMessageProcessors = []processor{
 // the shortLinkProcessor and will add a defaultLinkProcessor if defaultLink is
 // set, which changes every text node into a link to the passed default link.
 func RenderCommitMessage(
-	rawHTML []byte,
-	urlPrefix, defaultLink string,
-	metas map[string]string,
-) ([]byte, error) {
-	ctx := &postProcessCtx{
-		metas:     metas,
-		urlPrefix: urlPrefix,
-		procs:     commitMessageProcessors,
-	}
-	if defaultLink != "" {
+	ctx *RenderContext,
+	content string,
+) (string, error) {
+	var procs = commitMessageProcessors
+	if ctx.DefaultLink != "" {
 		// we don't have to fear data races, because being
 		// commitMessageProcessors of fixed len and cap, every time we append
 		// something to it the slice is realloc+copied, so append always
 		// generates the slice ex-novo.
-		ctx.procs = append(ctx.procs, genDefaultLinkProcessor(defaultLink))
+		procs = append(procs, genDefaultLinkProcessor(ctx.DefaultLink))
 	}
-	return ctx.postProcess(rawHTML)
+	return renderProcessString(ctx, procs, content)
 }
 
 var commitMessageSubjectProcessors = []processor{
@@ -245,83 +225,72 @@ var emojiProcessors = []processor{
 // emailAddressProcessor, will add a defaultLinkProcessor if defaultLink is set,
 // which changes every text node into a link to the passed default link.
 func RenderCommitMessageSubject(
-	rawHTML []byte,
-	urlPrefix, defaultLink string,
-	metas map[string]string,
-) ([]byte, error) {
-	ctx := &postProcessCtx{
-		metas:     metas,
-		urlPrefix: urlPrefix,
-		procs:     commitMessageSubjectProcessors,
-	}
-	if defaultLink != "" {
+	ctx *RenderContext,
+	content string,
+) (string, error) {
+	var procs = commitMessageSubjectProcessors
+	if ctx.DefaultLink != "" {
 		// we don't have to fear data races, because being
 		// commitMessageSubjectProcessors of fixed len and cap, every time we
 		// append something to it the slice is realloc+copied, so append always
 		// generates the slice ex-novo.
-		ctx.procs = append(ctx.procs, genDefaultLinkProcessor(defaultLink))
+		procs = append(procs, genDefaultLinkProcessor(ctx.DefaultLink))
 	}
-	return ctx.postProcess(rawHTML)
+	return renderProcessString(ctx, procs, content)
 }
 
 // RenderIssueTitle to process title on individual issue/pull page
 func RenderIssueTitle(
-	rawHTML []byte,
-	urlPrefix string,
-	metas map[string]string,
-) ([]byte, error) {
-	ctx := &postProcessCtx{
-		metas:     metas,
-		urlPrefix: urlPrefix,
-		procs: []processor{
-			issueIndexPatternProcessor,
-			sha1CurrentPatternProcessor,
-			emojiShortCodeProcessor,
-			emojiProcessor,
-		},
+	ctx *RenderContext,
+	title string,
+) (string, error) {
+	return renderProcessString(ctx, []processor{
+		issueIndexPatternProcessor,
+		sha1CurrentPatternProcessor,
+		emojiShortCodeProcessor,
+		emojiProcessor,
+	}, title)
+}
+
+func renderProcessString(ctx *RenderContext, procs []processor, content string) (string, error) {
+	var buf strings.Builder
+	if err := postProcess(ctx, procs, strings.NewReader(content), &buf); err != nil {
+		return "", err
 	}
-	return ctx.postProcess(rawHTML)
+	return buf.String(), nil
 }
 
 // RenderDescriptionHTML will use similar logic as PostProcess, but will
 // use a single special linkProcessor.
 func RenderDescriptionHTML(
-	rawHTML []byte,
-	urlPrefix string,
-	metas map[string]string,
-) ([]byte, error) {
-	ctx := &postProcessCtx{
-		metas:     metas,
-		urlPrefix: urlPrefix,
-		procs: []processor{
-			descriptionLinkProcessor,
-			emojiShortCodeProcessor,
-			emojiProcessor,
-		},
-	}
-	return ctx.postProcess(rawHTML)
+	ctx *RenderContext,
+	content string,
+) (string, error) {
+	return renderProcessString(ctx, []processor{
+		descriptionLinkProcessor,
+		emojiShortCodeProcessor,
+		emojiProcessor,
+	}, content)
 }
 
 // RenderEmoji for when we want to just process emoji and shortcodes
 // in various places it isn't already run through the normal markdown procesor
 func RenderEmoji(
-	rawHTML []byte,
-) ([]byte, error) {
-	ctx := &postProcessCtx{
-		procs: emojiProcessors,
-	}
-	return ctx.postProcess(rawHTML)
+	content string,
+) (string, error) {
+	return renderProcessString(&RenderContext{}, emojiProcessors, content)
 }
 
 var tagCleaner = regexp.MustCompile(`<((?:/?\w+/\w+)|(?:/[\w ]+/)|(/?[hH][tT][mM][lL]\b)|(/?[hH][eE][aA][dD]\b))`)
 var nulCleaner = strings.NewReplacer("\000", "")
 
-func (ctx *postProcessCtx) postProcess(rawHTML []byte) ([]byte, error) {
-	if ctx.procs == nil {
-		ctx.procs = defaultProcessors
+func postProcess(ctx *RenderContext, procs []processor, input io.Reader, output io.Writer) error {
+	// FIXME: don't read all content to memory
+	rawHTML, err := ioutil.ReadAll(input)
+	if err != nil {
+		return err
 	}
 
-	// give a generous extra 50 bytes
 	res := bytes.NewBuffer(make([]byte, 0, len(rawHTML)+50))
 	// prepend "<html><body>"
 	_, _ = res.WriteString("<html><body>")
@@ -335,11 +304,11 @@ func (ctx *postProcessCtx) postProcess(rawHTML []byte) ([]byte, error) {
 	// parse the HTML
 	nodes, err := html.ParseFragment(res, nil)
 	if err != nil {
-		return nil, &postProcessError{"invalid HTML", err}
+		return &postProcessError{"invalid HTML", err}
 	}
 
 	for _, node := range nodes {
-		ctx.visitNode(node, true)
+		visitNode(ctx, procs, node, true)
 	}
 
 	newNodes := make([]*html.Node, 0, len(nodes))
@@ -365,25 +334,17 @@ func (ctx *postProcessCtx) postProcess(rawHTML []byte) ([]byte, error) {
 		}
 	}
 
-	nodes = newNodes
-
-	// Create buffer in which the data will be placed again. We know that the
-	// length will be at least that of res; to spare a few alloc+copy, we
-	// reuse res, resetting its length to 0.
-	res.Reset()
 	// Render everything to buf.
-	for _, node := range nodes {
-		err = html.Render(res, node)
+	for _, node := range newNodes {
+		err = html.Render(output, node)
 		if err != nil {
-			return nil, &postProcessError{"error rendering processed HTML", err}
+			return &postProcessError{"error rendering processed HTML", err}
 		}
 	}
-
-	// Everything done successfully, return parsed data.
-	return res.Bytes(), nil
+	return nil
 }
 
-func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) {
+func visitNode(ctx *RenderContext, procs []processor, node *html.Node, visitText bool) {
 	// Add user-content- to IDs if they don't already have them
 	for idx, attr := range node.Attr {
 		if attr.Key == "id" && !(strings.HasPrefix(attr.Val, "user-content-") || blackfridayExtRegex.MatchString(attr.Val)) {
@@ -399,7 +360,7 @@ func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) {
 	switch node.Type {
 	case html.TextNode:
 		if visitText {
-			ctx.textNode(node)
+			textNode(ctx, procs, node)
 		}
 	case html.ElementNode:
 		if node.Data == "img" {
@@ -410,8 +371,8 @@ func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) {
 				}
 				link := []byte(attr.Val)
 				if len(link) > 0 && !IsLink(link) {
-					prefix := ctx.urlPrefix
-					if ctx.isWikiMarkdown {
+					prefix := ctx.URLPrefix
+					if ctx.IsWiki {
 						prefix = util.URLJoin(prefix, "wiki", "raw")
 					}
 					prefix = strings.Replace(prefix, "/src/", "/media/", 1)
@@ -449,7 +410,7 @@ func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) {
 			}
 		}
 		for n := node.FirstChild; n != nil; n = n.NextSibling {
-			ctx.visitNode(n, visitText)
+			visitNode(ctx, procs, n, visitText)
 		}
 	}
 	// ignore everything else
@@ -457,8 +418,8 @@ func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) {
 
 // textNode runs the passed node through various processors, in order to handle
 // all kinds of special links handled by the post-processing.
-func (ctx *postProcessCtx) textNode(node *html.Node) {
-	for _, processor := range ctx.procs {
+func textNode(ctx *RenderContext, procs []processor, node *html.Node) {
+	for _, processor := range procs {
 		processor(ctx, node)
 	}
 }
@@ -609,7 +570,7 @@ func replaceContentList(node *html.Node, i, j int, newNodes []*html.Node) {
 	}
 }
 
-func mentionProcessor(ctx *postProcessCtx, node *html.Node) {
+func mentionProcessor(ctx *RenderContext, node *html.Node) {
 	// We replace only the first mention; other mentions will be addressed later
 	found, loc := references.FindFirstMentionBytes([]byte(node.Data))
 	if !found {
@@ -617,26 +578,26 @@ func mentionProcessor(ctx *postProcessCtx, node *html.Node) {
 	}
 	mention := node.Data[loc.Start:loc.End]
 	var teams string
-	teams, ok := ctx.metas["teams"]
+	teams, ok := ctx.Metas["teams"]
 	// FIXME: util.URLJoin may not be necessary here:
 	// - setting.AppURL is defined to have a terminal '/' so unless mention[1:]
 	// is an AppSubURL link we can probably fallback to concatenation.
 	// team mention should follow @orgName/teamName style
 	if ok && strings.Contains(mention, "/") {
 		mentionOrgAndTeam := strings.Split(mention, "/")
-		if mentionOrgAndTeam[0][1:] == ctx.metas["org"] && strings.Contains(teams, ","+strings.ToLower(mentionOrgAndTeam[1])+",") {
-			replaceContent(node, loc.Start, loc.End, createLink(util.URLJoin(setting.AppURL, "org", ctx.metas["org"], "teams", mentionOrgAndTeam[1]), mention, "mention"))
+		if mentionOrgAndTeam[0][1:] == ctx.Metas["org"] && strings.Contains(teams, ","+strings.ToLower(mentionOrgAndTeam[1])+",") {
+			replaceContent(node, loc.Start, loc.End, createLink(util.URLJoin(setting.AppURL, "org", ctx.Metas["org"], "teams", mentionOrgAndTeam[1]), mention, "mention"))
 		}
 		return
 	}
 	replaceContent(node, loc.Start, loc.End, createLink(util.URLJoin(setting.AppURL, mention[1:]), mention, "mention"))
 }
 
-func shortLinkProcessor(ctx *postProcessCtx, node *html.Node) {
+func shortLinkProcessor(ctx *RenderContext, node *html.Node) {
 	shortLinkProcessorFull(ctx, node, false)
 }
 
-func shortLinkProcessorFull(ctx *postProcessCtx, node *html.Node, noLink bool) {
+func shortLinkProcessorFull(ctx *RenderContext, node *html.Node, noLink bool) {
 	m := shortLinkPattern.FindStringSubmatchIndex(node.Data)
 	if m == nil {
 		return
@@ -741,13 +702,13 @@ func shortLinkProcessorFull(ctx *postProcessCtx, node *html.Node, noLink bool) {
 			link = url.PathEscape(link)
 		}
 	}
-	urlPrefix := ctx.urlPrefix
+	urlPrefix := ctx.URLPrefix
 	if image {
 		if !absoluteLink {
 			if IsSameDomain(urlPrefix) {
 				urlPrefix = strings.Replace(urlPrefix, "/src/", "/raw/", 1)
 			}
-			if ctx.isWikiMarkdown {
+			if ctx.IsWiki {
 				link = util.URLJoin("wiki", "raw", link)
 			}
 			link = util.URLJoin(urlPrefix, link)
@@ -778,7 +739,7 @@ func shortLinkProcessorFull(ctx *postProcessCtx, node *html.Node, noLink bool) {
 		}
 	} else {
 		if !absoluteLink {
-			if ctx.isWikiMarkdown {
+			if ctx.IsWiki {
 				link = util.URLJoin("wiki", link)
 			}
 			link = util.URLJoin(urlPrefix, link)
@@ -794,8 +755,8 @@ func shortLinkProcessorFull(ctx *postProcessCtx, node *html.Node, noLink bool) {
 	replaceContent(node, m[0], m[1], linkNode)
 }
 
-func fullIssuePatternProcessor(ctx *postProcessCtx, node *html.Node) {
-	if ctx.metas == nil {
+func fullIssuePatternProcessor(ctx *RenderContext, node *html.Node) {
+	if ctx.Metas == nil {
 		return
 	}
 	m := getIssueFullPattern().FindStringSubmatchIndex(node.Data)
@@ -811,7 +772,7 @@ func fullIssuePatternProcessor(ctx *postProcessCtx, node *html.Node) {
 	matchOrg := linkParts[len(linkParts)-4]
 	matchRepo := linkParts[len(linkParts)-3]
 
-	if matchOrg == ctx.metas["user"] && matchRepo == ctx.metas["repo"] {
+	if matchOrg == ctx.Metas["user"] && matchRepo == ctx.Metas["repo"] {
 		// TODO if m[4]:m[5] is not nil, then link is to a comment,
 		// and we should indicate that in the text somehow
 		replaceContent(node, m[0], m[1], createLink(link, id, "ref-issue"))
@@ -822,8 +783,8 @@ func fullIssuePatternProcessor(ctx *postProcessCtx, node *html.Node) {
 	}
 }
 
-func issueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) {
-	if ctx.metas == nil {
+func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) {
+	if ctx.Metas == nil {
 		return
 	}
 
@@ -832,8 +793,8 @@ func issueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) {
 		ref   *references.RenderizableReference
 	)
 
-	_, exttrack := ctx.metas["format"]
-	alphanum := ctx.metas["style"] == IssueNameStyleAlphanumeric
+	_, exttrack := ctx.Metas["format"]
+	alphanum := ctx.Metas["style"] == IssueNameStyleAlphanumeric
 
 	// Repos with external issue trackers might still need to reference local PRs
 	// We need to concern with the first one that shows up in the text, whichever it is
@@ -853,8 +814,8 @@ func issueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) {
 	var link *html.Node
 	reftext := node.Data[ref.RefLocation.Start:ref.RefLocation.End]
 	if exttrack && !ref.IsPull {
-		ctx.metas["index"] = ref.Issue
-		link = createLink(com.Expand(ctx.metas["format"], ctx.metas), reftext, "ref-issue")
+		ctx.Metas["index"] = ref.Issue
+		link = createLink(com.Expand(ctx.Metas["format"], ctx.Metas), reftext, "ref-issue")
 	} else {
 		// Path determines the type of link that will be rendered. It's unknown at this point whether
 		// the linked item is actually a PR or an issue. Luckily it's of no real consequence because
@@ -864,7 +825,7 @@ func issueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) {
 			path = "pulls"
 		}
 		if ref.Owner == "" {
-			link = createLink(util.URLJoin(setting.AppURL, ctx.metas["user"], ctx.metas["repo"], path, ref.Issue), reftext, "ref-issue")
+			link = createLink(util.URLJoin(setting.AppURL, ctx.Metas["user"], ctx.Metas["repo"], path, ref.Issue), reftext, "ref-issue")
 		} else {
 			link = createLink(util.URLJoin(setting.AppURL, ref.Owner, ref.Name, path, ref.Issue), reftext, "ref-issue")
 		}
@@ -893,8 +854,8 @@ func issueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) {
 }
 
 // fullSha1PatternProcessor renders SHA containing URLs
-func fullSha1PatternProcessor(ctx *postProcessCtx, node *html.Node) {
-	if ctx.metas == nil {
+func fullSha1PatternProcessor(ctx *RenderContext, node *html.Node) {
+	if ctx.Metas == nil {
 		return
 	}
 	m := anySHA1Pattern.FindStringSubmatchIndex(node.Data)
@@ -944,8 +905,7 @@ func fullSha1PatternProcessor(ctx *postProcessCtx, node *html.Node) {
 }
 
 // emojiShortCodeProcessor for rendering text like :smile: into emoji
-func emojiShortCodeProcessor(ctx *postProcessCtx, node *html.Node) {
-
+func emojiShortCodeProcessor(ctx *RenderContext, node *html.Node) {
 	m := EmojiShortCodeRegex.FindStringSubmatchIndex(node.Data)
 	if m == nil {
 		return
@@ -968,7 +928,7 @@ func emojiShortCodeProcessor(ctx *postProcessCtx, node *html.Node) {
 }
 
 // emoji processor to match emoji and add emoji class
-func emojiProcessor(ctx *postProcessCtx, node *html.Node) {
+func emojiProcessor(ctx *RenderContext, node *html.Node) {
 	m := emoji.FindEmojiSubmatchIndex(node.Data)
 	if m == nil {
 		return
@@ -983,8 +943,8 @@ func emojiProcessor(ctx *postProcessCtx, node *html.Node) {
 
 // sha1CurrentPatternProcessor renders SHA1 strings to corresponding links that
 // are assumed to be in the same repository.
-func sha1CurrentPatternProcessor(ctx *postProcessCtx, node *html.Node) {
-	if ctx.metas == nil || ctx.metas["user"] == "" || ctx.metas["repo"] == "" || ctx.metas["repoPath"] == "" {
+func sha1CurrentPatternProcessor(ctx *RenderContext, node *html.Node) {
+	if ctx.Metas == nil || ctx.Metas["user"] == "" || ctx.Metas["repo"] == "" || ctx.Metas["repoPath"] == "" {
 		return
 	}
 	m := sha1CurrentPattern.FindStringSubmatchIndex(node.Data)
@@ -1000,7 +960,7 @@ func sha1CurrentPatternProcessor(ctx *postProcessCtx, node *html.Node) {
 	// as used by git and github for linking and thus we have to do similar.
 	// Because of this, we check to make sure that a matched hash is actually
 	// a commit in the repository before making it a link.
-	if _, err := git.NewCommand("rev-parse", "--verify", hash).RunInDirBytes(ctx.metas["repoPath"]); err != nil {
+	if _, err := git.NewCommand("rev-parse", "--verify", hash).RunInDirBytes(ctx.Metas["repoPath"]); err != nil {
 		if !strings.Contains(err.Error(), "fatal: Needed a single revision") {
 			log.Debug("sha1CurrentPatternProcessor git rev-parse: %v", err)
 		}
@@ -1008,11 +968,11 @@ func sha1CurrentPatternProcessor(ctx *postProcessCtx, node *html.Node) {
 	}
 
 	replaceContent(node, m[2], m[3],
-		createCodeLink(util.URLJoin(setting.AppURL, ctx.metas["user"], ctx.metas["repo"], "commit", hash), base.ShortSha(hash), "commit"))
+		createCodeLink(util.URLJoin(setting.AppURL, ctx.Metas["user"], ctx.Metas["repo"], "commit", hash), base.ShortSha(hash), "commit"))
 }
 
 // emailAddressProcessor replaces raw email addresses with a mailto: link.
-func emailAddressProcessor(ctx *postProcessCtx, node *html.Node) {
+func emailAddressProcessor(ctx *RenderContext, node *html.Node) {
 	m := emailRegex.FindStringSubmatchIndex(node.Data)
 	if m == nil {
 		return
@@ -1023,7 +983,7 @@ func emailAddressProcessor(ctx *postProcessCtx, node *html.Node) {
 
 // linkProcessor creates links for any HTTP or HTTPS URL not captured by
 // markdown.
-func linkProcessor(ctx *postProcessCtx, node *html.Node) {
+func linkProcessor(ctx *RenderContext, node *html.Node) {
 	m := common.LinkRegex.FindStringIndex(node.Data)
 	if m == nil {
 		return
@@ -1033,7 +993,7 @@ func linkProcessor(ctx *postProcessCtx, node *html.Node) {
 }
 
 func genDefaultLinkProcessor(defaultLink string) processor {
-	return func(ctx *postProcessCtx, node *html.Node) {
+	return func(ctx *RenderContext, node *html.Node) {
 		ch := &html.Node{
 			Parent: node,
 			Type:   html.TextNode,
@@ -1052,7 +1012,7 @@ func genDefaultLinkProcessor(defaultLink string) processor {
 }
 
 // descriptionLinkProcessor creates links for DescriptionHTML
-func descriptionLinkProcessor(ctx *postProcessCtx, node *html.Node) {
+func descriptionLinkProcessor(ctx *RenderContext, node *html.Node) {
 	m := common.LinkRegex.FindStringIndex(node.Data)
 	if m == nil {
 		return

modules/markup/html_internal_test.go

@@ -61,8 +61,8 @@ var localMetas = map[string]string{
 func TestRender_IssueIndexPattern(t *testing.T) {
 	// numeric: render inputs without valid mentions
 	test := func(s string) {
-		testRenderIssueIndexPattern(t, s, s, nil)
-		testRenderIssueIndexPattern(t, s, s, &postProcessCtx{metas: numericMetas})
+		testRenderIssueIndexPattern(t, s, s, &RenderContext{})
+		testRenderIssueIndexPattern(t, s, s, &RenderContext{Metas: numericMetas})
 	}
 
 	// should not render anything when there are no mentions
@@ -109,13 +109,13 @@ func TestRender_IssueIndexPattern2(t *testing.T) {
 			links[i] = numericIssueLink(util.URLJoin(setting.AppSubURL, path), "ref-issue", index, marker)
 		}
 		expectedNil := fmt.Sprintf(expectedFmt, links...)
-		testRenderIssueIndexPattern(t, s, expectedNil, &postProcessCtx{metas: localMetas})
+		testRenderIssueIndexPattern(t, s, expectedNil, &RenderContext{Metas: localMetas})
 
 		for i, index := range indices {
 			links[i] = numericIssueLink(prefix, "ref-issue", index, marker)
 		}
 		expectedNum := fmt.Sprintf(expectedFmt, links...)
-		testRenderIssueIndexPattern(t, s, expectedNum, &postProcessCtx{metas: numericMetas})
+		testRenderIssueIndexPattern(t, s, expectedNum, &RenderContext{Metas: numericMetas})
 	}
 
 	// should render freestanding mentions
@@ -150,7 +150,7 @@ func TestRender_IssueIndexPattern3(t *testing.T) {
 
 	// alphanumeric: render inputs without valid mentions
 	test := func(s string) {
-		testRenderIssueIndexPattern(t, s, s, &postProcessCtx{metas: alphanumericMetas})
+		testRenderIssueIndexPattern(t, s, s, &RenderContext{Metas: alphanumericMetas})
 	}
 	test("")
 	test("this is a test")
@@ -181,25 +181,22 @@ func TestRender_IssueIndexPattern4(t *testing.T) {
 			links[i] = alphanumIssueLink("https://someurl.com/someUser/someRepo/", "ref-issue", name)
 		}
 		expected := fmt.Sprintf(expectedFmt, links...)
-		testRenderIssueIndexPattern(t, s, expected, &postProcessCtx{metas: alphanumericMetas})
+		testRenderIssueIndexPattern(t, s, expected, &RenderContext{Metas: alphanumericMetas})
 	}
 	test("OTT-1234 test", "%s test", "OTT-1234")
 	test("test T-12 issue", "test %s issue", "T-12")
 	test("test issue ABCDEFGHIJ-1234567890", "test issue %s", "ABCDEFGHIJ-1234567890")
 }
 
-func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *postProcessCtx) {
-	if ctx == nil {
-		ctx = new(postProcessCtx)
-	}
-	ctx.procs = []processor{issueIndexPatternProcessor}
-	if ctx.urlPrefix == "" {
-		ctx.urlPrefix = AppSubURL
+func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *RenderContext) {
+	if ctx.URLPrefix == "" {
+		ctx.URLPrefix = AppSubURL
 	}
 
-	res, err := ctx.postProcess([]byte(input))
+	var buf strings.Builder
+	err := postProcess(ctx, []processor{issueIndexPatternProcessor}, strings.NewReader(input), &buf)
 	assert.NoError(t, err)
-	assert.Equal(t, expected, string(res))
+	assert.Equal(t, expected, buf.String())
 }
 
 func TestRender_AutoLink(t *testing.T) {
@@ -207,12 +204,22 @@ func TestRender_AutoLink(t *testing.T) {
 	setting.AppSubURL = AppSubURL
 
 	test := func(input, expected string) {
-		buffer, err := PostProcess([]byte(input), setting.AppSubURL, localMetas, false)
+		var buffer strings.Builder
+		err := PostProcess(&RenderContext{
+			URLPrefix: setting.AppSubURL,
+			Metas:     localMetas,
+		}, strings.NewReader(input), &buffer)
 		assert.Equal(t, err, nil)
-		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer)))
-		buffer, err = PostProcess([]byte(input), setting.AppSubURL, localMetas, true)
+		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer.String()))
+
+		buffer.Reset()
+		err = PostProcess(&RenderContext{
+			URLPrefix: setting.AppSubURL,
+			Metas:     localMetas,
+			IsWiki:    true,
+		}, strings.NewReader(input), &buffer)
 		assert.Equal(t, err, nil)
-		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer)))
+		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer.String()))
 	}
 
 	// render valid issue URLs
@@ -235,15 +242,13 @@ func TestRender_FullIssueURLs(t *testing.T) {
 	setting.AppSubURL = AppSubURL
 
 	test := func(input, expected string) {
-		ctx := new(postProcessCtx)
-		ctx.procs = []processor{fullIssuePatternProcessor}
-		if ctx.urlPrefix == "" {
-			ctx.urlPrefix = AppSubURL
-		}
-		ctx.metas = localMetas
-		result, err := ctx.postProcess([]byte(input))
+		var result strings.Builder
+		err := postProcess(&RenderContext{
+			URLPrefix: AppSubURL,
+			Metas:     localMetas,
+		}, []processor{fullIssuePatternProcessor}, strings.NewReader(input), &result)
 		assert.NoError(t, err)
-		assert.Equal(t, expected, string(result))
+		assert.Equal(t, expected, result.String())
 	}
 	test("Here is a link https://git.osgeo.org/gogs/postgis/postgis/pulls/6",
 		"Here is a link https://git.osgeo.org/gogs/postgis/postgis/pulls/6")

modules/markup/html_test.go

@@ -28,7 +28,12 @@ func TestRender_Commits(t *testing.T) {
 	setting.AppSubURL = AppSubURL
 
 	test := func(input, expected string) {
-		buffer := RenderString(".md", input, setting.AppSubURL, localMetas)
+		buffer, err := RenderString(&RenderContext{
+			Filename:  ".md",
+			URLPrefix: setting.AppSubURL,
+			Metas:     localMetas,
+		}, input)
+		assert.NoError(t, err)
 		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
 	}
 
@@ -59,7 +64,12 @@ func TestRender_CrossReferences(t *testing.T) {
 	setting.AppSubURL = AppSubURL
 
 	test := func(input, expected string) {
-		buffer := RenderString("a.md", input, setting.AppSubURL, localMetas)
+		buffer, err := RenderString(&RenderContext{
+			Filename:  "a.md",
+			URLPrefix: setting.AppSubURL,
+			Metas:     localMetas,
+		}, input)
+		assert.NoError(t, err)
 		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
 	}
 
@@ -91,7 +101,11 @@ func TestRender_links(t *testing.T) {
 	setting.AppSubURL = AppSubURL
 
 	test := func(input, expected string) {
-		buffer := RenderString("a.md", input, setting.AppSubURL, nil)
+		buffer, err := RenderString(&RenderContext{
+			Filename:  "a.md",
+			URLPrefix: setting.AppSubURL,
+		}, input)
+		assert.NoError(t, err)
 		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
 	}
 	// Text that should be turned into URL
@@ -187,8 +201,12 @@ func TestRender_email(t *testing.T) {
 	setting.AppSubURL = AppSubURL
 
 	test := func(input, expected string) {
-		buffer := RenderString("a.md", input, setting.AppSubURL, nil)
-		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
+		res, err := RenderString(&RenderContext{
+			Filename:  "a.md",
+			URLPrefix: setting.AppSubURL,
+		}, input)
+		assert.NoError(t, err)
+		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(res))
 	}
 	// Text that should be turned into email link
 
@@ -242,7 +260,11 @@ func TestRender_emoji(t *testing.T) {
 
 	test := func(input, expected string) {
 		expected = strings.ReplaceAll(expected, "&", "&")
-		buffer := RenderString("a.md", input, setting.AppSubURL, nil)
+		buffer, err := RenderString(&RenderContext{
+			Filename:  "a.md",
+			URLPrefix: setting.AppSubURL,
+		}, input)
+		assert.NoError(t, err)
 		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
 	}
 
@@ -291,9 +313,17 @@ func TestRender_ShortLinks(t *testing.T) {
 	tree := util.URLJoin(AppSubURL, "src", "master")
 
 	test := func(input, expected, expectedWiki string) {
-		buffer := markdown.RenderString(input, tree, nil)
+		buffer, err := markdown.RenderString(&RenderContext{
+			URLPrefix: tree,
+		}, input)
+		assert.NoError(t, err)
 		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
-		buffer = markdown.RenderWiki([]byte(input), setting.AppSubURL, localMetas)
+		buffer, err = markdown.RenderString(&RenderContext{
+			URLPrefix: setting.AppSubURL,
+			Metas:     localMetas,
+			IsWiki:    true,
+		}, input)
+		assert.NoError(t, err)
 		assert.Equal(t, strings.TrimSpace(expectedWiki), strings.TrimSpace(buffer))
 	}
 
@@ -395,16 +425,22 @@ func Test_ParseClusterFuzz(t *testing.T) {
 
 	data := "<A><maTH><tr><MN><bodY ÿ><temPlate></template><tH><tr></A><tH><d<bodY "
 
-	val, err := PostProcess([]byte(data), "https://example.com", localMetas, false)
-
+	var res strings.Builder
+	err := PostProcess(&RenderContext{
+		URLPrefix: "https://example.com",
+		Metas:     localMetas,
+	}, strings.NewReader(data), &res)
 	assert.NoError(t, err)
-	assert.NotContains(t, string(val), "<html")
+	assert.NotContains(t, res.String(), "<html")
 
 	data = "<!DOCTYPE html>\n<A><maTH><tr><MN><bodY ÿ><temPlate></template><tH><tr></A><tH><d<bodY "
 
-	val, err = PostProcess([]byte(data), "https://example.com", localMetas, false)
+	res.Reset()
+	err = PostProcess(&RenderContext{
+		URLPrefix: "https://example.com",
+		Metas:     localMetas,
+	}, strings.NewReader(data), &res)
 
 	assert.NoError(t, err)
-
-	assert.NotContains(t, string(val), "<html")
+	assert.NotContains(t, res.String(), "<html")
 }

modules/markup/markdown/markdown.go

@@ -8,6 +8,7 @@ package markdown
 import (
 	"fmt"
 	"io"
+	"io/ioutil"
 	"strings"
 	"sync"
 
@@ -73,17 +74,17 @@ func (l *limitWriter) CloseWithError(err error) error {
 	return l.w.CloseWithError(err)
 }
 
-// NewGiteaParseContext creates a parser.Context with the gitea context set
-func NewGiteaParseContext(urlPrefix string, metas map[string]string, isWiki bool) parser.Context {
+// newParserContext creates a parser.Context with the render context set
+func newParserContext(ctx *markup.RenderContext) parser.Context {
 	pc := parser.NewContext(parser.WithIDs(newPrefixedIDs()))
-	pc.Set(urlPrefixKey, urlPrefix)
-	pc.Set(isWikiKey, isWiki)
-	pc.Set(renderMetasKey, metas)
+	pc.Set(urlPrefixKey, ctx.URLPrefix)
+	pc.Set(isWikiKey, ctx.IsWiki)
+	pc.Set(renderMetasKey, ctx.Metas)
 	return pc
 }
 
 // actualRender renders Markdown to HTML without handling special links.
-func actualRender(body []byte, urlPrefix string, metas map[string]string, wikiMarkdown bool) []byte {
+func actualRender(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
 	once.Do(func() {
 		converter = goldmark.New(
 			goldmark.WithExtensions(extension.Table,
@@ -169,7 +170,7 @@ func actualRender(body []byte, urlPrefix string, metas map[string]string, wikiMa
 		limit: setting.UI.MaxDisplayFileSize * 3,
 	}
 
-	// FIXME: should we include a timeout that closes the pipe to abort the parser and sanitizer if it takes too long?
+	// FIXME: should we include a timeout that closes the pipe to abort the renderer and sanitizer if it takes too long?
 	go func() {
 		defer func() {
 			err := recover()
@@ -184,18 +185,26 @@ func actualRender(body []byte, urlPrefix string, metas map[string]string, wikiMa
 			_ = lw.CloseWithError(fmt.Errorf("%v", err))
 		}()
 
-		pc := NewGiteaParseContext(urlPrefix, metas, wikiMarkdown)
-		if err := converter.Convert(giteautil.NormalizeEOL(body), lw, parser.WithContext(pc)); err != nil {
+		// FIXME: Don't read all to memory, but goldmark doesn't support
+		pc := newParserContext(ctx)
+		buf, err := ioutil.ReadAll(input)
+		if err != nil {
+			log.Error("Unable to ReadAll: %v", err)
+			return
+		}
+		if err := converter.Convert(giteautil.NormalizeEOL(buf), lw, parser.WithContext(pc)); err != nil {
 			log.Error("Unable to render: %v", err)
 			_ = lw.CloseWithError(err)
 			return
 		}
 		_ = lw.Close()
 	}()
-	return markup.SanitizeReader(rd).Bytes()
+	buf := markup.SanitizeReader(rd)
+	_, err := io.Copy(output, buf)
+	return err
 }
 
-func render(body []byte, urlPrefix string, metas map[string]string, wikiMarkdown bool) (ret []byte) {
+func render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
 	defer func() {
 		err := recover()
 		if err == nil {
@@ -206,9 +215,13 @@ func render(body []byte, urlPrefix string, metas map[string]string, wikiMarkdown
 		if log.IsDebug() {
 			log.Debug("Panic in markdown: %v\n%s", err, string(log.Stack(2)))
 		}
-		ret = markup.SanitizeBytes(body)
+		ret := markup.SanitizeReader(input)
+		_, err = io.Copy(output, ret)
+		if err != nil {
+			log.Error("SanitizeReader failed: %v", err)
+		}
 	}()
-	return actualRender(body, urlPrefix, metas, wikiMarkdown)
+	return actualRender(ctx, input, output)
 }
 
 var (
@@ -217,48 +230,59 @@ var (
 )
 
 func init() {
-	markup.RegisterParser(Parser{})
+	markup.RegisterRenderer(Renderer{})
 }
 
-// Parser implements markup.Parser
-type Parser struct{}
+// Renderer implements markup.Renderer
+type Renderer struct{}
 
-// Name implements markup.Parser
-func (Parser) Name() string {
+// Name implements markup.Renderer
+func (Renderer) Name() string {
 	return MarkupName
 }
 
-// NeedPostProcess implements markup.Parser
-func (Parser) NeedPostProcess() bool { return true }
+// NeedPostProcess implements markup.Renderer
+func (Renderer) NeedPostProcess() bool { return true }
 
-// Extensions implements markup.Parser
-func (Parser) Extensions() []string {
+// Extensions implements markup.Renderer
+func (Renderer) Extensions() []string {
 	return setting.Markdown.FileExtensions
 }
 
-// Render implements markup.Parser
-func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
-	return render(rawBytes, urlPrefix, metas, isWiki)
+// Render implements markup.Renderer
+func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
+	return render(ctx, input, output)
 }
 
 // Render renders Markdown to HTML with all specific handling stuff.
-func Render(rawBytes []byte, urlPrefix string, metas map[string]string) []byte {
-	return markup.Render("a.md", rawBytes, urlPrefix, metas)
+func Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
+	if ctx.Filename == "" {
+		ctx.Filename = "a.md"
+	}
+	return markup.Render(ctx, input, output)
+}
+
+// RenderString renders Markdown string to HTML with all specific handling stuff and return string
+func RenderString(ctx *markup.RenderContext, content string) (string, error) {
+	var buf strings.Builder
+	if err := Render(ctx, strings.NewReader(content), &buf); err != nil {
+		return "", err
+	}
+	return buf.String(), nil
 }
 
 // RenderRaw renders Markdown to HTML without handling special links.
-func RenderRaw(body []byte, urlPrefix string, wikiMarkdown bool) []byte {
-	return render(body, urlPrefix, map[string]string{}, wikiMarkdown)
+func RenderRaw(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
+	return render(ctx, input, output)
 }
 
-// RenderString renders Markdown to HTML with special links and returns string type.
-func RenderString(raw, urlPrefix string, metas map[string]string) string {
-	return markup.RenderString("a.md", raw, urlPrefix, metas)
-}
-
-// RenderWiki renders markdown wiki page to HTML and return HTML string
-func RenderWiki(rawBytes []byte, urlPrefix string, metas map[string]string) string {
-	return markup.RenderWiki("a.md", rawBytes, urlPrefix, metas)
+// RenderRawString renders Markdown to HTML without handling special links and return string
+func RenderRawString(ctx *markup.RenderContext, content string) (string, error) {
+	var buf strings.Builder
+	if err := RenderRaw(ctx, strings.NewReader(content), &buf); err != nil {
+		return "", err
+	}
+	return buf.String(), nil
 }
 
 // IsMarkdownFile reports whether name looks like a Markdown file

modules/markup/markdown/markdown_test.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"testing"
 
+	"code.gitea.io/gitea/modules/markup"
 	. "code.gitea.io/gitea/modules/markup/markdown"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
@@ -31,10 +32,17 @@ func TestRender_StandardLinks(t *testing.T) {
 	setting.AppSubURL = AppSubURL
 
 	test := func(input, expected, expectedWiki string) {
-		buffer := RenderString(input, setting.AppSubURL, nil)
+		buffer, err := RenderString(&markup.RenderContext{
+			URLPrefix: setting.AppSubURL,
+		}, input)
+		assert.NoError(t, err)
 		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
-		bufferWiki := RenderWiki([]byte(input), setting.AppSubURL, nil)
-		assert.Equal(t, strings.TrimSpace(expectedWiki), strings.TrimSpace(bufferWiki))
+
+		buffer, err = RenderString(&markup.RenderContext{
+			URLPrefix: setting.AppSubURL,
+			IsWiki:    true,
+		}, input)
+		assert.Equal(t, strings.TrimSpace(expectedWiki), strings.TrimSpace(buffer))
 	}
 
 	googleRendered := `<p><a href="https://google.com/" rel="nofollow">https://google.com/</a></p>`
@@ -74,7 +82,10 @@ func TestRender_Images(t *testing.T) {
 	setting.AppSubURL = AppSubURL
 
 	test := func(input, expected string) {
-		buffer := RenderString(input, setting.AppSubURL, nil)
+		buffer, err := RenderString(&markup.RenderContext{
+			URLPrefix: setting.AppSubURL,
+		}, input)
+		assert.NoError(t, err)
 		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
 	}
 
@@ -261,7 +272,12 @@ func TestTotal_RenderWiki(t *testing.T) {
 	answers := testAnswers(util.URLJoin(AppSubURL, "wiki/"), util.URLJoin(AppSubURL, "wiki", "raw/"))
 
 	for i := 0; i < len(sameCases); i++ {
-		line := RenderWiki([]byte(sameCases[i]), AppSubURL, localMetas)
+		line, err := RenderString(&markup.RenderContext{
+			URLPrefix: AppSubURL,
+			Metas:     localMetas,
+			IsWiki:    true,
+		}, sameCases[i])
+		assert.NoError(t, err)
 		assert.Equal(t, answers[i], line)
 	}
 
@@ -279,7 +295,11 @@ func TestTotal_RenderWiki(t *testing.T) {
 	}
 
 	for i := 0; i < len(testCases); i += 2 {
-		line := RenderWiki([]byte(testCases[i]), AppSubURL, nil)
+		line, err := RenderString(&markup.RenderContext{
+			URLPrefix: AppSubURL,
+			IsWiki:    true,
+		}, testCases[i])
+		assert.NoError(t, err)
 		assert.Equal(t, testCases[i+1], line)
 	}
 }
@@ -288,31 +308,40 @@ func TestTotal_RenderString(t *testing.T) {
 	answers := testAnswers(util.URLJoin(AppSubURL, "src", "master/"), util.URLJoin(AppSubURL, "raw", "master/"))
 
 	for i := 0; i < len(sameCases); i++ {
-		line := RenderString(sameCases[i], util.URLJoin(AppSubURL, "src", "master/"), localMetas)
+		line, err := RenderString(&markup.RenderContext{
+			URLPrefix: util.URLJoin(AppSubURL, "src", "master/"),
+			Metas:     localMetas,
+		}, sameCases[i])
+		assert.NoError(t, err)
 		assert.Equal(t, answers[i], line)
 	}
 
 	testCases := []string{}
 
 	for i := 0; i < len(testCases); i += 2 {
-		line := RenderString(testCases[i], AppSubURL, nil)
+		line, err := RenderString(&markup.RenderContext{
+			URLPrefix: AppSubURL,
+		}, testCases[i])
+		assert.NoError(t, err)
 		assert.Equal(t, testCases[i+1], line)
 	}
 }
 
 func TestRender_RenderParagraphs(t *testing.T) {
 	test := func(t *testing.T, str string, cnt int) {
-		unix := []byte(str)
-		res := string(RenderRaw(unix, "", false))
-		assert.Equal(t, strings.Count(res, "<p"), cnt, "Rendered result for unix should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res)
+		res, err := RenderRawString(&markup.RenderContext{}, str)
+		assert.NoError(t, err)
+		assert.Equal(t, cnt, strings.Count(res, "<p"), "Rendered result for unix should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res)
 
-		mac := []byte(strings.ReplaceAll(str, "\n", "\r"))
-		res = string(RenderRaw(mac, "", false))
-		assert.Equal(t, strings.Count(res, "<p"), cnt, "Rendered result for mac should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res)
+		mac := strings.ReplaceAll(str, "\n", "\r")
+		res, err = RenderRawString(&markup.RenderContext{}, mac)
+		assert.NoError(t, err)
+		assert.Equal(t, cnt, strings.Count(res, "<p"), "Rendered result for mac should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res)
 
-		dos := []byte(strings.ReplaceAll(str, "\n", "\r\n"))
-		res = string(RenderRaw(dos, "", false))
-		assert.Equal(t, strings.Count(res, "<p"), cnt, "Rendered result for windows should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res)
+		dos := strings.ReplaceAll(str, "\n", "\r\n")
+		res, err = RenderRawString(&markup.RenderContext{}, dos)
+		assert.NoError(t, err)
+		assert.Equal(t, cnt, strings.Count(res, "<p"), "Rendered result for windows should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res)
 	}
 
 	test(t, "\nOne\nTwo\nThree", 1)
@@ -337,7 +366,8 @@ func TestMarkdownRenderRaw(t *testing.T) {
 	}
 
 	for _, testcase := range testcases {
-		_ = RenderRaw(testcase, "", false)
+		_, err := RenderRawString(&markup.RenderContext{}, string(testcase))
+		assert.NoError(t, err)
 	}
 }
 
@@ -348,7 +378,8 @@ func TestRenderSiblingImages_Issue12925(t *testing.T) {
 	expected := `<p><a href="/image1" rel="nofollow"><img src="/image1" alt="image1"></a><br>
 <a href="/image2" rel="nofollow"><img src="/image2" alt="image2"></a></p>
 `
-	res := string(RenderRaw([]byte(testcase), "", false))
+	res, err := RenderRawString(&markup.RenderContext{}, testcase)
+	assert.NoError(t, err)
 	assert.Equal(t, expected, res)
 
 }

modules/markup/markup.go

@@ -1,143 +0,0 @@
-// Copyright 2017 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package markup
-
-import (
-	"path/filepath"
-	"strings"
-
-	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/setting"
-)
-
-// Init initialize regexps for markdown parsing
-func Init() {
-	getIssueFullPattern()
-	NewSanitizer()
-	if len(setting.Markdown.CustomURLSchemes) > 0 {
-		CustomLinkURLSchemes(setting.Markdown.CustomURLSchemes)
-	}
-
-	// since setting maybe changed extensions, this will reload all parser extensions mapping
-	extParsers = make(map[string]Parser)
-	for _, parser := range parsers {
-		for _, ext := range parser.Extensions() {
-			extParsers[strings.ToLower(ext)] = parser
-		}
-	}
-}
-
-// Parser defines an interface for parsering markup file to HTML
-type Parser interface {
-	Name() string // markup format name
-	Extensions() []string
-	NeedPostProcess() bool
-	Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte
-}
-
-var (
-	extParsers = make(map[string]Parser)
-	parsers    = make(map[string]Parser)
-)
-
-// RegisterParser registers a new markup file parser
-func RegisterParser(parser Parser) {
-	parsers[parser.Name()] = parser
-	for _, ext := range parser.Extensions() {
-		extParsers[strings.ToLower(ext)] = parser
-	}
-}
-
-// GetParserByFileName get parser by filename
-func GetParserByFileName(filename string) Parser {
-	extension := strings.ToLower(filepath.Ext(filename))
-	return extParsers[extension]
-}
-
-// GetParserByType returns a parser according type
-func GetParserByType(tp string) Parser {
-	return parsers[tp]
-}
-
-// Render renders markup file to HTML with all specific handling stuff.
-func Render(filename string, rawBytes []byte, urlPrefix string, metas map[string]string) []byte {
-	return renderFile(filename, rawBytes, urlPrefix, metas, false)
-}
-
-// RenderByType renders markup to HTML with special links and returns string type.
-func RenderByType(tp string, rawBytes []byte, urlPrefix string, metas map[string]string) []byte {
-	return renderByType(tp, rawBytes, urlPrefix, metas, false)
-}
-
-// RenderString renders Markdown to HTML with special links and returns string type.
-func RenderString(filename string, raw, urlPrefix string, metas map[string]string) string {
-	return string(renderFile(filename, []byte(raw), urlPrefix, metas, false))
-}
-
-// RenderWiki renders markdown wiki page to HTML and return HTML string
-func RenderWiki(filename string, rawBytes []byte, urlPrefix string, metas map[string]string) string {
-	return string(renderFile(filename, rawBytes, urlPrefix, metas, true))
-}
-
-func render(parser Parser, rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
-	result := parser.Render(rawBytes, urlPrefix, metas, isWiki)
-	if parser.NeedPostProcess() {
-		var err error
-		// TODO: one day the error should be returned.
-		result, err = PostProcess(result, urlPrefix, metas, isWiki)
-		if err != nil {
-			log.Error("PostProcess: %v", err)
-		}
-	}
-	return SanitizeBytes(result)
-}
-
-func renderByType(tp string, rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
-	if parser, ok := parsers[tp]; ok {
-		return render(parser, rawBytes, urlPrefix, metas, isWiki)
-	}
-	return nil
-}
-
-func renderFile(filename string, rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
-	extension := strings.ToLower(filepath.Ext(filename))
-	if parser, ok := extParsers[extension]; ok {
-		return render(parser, rawBytes, urlPrefix, metas, isWiki)
-	}
-	return nil
-}
-
-// Type returns if markup format via the filename
-func Type(filename string) string {
-	if parser := GetParserByFileName(filename); parser != nil {
-		return parser.Name()
-	}
-	return ""
-}
-
-// IsMarkupFile reports whether file is a markup type file
-func IsMarkupFile(name, markup string) bool {
-	if parser := GetParserByFileName(name); parser != nil {
-		return parser.Name() == markup
-	}
-	return false
-}
-
-// IsReadmeFile reports whether name looks like a README file
-// based on its name. If an extension is provided, it will strictly
-// match that extension.
-// Note that the '.' should be provided in ext, e.g ".md"
-func IsReadmeFile(name string, ext ...string) bool {
-	name = strings.ToLower(name)
-	if len(ext) > 0 {
-		return name == "readme"+ext[0]
-	}
-	if len(name) < 6 {
-		return false
-	} else if len(name) == 6 {
-		return name == "readme"
-	}
-	return name[:7] == "readme."
-}

modules/markup/orgmode/orgmode.go

@@ -8,9 +8,9 @@ import (
 	"bytes"
 	"fmt"
 	"html"
+	"io"
 	"strings"
 
-	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/util"
 
@@ -18,58 +18,62 @@ import (
 )
 
 func init() {
-	markup.RegisterParser(Parser{})
+	markup.RegisterRenderer(Renderer{})
 }
 
-// Parser implements markup.Parser for orgmode
-type Parser struct {
+// Renderer implements markup.Renderer for orgmode
+type Renderer struct {
 }
 
-// Name implements markup.Parser
-func (Parser) Name() string {
+// Name implements markup.Renderer
+func (Renderer) Name() string {
 	return "orgmode"
 }
 
-// NeedPostProcess implements markup.Parser
-func (Parser) NeedPostProcess() bool { return true }
+// NeedPostProcess implements markup.Renderer
+func (Renderer) NeedPostProcess() bool { return true }
 
-// Extensions implements markup.Parser
-func (Parser) Extensions() []string {
+// Extensions implements markup.Renderer
+func (Renderer) Extensions() []string {
 	return []string{".org"}
 }
 
 // Render renders orgmode rawbytes to HTML
-func Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
+func Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
 	htmlWriter := org.NewHTMLWriter()
 
-	renderer := &Renderer{
+	w := &Writer{
 		HTMLWriter: htmlWriter,
-		URLPrefix:  urlPrefix,
-		IsWiki:     isWiki,
+		URLPrefix:  ctx.URLPrefix,
+		IsWiki:     ctx.IsWiki,
 	}
 
-	htmlWriter.ExtendingWriter = renderer
+	htmlWriter.ExtendingWriter = w
 
-	res, err := org.New().Silent().Parse(bytes.NewReader(rawBytes), "").Write(renderer)
+	res, err := org.New().Silent().Parse(input, "").Write(w)
 	if err != nil {
-		log.Error("Panic in orgmode.Render: %v Just returning the rawBytes", err)
-		return rawBytes
+		return fmt.Errorf("orgmode.Render failed: %v", err)
 	}
-	return []byte(res)
+	_, err = io.Copy(output, strings.NewReader(res))
+	return err
 }
 
-// RenderString reners orgmode string to HTML string
-func RenderString(rawContent string, urlPrefix string, metas map[string]string, isWiki bool) string {
-	return string(Render([]byte(rawContent), urlPrefix, metas, isWiki))
+// RenderString renders orgmode string to HTML string
+func RenderString(ctx *markup.RenderContext, content string) (string, error) {
+	var buf strings.Builder
+	if err := Render(ctx, strings.NewReader(content), &buf); err != nil {
+		return "", err
+	}
+	return buf.String(), nil
 }
 
-// Render reners orgmode string to HTML string
-func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
-	return Render(rawBytes, urlPrefix, metas, isWiki)
+// Render renders orgmode string to HTML string
+func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
+	return Render(ctx, input, output)
 }
 
-// Renderer implements org.Writer
-type Renderer struct {
+// Writer implements org.Writer
+type Writer struct {
 	*org.HTMLWriter
 	URLPrefix string
 	IsWiki    bool
@@ -78,7 +82,7 @@ type Renderer struct {
 var byteMailto = []byte("mailto:")
 
 // WriteRegularLink renders images, links or videos
-func (r *Renderer) WriteRegularLink(l org.RegularLink) {
+func (r *Writer) WriteRegularLink(l org.RegularLink) {
 	link := []byte(html.EscapeString(l.URL))
 	if l.Protocol == "file" {
 		link = link[len("file:"):]

modules/markup/orgmode/orgmode_test.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"testing"
 
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
 
@@ -23,7 +24,10 @@ func TestRender_StandardLinks(t *testing.T) {
 	setting.AppSubURL = AppSubURL
 
 	test := func(input, expected string) {
-		buffer := RenderString(input, setting.AppSubURL, nil, false)
+		buffer, err := RenderString(&markup.RenderContext{
+			URLPrefix: setting.AppSubURL,
+		}, input)
+		assert.NoError(t, err)
 		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
 	}
 
@@ -40,7 +44,10 @@ func TestRender_Images(t *testing.T) {
 	setting.AppSubURL = AppSubURL
 
 	test := func(input, expected string) {
-		buffer := RenderString(input, setting.AppSubURL, nil, false)
+		buffer, err := RenderString(&markup.RenderContext{
+			URLPrefix: setting.AppSubURL,
+		}, input)
+		assert.NoError(t, err)
 		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
 	}
 

modules/markup/renderer.go

@@ -0,0 +1,201 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package markup
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"code.gitea.io/gitea/modules/setting"
+)
+
+// Init initialize regexps for markdown parsing
+func Init() {
+	getIssueFullPattern()
+	NewSanitizer()
+	if len(setting.Markdown.CustomURLSchemes) > 0 {
+		CustomLinkURLSchemes(setting.Markdown.CustomURLSchemes)
+	}
+
+	// since setting maybe changed extensions, this will reload all renderer extensions mapping
+	extRenderers = make(map[string]Renderer)
+	for _, renderer := range renderers {
+		for _, ext := range renderer.Extensions() {
+			extRenderers[strings.ToLower(ext)] = renderer
+		}
+	}
+}
+
+// RenderContext represents a render context
+type RenderContext struct {
+	Ctx         context.Context
+	Filename    string
+	Type        string
+	IsWiki      bool
+	URLPrefix   string
+	Metas       map[string]string
+	DefaultLink string
+}
+
+// Renderer defines an interface for rendering markup file to HTML
+type Renderer interface {
+	Name() string // markup format name
+	Extensions() []string
+	Render(ctx *RenderContext, input io.Reader, output io.Writer) error
+}
+
+var (
+	extRenderers = make(map[string]Renderer)
+	renderers    = make(map[string]Renderer)
+)
+
+// RegisterRenderer registers a new markup file renderer
+func RegisterRenderer(renderer Renderer) {
+	renderers[renderer.Name()] = renderer
+	for _, ext := range renderer.Extensions() {
+		extRenderers[strings.ToLower(ext)] = renderer
+	}
+}
+
+// GetRendererByFileName get renderer by filename
+func GetRendererByFileName(filename string) Renderer {
+	extension := strings.ToLower(filepath.Ext(filename))
+	return extRenderers[extension]
+}
+
+// GetRendererByType returns a renderer according type
+func GetRendererByType(tp string) Renderer {
+	return renderers[tp]
+}
+
+// Render renders markup file to HTML with all specific handling stuff.
+func Render(ctx *RenderContext, input io.Reader, output io.Writer) error {
+	if ctx.Type != "" {
+		return renderByType(ctx, input, output)
+	} else if ctx.Filename != "" {
+		return renderFile(ctx, input, output)
+	}
+	return errors.New("Render options both filename and type missing")
+}
+
+// RenderString renders Markup string to HTML with all specific handling stuff and return string
+func RenderString(ctx *RenderContext, content string) (string, error) {
+	var buf strings.Builder
+	if err := Render(ctx, strings.NewReader(content), &buf); err != nil {
+		return "", err
+	}
+	return buf.String(), nil
+}
+
+func render(ctx *RenderContext, parser Renderer, input io.Reader, output io.Writer) error {
+	var wg sync.WaitGroup
+	var err error
+	pr, pw := io.Pipe()
+	defer func() {
+		_ = pr.Close()
+		_ = pw.Close()
+	}()
+
+	pr2, pw2 := io.Pipe()
+	defer func() {
+		_ = pr2.Close()
+		_ = pw2.Close()
+	}()
+
+	wg.Add(1)
+	go func() {
+		buf := SanitizeReader(pr2)
+		_, err = io.Copy(output, buf)
+		_ = pr2.Close()
+		wg.Done()
+	}()
+
+	wg.Add(1)
+	go func() {
+		err = PostProcess(ctx, pr, pw2)
+		_ = pr.Close()
+		_ = pw2.Close()
+		wg.Done()
+	}()
+
+	if err1 := parser.Render(ctx, input, pw); err1 != nil {
+		return err1
+	}
+	_ = pw.Close()
+
+	wg.Wait()
+	return err
+}
+
+// ErrUnsupportedRenderType represents
+type ErrUnsupportedRenderType struct {
+	Type string
+}
+
+func (err ErrUnsupportedRenderType) Error() string {
+	return fmt.Sprintf("Unsupported render type: %s", err.Type)
+}
+
+func renderByType(ctx *RenderContext, input io.Reader, output io.Writer) error {
+	if renderer, ok := renderers[ctx.Type]; ok {
+		return render(ctx, renderer, input, output)
+	}
+	return ErrUnsupportedRenderType{ctx.Type}
+}
+
+// ErrUnsupportedRenderExtension represents the error when extension doesn't supported to render
+type ErrUnsupportedRenderExtension struct {
+	Extension string
+}
+
+func (err ErrUnsupportedRenderExtension) Error() string {
+	return fmt.Sprintf("Unsupported render extension: %s", err.Extension)
+}
+
+func renderFile(ctx *RenderContext, input io.Reader, output io.Writer) error {
+	extension := strings.ToLower(filepath.Ext(ctx.Filename))
+	if renderer, ok := extRenderers[extension]; ok {
+		return render(ctx, renderer, input, output)
+	}
+	return ErrUnsupportedRenderExtension{extension}
+}
+
+// Type returns if markup format via the filename
+func Type(filename string) string {
+	if parser := GetRendererByFileName(filename); parser != nil {
+		return parser.Name()
+	}
+	return ""
+}
+
+// IsMarkupFile reports whether file is a markup type file
+func IsMarkupFile(name, markup string) bool {
+	if parser := GetRendererByFileName(name); parser != nil {
+		return parser.Name() == markup
+	}
+	return false
+}
+
+// IsReadmeFile reports whether name looks like a README file
+// based on its name. If an extension is provided, it will strictly
+// match that extension.
+// Note that the '.' should be provided in ext, e.g ".md"
+func IsReadmeFile(name string, ext ...string) bool {
+	name = strings.ToLower(name)
+	if len(ext) > 0 {
+		return name == "readme"+ext[0]
+	}
+	if len(name) < 6 {
+		return false
+	} else if len(name) == 6 {
+		return name == "readme"
+	}
+	return name[:7] == "readme."
+}

modules/notification/mail/mail.go

@@ -104,14 +104,18 @@ func (m *mailNotifier) NotifyIssueChangeAssignee(doer *models.User, issue *model
 	// mail only sent to added assignees and not self-assignee
 	if !removed && doer.ID != assignee.ID && assignee.EmailNotifications() == models.EmailNotificationsEnabled {
 		ct := fmt.Sprintf("Assigned #%d.", issue.Index)
-		mailer.SendIssueAssignedMail(issue, doer, ct, comment, []*models.User{assignee})
+		if err := mailer.SendIssueAssignedMail(issue, doer, ct, comment, []*models.User{assignee}); err != nil {
+			log.Error("Error in SendIssueAssignedMail for issue[%d] to assignee[%d]: %v", issue.ID, assignee.ID, err)
+		}
 	}
 }
 
 func (m *mailNotifier) NotifyPullReviewRequest(doer *models.User, issue *models.Issue, reviewer *models.User, isRequest bool, comment *models.Comment) {
 	if isRequest && doer.ID != reviewer.ID && reviewer.EmailNotifications() == models.EmailNotificationsEnabled {
 		ct := fmt.Sprintf("Requested to review %s.", issue.HTMLURL())
-		mailer.SendIssueAssignedMail(issue, doer, ct, comment, []*models.User{reviewer})
+		if err := mailer.SendIssueAssignedMail(issue, doer, ct, comment, []*models.User{reviewer}); err != nil {
+			log.Error("Error in SendIssueAssignedMail for issue[%d] to reviewer[%d]: %v", issue.ID, reviewer.ID, err)
+		}
 	}
 }
 

modules/setting/markup.go

@@ -13,14 +13,14 @@ import (
 	"gopkg.in/ini.v1"
 )
 
-// ExternalMarkupParsers represents the external markup parsers
+// ExternalMarkupRenderers represents the external markup renderers
 var (
-	ExternalMarkupParsers  []MarkupParser
-	ExternalSanitizerRules []MarkupSanitizerRule
+	ExternalMarkupRenderers []MarkupRenderer
+	ExternalSanitizerRules  []MarkupSanitizerRule
 )
 
-// MarkupParser defines the external parser configured in ini
-type MarkupParser struct {
+// MarkupRenderer defines the external parser configured in ini
+type MarkupRenderer struct {
 	Enabled         bool
 	MarkupName      string
 	Command         string
@@ -124,7 +124,7 @@ func newMarkupRenderer(name string, sec *ini.Section) {
 		return
 	}
 
-	ExternalMarkupParsers = append(ExternalMarkupParsers, MarkupParser{
+	ExternalMarkupRenderers = append(ExternalMarkupRenderers, MarkupRenderer{
 		Enabled:         sec.Key("ENABLED").MustBool(false),
 		MarkupName:      name,
 		FileExtensions:  exts,

modules/templates/helper.go

@@ -665,7 +665,11 @@ func RenderCommitMessageLink(msg, urlPrefix, urlDefault string, metas map[string
 	cleanMsg := template.HTMLEscapeString(msg)
 	// we can safely assume that it will not return any error, since there
 	// shouldn't be any special HTML.
-	fullMessage, err := markup.RenderCommitMessage([]byte(cleanMsg), urlPrefix, urlDefault, metas)
+	fullMessage, err := markup.RenderCommitMessage(&markup.RenderContext{
+		URLPrefix:   urlPrefix,
+		DefaultLink: urlDefault,
+		Metas:       metas,
+	}, cleanMsg)
 	if err != nil {
 		log.Error("RenderCommitMessage: %v", err)
 		return ""
@@ -692,7 +696,11 @@ func RenderCommitMessageLinkSubject(msg, urlPrefix, urlDefault string, metas map
 
 	// we can safely assume that it will not return any error, since there
 	// shouldn't be any special HTML.
-	renderedMessage, err := markup.RenderCommitMessageSubject([]byte(template.HTMLEscapeString(msgLine)), urlPrefix, urlDefault, metas)
+	renderedMessage, err := markup.RenderCommitMessageSubject(&markup.RenderContext{
+		URLPrefix:   urlPrefix,
+		DefaultLink: urlDefault,
+		Metas:       metas,
+	}, template.HTMLEscapeString(msgLine))
 	if err != nil {
 		log.Error("RenderCommitMessageSubject: %v", err)
 		return template.HTML("")
@@ -714,7 +722,10 @@ func RenderCommitBody(msg, urlPrefix string, metas map[string]string) template.H
 		return template.HTML("")
 	}
 
-	renderedMessage, err := markup.RenderCommitMessage([]byte(template.HTMLEscapeString(msgLine)), urlPrefix, "", metas)
+	renderedMessage, err := markup.RenderCommitMessage(&markup.RenderContext{
+		URLPrefix: urlPrefix,
+		Metas:     metas,
+	}, template.HTMLEscapeString(msgLine))
 	if err != nil {
 		log.Error("RenderCommitMessage: %v", err)
 		return ""
@@ -724,7 +735,10 @@ func RenderCommitBody(msg, urlPrefix string, metas map[string]string) template.H
 
 // RenderIssueTitle renders issue/pull title with defined post processors
 func RenderIssueTitle(text, urlPrefix string, metas map[string]string) template.HTML {
-	renderedText, err := markup.RenderIssueTitle([]byte(template.HTMLEscapeString(text)), urlPrefix, metas)
+	renderedText, err := markup.RenderIssueTitle(&markup.RenderContext{
+		URLPrefix: urlPrefix,
+		Metas:     metas,
+	}, template.HTMLEscapeString(text))
 	if err != nil {
 		log.Error("RenderIssueTitle: %v", err)
 		return template.HTML("")
@@ -734,7 +748,7 @@ func RenderIssueTitle(text, urlPrefix string, metas map[string]string) template.
 
 // RenderEmoji renders html text with emoji post processors
 func RenderEmoji(text string) template.HTML {
-	renderedText, err := markup.RenderEmoji([]byte(template.HTMLEscapeString(text)))
+	renderedText, err := markup.RenderEmoji(template.HTMLEscapeString(text))
 	if err != nil {
 		log.Error("RenderEmoji: %v", err)
 		return template.HTML("")
@@ -758,7 +772,10 @@ func ReactionToEmoji(reaction string) template.HTML {
 // RenderNote renders the contents of a git-notes file as a commit message.
 func RenderNote(msg, urlPrefix string, metas map[string]string) template.HTML {
 	cleanMsg := template.HTMLEscapeString(msg)
-	fullMessage, err := markup.RenderCommitMessage([]byte(cleanMsg), urlPrefix, "", metas)
+	fullMessage, err := markup.RenderCommitMessage(&markup.RenderContext{
+		URLPrefix: urlPrefix,
+		Metas:     metas,
+	}, cleanMsg)
 	if err != nil {
 		log.Error("RenderNote: %v", err)
 		return ""

routers/api/v1/misc/markdown.go

@@ -5,11 +5,11 @@
 package misc
 
 import (
-	"io/ioutil"
 	"net/http"
 	"strings"
 
 	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/markdown"
 	"code.gitea.io/gitea/modules/setting"
 	api "code.gitea.io/gitea/modules/structs"
@@ -55,7 +55,6 @@ func Markdown(ctx *context.APIContext) {
 	case "comment":
 		fallthrough
 	case "gfm":
-		md := []byte(form.Text)
 		urlPrefix := form.Context
 		meta := map[string]string{}
 		if !strings.HasPrefix(setting.AppSubURL+"/", urlPrefix) {
@@ -77,22 +76,19 @@ func Markdown(ctx *context.APIContext) {
 		if form.Mode == "gfm" {
 			meta["mode"] = "document"
 		}
-		if form.Wiki {
-			_, err := ctx.Write([]byte(markdown.RenderWiki(md, urlPrefix, meta)))
-			if err != nil {
-				ctx.InternalServerError(err)
-				return
-			}
-		} else {
-			_, err := ctx.Write(markdown.Render(md, urlPrefix, meta))
-			if err != nil {
-				ctx.InternalServerError(err)
-				return
-			}
+
+		if err := markdown.Render(&markup.RenderContext{
+			URLPrefix: urlPrefix,
+			Metas:     meta,
+			IsWiki:    form.Wiki,
+		}, strings.NewReader(form.Text), ctx.Resp); err != nil {
+			ctx.InternalServerError(err)
+			return
 		}
 	default:
-		_, err := ctx.Write(markdown.RenderRaw([]byte(form.Text), "", false))
-		if err != nil {
+		if err := markdown.RenderRaw(&markup.RenderContext{
+			URLPrefix: form.Context,
+		}, strings.NewReader(form.Text), ctx.Resp); err != nil {
 			ctx.InternalServerError(err)
 			return
 		}
@@ -120,14 +116,8 @@ func MarkdownRaw(ctx *context.APIContext) {
 	//     "$ref": "#/responses/MarkdownRender"
 	//   "422":
 	//     "$ref": "#/responses/validationError"
-
-	body, err := ioutil.ReadAll(ctx.Req.Body)
-	if err != nil {
-		ctx.Error(http.StatusUnprocessableEntity, "", err)
-		return
-	}
-	_, err = ctx.Write(markdown.RenderRaw(body, "", false))
-	if err != nil {
+	defer ctx.Req.Body.Close()
+	if err := markdown.RenderRaw(&markup.RenderContext{}, ctx.Req.Body, ctx.Resp); err != nil {
 		ctx.InternalServerError(err)
 		return
 	}

routers/init.go

@@ -143,7 +143,7 @@ func GlobalInit(ctx context.Context) {
 	NewServices()
 
 	highlight.NewContext()
-	external.RegisterParsers()
+	external.RegisterRenderers()
 	markup.Init()
 
 	if setting.EnableSQLite3 {

routers/org/home.go

@@ -11,6 +11,7 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/markdown"
 	"code.gitea.io/gitea/modules/setting"
 )
@@ -37,7 +38,15 @@ func Home(ctx *context.Context) {
 	ctx.Data["PageIsUserProfile"] = true
 	ctx.Data["Title"] = org.DisplayName()
 	if len(org.Description) != 0 {
-		ctx.Data["RenderedDescription"] = string(markdown.Render([]byte(org.Description), ctx.Repo.RepoLink, map[string]string{"mode": "document"}))
+		desc, err := markdown.RenderString(&markup.RenderContext{
+			URLPrefix: ctx.Repo.RepoLink,
+			Metas:     map[string]string{"mode": "document"},
+		}, org.Description)
+		if err != nil {
+			ctx.ServerError("RenderString", err)
+			return
+		}
+		ctx.Data["RenderedDescription"] = desc
 	}
 
 	var orderBy models.SearchOrderBy

routers/repo/compare.go

@@ -10,7 +10,6 @@ import (
 	"errors"
 	"fmt"
 	"html"
-	"io/ioutil"
 	"net/http"
 	"path"
 	"path/filepath"
@@ -117,14 +116,7 @@ func setCsvCompareContext(ctx *context.Context) {
 			}
 			defer reader.Close()
 
-			b, err := ioutil.ReadAll(reader)
-			if err != nil {
-				return nil, err
-			}
-
-			b = charset.ToUTF8WithFallback(b)
-
-			return csv_module.CreateReaderAndGuessDelimiter(b), nil
+			return csv_module.CreateReaderAndGuessDelimiter(charset.ToUTF8WithFallbackReader(reader))
 		}
 
 		baseReader, err := csvReaderFromCommit(baseCommit)

routers/repo/issue.go

@@ -1131,8 +1131,14 @@ func ViewIssue(ctx *context.Context) {
 	}
 	ctx.Data["IssueWatch"] = iw
 
-	issue.RenderedContent = string(markdown.Render([]byte(issue.Content), ctx.Repo.RepoLink,
-		ctx.Repo.Repository.ComposeMetas()))
+	issue.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
+		URLPrefix: ctx.Repo.RepoLink,
+		Metas:     ctx.Repo.Repository.ComposeMetas(),
+	}, issue.Content)
+	if err != nil {
+		ctx.ServerError("RenderString", err)
+		return
+	}
 
 	repo := ctx.Repo.Repository
 
@@ -1289,9 +1295,14 @@ func ViewIssue(ctx *context.Context) {
 				return
 			}
 
-			comment.RenderedContent = string(markdown.Render([]byte(comment.Content), ctx.Repo.RepoLink,
-				ctx.Repo.Repository.ComposeMetas()))
-
+			comment.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
+				URLPrefix: ctx.Repo.RepoLink,
+				Metas:     ctx.Repo.Repository.ComposeMetas(),
+			}, comment.Content)
+			if err != nil {
+				ctx.ServerError("RenderString", err)
+				return
+			}
 			// Check tag.
 			tag, ok = marked[comment.PosterID]
 			if ok {
@@ -1359,8 +1370,14 @@ func ViewIssue(ctx *context.Context) {
 				}
 			}
 		} else if comment.Type == models.CommentTypeCode || comment.Type == models.CommentTypeReview || comment.Type == models.CommentTypeDismissReview {
-			comment.RenderedContent = string(markdown.Render([]byte(comment.Content), ctx.Repo.RepoLink,
-				ctx.Repo.Repository.ComposeMetas()))
+			comment.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
+				URLPrefix: ctx.Repo.RepoLink,
+				Metas:     ctx.Repo.Repository.ComposeMetas(),
+			}, comment.Content)
+			if err != nil {
+				ctx.ServerError("RenderString", err)
+				return
+			}
 			if err = comment.LoadReview(); err != nil && !models.IsErrReviewNotExist(err) {
 				ctx.ServerError("LoadReview", err)
 				return
@@ -1708,10 +1725,20 @@ func UpdateIssueContent(ctx *context.Context) {
 	files := ctx.QueryStrings("files[]")
 	if err := updateAttachments(issue, files); err != nil {
 		ctx.ServerError("UpdateAttachments", err)
+		return
+	}
+
+	content, err := markdown.RenderString(&markup.RenderContext{
+		URLPrefix: ctx.Query("context"),
+		Metas:     ctx.Repo.Repository.ComposeMetas(),
+	}, issue.Content)
+	if err != nil {
+		ctx.ServerError("RenderString", err)
+		return
 	}
 
 	ctx.JSON(http.StatusOK, map[string]interface{}{
-		"content":     string(markdown.Render([]byte(issue.Content), ctx.Query("context"), ctx.Repo.Repository.ComposeMetas())),
+		"content":     content,
 		"attachments": attachmentsHTML(ctx, issue.Attachments, issue.Content),
 	})
 }
@@ -2125,10 +2152,20 @@ func UpdateCommentContent(ctx *context.Context) {
 	files := ctx.QueryStrings("files[]")
 	if err := updateAttachments(comment, files); err != nil {
 		ctx.ServerError("UpdateAttachments", err)
+		return
+	}
+
+	content, err := markdown.RenderString(&markup.RenderContext{
+		URLPrefix: ctx.Query("context"),
+		Metas:     ctx.Repo.Repository.ComposeMetas(),
+	}, comment.Content)
+	if err != nil {
+		ctx.ServerError("RenderString", err)
+		return
 	}
 
 	ctx.JSON(http.StatusOK, map[string]interface{}{
-		"content":     string(markdown.Render([]byte(comment.Content), ctx.Query("context"), ctx.Repo.Repository.ComposeMetas())),
+		"content":     content,
 		"attachments": attachmentsHTML(ctx, comment.Attachments, comment.Content),
 	})
 }

routers/repo/lfs.go

@@ -296,20 +296,13 @@ func LFSFileGet(ctx *context.Context) {
 			break
 		}
 
-		d, _ := ioutil.ReadAll(dataRc)
-		buf = charset.ToUTF8WithFallback(append(buf, d...))
+		buf := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc))
 
 		// Building code view blocks with line number on server side.
-		var fileContent string
-		if content, err := charset.ToUTF8WithErr(buf); err != nil {
-			log.Error("ToUTF8WithErr: %v", err)
-			fileContent = string(buf)
-		} else {
-			fileContent = content
-		}
+		fileContent, _ := ioutil.ReadAll(buf)
 
 		var output bytes.Buffer
-		lines := strings.Split(fileContent, "\n")
+		lines := strings.Split(string(fileContent), "\n")
 		//Remove blank line at the end of file
 		if len(lines) > 0 && lines[len(lines)-1] == "" {
 			lines = lines[:len(lines)-1]

routers/repo/milestone.go

@@ -12,6 +12,7 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/markdown"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/structs"
@@ -84,7 +85,14 @@ func Milestones(ctx *context.Context) {
 		}
 	}
 	for _, m := range miles {
-		m.RenderedContent = string(markdown.Render([]byte(m.Content), ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas()))
+		m.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
+			URLPrefix: ctx.Repo.RepoLink,
+			Metas:     ctx.Repo.Repository.ComposeMetas(),
+		}, m.Content)
+		if err != nil {
+			ctx.ServerError("RenderString", err)
+			return
+		}
 	}
 	ctx.Data["Milestones"] = miles
 
@@ -269,7 +277,14 @@ func MilestoneIssuesAndPulls(ctx *context.Context) {
 		return
 	}
 
-	milestone.RenderedContent = string(markdown.Render([]byte(milestone.Content), ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas()))
+	milestone.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
+		URLPrefix: ctx.Repo.RepoLink,
+		Metas:     ctx.Repo.Repository.ComposeMetas(),
+	}, milestone.Content)
+	if err != nil {
+		ctx.ServerError("RenderString", err)
+		return
+	}
 
 	ctx.Data["Title"] = milestone.Name
 	ctx.Data["Milestone"] = milestone

routers/repo/projects.go

@@ -12,6 +12,7 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/markdown"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
@@ -77,7 +78,14 @@ func Projects(ctx *context.Context) {
 	}
 
 	for i := range projects {
-		projects[i].RenderedContent = string(markdown.Render([]byte(projects[i].Description), ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas()))
+		projects[i].RenderedContent, err = markdown.RenderString(&markup.RenderContext{
+			URLPrefix: ctx.Repo.RepoLink,
+			Metas:     ctx.Repo.Repository.ComposeMetas(),
+		}, projects[i].Description)
+		if err != nil {
+			ctx.ServerError("RenderString", err)
+			return
+		}
 	}
 
 	ctx.Data["Projects"] = projects
@@ -311,7 +319,14 @@ func ViewProject(ctx *context.Context) {
 	}
 	ctx.Data["LinkedPRs"] = linkedPrsMap
 
-	project.RenderedContent = string(markdown.Render([]byte(project.Description), ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas()))
+	project.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
+		URLPrefix: ctx.Repo.RepoLink,
+		Metas:     ctx.Repo.Repository.ComposeMetas(),
+	}, project.Description)
+	if err != nil {
+		ctx.ServerError("RenderString", err)
+		return
+	}
 
 	ctx.Data["CanWriteProjects"] = ctx.Repo.Permission.CanWrite(models.UnitTypeProjects)
 	ctx.Data["Project"] = project

routers/repo/release.go

@@ -15,6 +15,7 @@ import (
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/convert"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/markdown"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/upload"
@@ -132,7 +133,14 @@ func releasesOrTags(ctx *context.Context, isTagList bool) {
 			ctx.ServerError("calReleaseNumCommitsBehind", err)
 			return
 		}
-		r.Note = markdown.RenderString(r.Note, ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas())
+		r.Note, err = markdown.RenderString(&markup.RenderContext{
+			URLPrefix: ctx.Repo.RepoLink,
+			Metas:     ctx.Repo.Repository.ComposeMetas(),
+		}, r.Note)
+		if err != nil {
+			ctx.ServerError("RenderString", err)
+			return
+		}
 	}
 
 	ctx.Data["Releases"] = releases
@@ -182,7 +190,14 @@ func SingleRelease(ctx *context.Context) {
 		ctx.ServerError("calReleaseNumCommitsBehind", err)
 		return
 	}
-	release.Note = markdown.RenderString(release.Note, ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas())
+	release.Note, err = markdown.RenderString(&markup.RenderContext{
+		URLPrefix: ctx.Repo.RepoLink,
+		Metas:     ctx.Repo.Repository.ComposeMetas(),
+	}, release.Note)
+	if err != nil {
+		ctx.ServerError("RenderString", err)
+		return
+	}
 
 	ctx.Data["Releases"] = []*models.Release{release}
 	ctx.HTML(http.StatusOK, tplReleases)

routers/repo/view.go

@@ -324,13 +324,26 @@ func renderDirectory(ctx *context.Context, treeLink string) {
 				ctx.Data["IsTextFile"] = true
 				ctx.Data["FileSize"] = fileSize
 			} else {
-				d, _ := ioutil.ReadAll(dataRc)
-				buf = charset.ToUTF8WithFallback(append(buf, d...))
+				rd := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc))
 
 				if markupType := markup.Type(readmeFile.name); markupType != "" {
 					ctx.Data["IsMarkup"] = true
 					ctx.Data["MarkupType"] = string(markupType)
-					ctx.Data["FileContent"] = string(markup.Render(readmeFile.name, buf, readmeTreelink, ctx.Repo.Repository.ComposeDocumentMetas()))
+					var result strings.Builder
+					err := markup.Render(&markup.RenderContext{
+						Filename:  readmeFile.name,
+						URLPrefix: readmeTreelink,
+						Metas:     ctx.Repo.Repository.ComposeDocumentMetas(),
+					}, rd, &result)
+					if err != nil {
+						log.Error("Render failed: %v then fallback", err)
+						bs, _ := ioutil.ReadAll(rd)
+						ctx.Data["FileContent"] = strings.ReplaceAll(
+							gotemplate.HTMLEscapeString(string(bs)), "\n", `<br>`,
+						)
+					} else {
+						ctx.Data["FileContent"] = result.String()
+					}
 				} else {
 					ctx.Data["IsRenderedHTML"] = true
 					ctx.Data["FileContent"] = strings.ReplaceAll(
@@ -481,21 +494,30 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
 			break
 		}
 
-		d, _ := ioutil.ReadAll(dataRc)
-		buf = charset.ToUTF8WithFallback(append(buf, d...))
+		rd := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc))
 		readmeExist := markup.IsReadmeFile(blob.Name())
 		ctx.Data["ReadmeExist"] = readmeExist
 		if markupType := markup.Type(blob.Name()); markupType != "" {
 			ctx.Data["IsMarkup"] = true
 			ctx.Data["MarkupType"] = markupType
-			ctx.Data["FileContent"] = string(markup.Render(blob.Name(), buf, path.Dir(treeLink), ctx.Repo.Repository.ComposeDocumentMetas()))
+			var result strings.Builder
+			err := markup.Render(&markup.RenderContext{
+				Filename:  blob.Name(),
+				URLPrefix: path.Dir(treeLink),
+				Metas:     ctx.Repo.Repository.ComposeDocumentMetas(),
+			}, rd, &result)
+			if err != nil {
+				ctx.ServerError("Render", err)
+				return
+			}
+			ctx.Data["FileContent"] = result.String()
 		} else if readmeExist {
 			ctx.Data["IsRenderedHTML"] = true
 			ctx.Data["FileContent"] = strings.ReplaceAll(
 				gotemplate.HTMLEscapeString(string(buf)), "\n", `<br>`,
 			)
 		} else {
-			buf = charset.ToUTF8WithFallback(buf)
+			buf, _ := ioutil.ReadAll(rd)
 			lineNums := linesBytesCount(buf)
 			ctx.Data["NumLines"] = strconv.Itoa(lineNums)
 			ctx.Data["NumLinesSet"] = true
@@ -532,11 +554,20 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
 		}
 
 		if markupType := markup.Type(blob.Name()); markupType != "" {
-			d, _ := ioutil.ReadAll(dataRc)
-			buf = append(buf, d...)
+			rd := io.MultiReader(bytes.NewReader(buf), dataRc)
 			ctx.Data["IsMarkup"] = true
 			ctx.Data["MarkupType"] = markupType
-			ctx.Data["FileContent"] = string(markup.Render(blob.Name(), buf, path.Dir(treeLink), ctx.Repo.Repository.ComposeDocumentMetas()))
+			var result strings.Builder
+			err := markup.Render(&markup.RenderContext{
+				Filename:  blob.Name(),
+				URLPrefix: path.Dir(treeLink),
+				Metas:     ctx.Repo.Repository.ComposeDocumentMetas(),
+			}, rd, &result)
+			if err != nil {
+				ctx.ServerError("Render", err)
+				return
+			}
+			ctx.Data["FileContent"] = result.String()
 		}
 	}
 

routers/repo/wiki.go

@@ -6,6 +6,7 @@
 package repo
 
 import (
+	"bytes"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -211,12 +212,34 @@ func renderViewPage(ctx *context.Context) (*git.Repository, *git.TreeEntry) {
 		return nil, nil
 	}
 
-	metas := ctx.Repo.Repository.ComposeDocumentMetas()
-	ctx.Data["content"] = markdown.RenderWiki(data, ctx.Repo.RepoLink, metas)
+	var rctx = &markup.RenderContext{
+		URLPrefix: ctx.Repo.RepoLink,
+		Metas:     ctx.Repo.Repository.ComposeDocumentMetas(),
+		IsWiki:    true,
+	}
+
+	var buf strings.Builder
+	if err := markdown.Render(rctx, bytes.NewReader(data), &buf); err != nil {
+		ctx.ServerError("Render", err)
+		return nil, nil
+	}
+	ctx.Data["content"] = buf.String()
+
+	buf.Reset()
+	if err := markdown.Render(rctx, bytes.NewReader(sidebarContent), &buf); err != nil {
+		ctx.ServerError("Render", err)
+		return nil, nil
+	}
 	ctx.Data["sidebarPresent"] = sidebarContent != nil
-	ctx.Data["sidebarContent"] = markdown.RenderWiki(sidebarContent, ctx.Repo.RepoLink, metas)
+	ctx.Data["sidebarContent"] = buf.String()
+
+	buf.Reset()
+	if err := markdown.Render(rctx, bytes.NewReader(footerContent), &buf); err != nil {
+		ctx.ServerError("Render", err)
+		return nil, nil
+	}
 	ctx.Data["footerPresent"] = footerContent != nil
-	ctx.Data["footerContent"] = markdown.RenderWiki(footerContent, ctx.Repo.RepoLink, metas)
+	ctx.Data["footerContent"] = buf.String()
 
 	// get commit count - wiki revisions
 	commitsCount, _ := wikiRepo.FileCommitsCount("master", pageFilename)

routers/user/home.go

@@ -19,6 +19,7 @@ import (
 	"code.gitea.io/gitea/modules/context"
 	issue_indexer "code.gitea.io/gitea/modules/indexer/issues"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/markdown"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
@@ -267,7 +268,15 @@ func Milestones(ctx *context.Context) {
 			continue
 		}
 
-		milestones[i].RenderedContent = string(markdown.Render([]byte(milestones[i].Content), milestones[i].Repo.Link(), milestones[i].Repo.ComposeMetas()))
+		milestones[i].RenderedContent, err = markdown.RenderString(&markup.RenderContext{
+			URLPrefix: milestones[i].Repo.Link(),
+			Metas:     milestones[i].Repo.ComposeMetas(),
+		}, milestones[i].Content)
+		if err != nil {
+			ctx.ServerError("RenderString", err)
+			return
+		}
+
 		if milestones[i].Repo.IsTimetrackerEnabled() {
 			err := milestones[i].LoadTotalTrackedTime()
 			if err != nil {

routers/user/profile.go

@@ -13,6 +13,7 @@ import (
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/markdown"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
@@ -110,7 +111,15 @@ func Profile(ctx *context.Context) {
 	}
 
 	if len(ctxUser.Description) != 0 {
-		ctx.Data["RenderedDescription"] = string(markdown.Render([]byte(ctxUser.Description), ctx.Repo.RepoLink, map[string]string{"mode": "document"}))
+		content, err := markdown.RenderString(&markup.RenderContext{
+			URLPrefix: ctx.Repo.RepoLink,
+			Metas:     map[string]string{"mode": "document"},
+		}, ctxUser.Description)
+		if err != nil {
+			ctx.ServerError("RenderString", err)
+			return
+		}
+		ctx.Data["RenderedDescription"] = content
 	}
 
 	showPrivate := ctx.IsSigned && (ctx.User.IsAdmin || ctx.User.ID == ctxUser.ID)

services/gitdiff/csv_test.go

@@ -95,11 +95,17 @@ func TestCSVDiff(t *testing.T) {
 
 		var baseReader *csv.Reader
 		if len(c.base) > 0 {
-			baseReader = csv_module.CreateReaderAndGuessDelimiter([]byte(c.base))
+			baseReader, err = csv_module.CreateReaderAndGuessDelimiter(strings.NewReader(c.base))
+			if err != nil {
+				t.Errorf("CreateReaderAndGuessDelimiter failed: %s", err)
+			}
 		}
 		var headReader *csv.Reader
 		if len(c.head) > 0 {
-			headReader = csv_module.CreateReaderAndGuessDelimiter([]byte(c.head))
+			headReader, err = csv_module.CreateReaderAndGuessDelimiter(strings.NewReader(c.head))
+			if err != nil {
+				t.Errorf("CreateReaderAndGuessDelimiter failed: %s", err)
+			}
 		}
 
 		result, err := CreateCsvDiff(diff.Files[0], baseReader, headReader)

services/mailer/mail.go

@@ -174,8 +174,7 @@ func SendCollaboratorMail(u, doer *models.User, repo *models.Repository) {
 	SendAsync(msg)
 }
 
-func composeIssueCommentMessages(ctx *mailCommentContext, lang string, tos []string, fromMention bool, info string) []*Message {
-
+func composeIssueCommentMessages(ctx *mailCommentContext, lang string, tos []string, fromMention bool, info string) ([]*Message, error) {
 	var (
 		subject string
 		link    string
@@ -199,7 +198,14 @@ func composeIssueCommentMessages(ctx *mailCommentContext, lang string, tos []str
 	}
 
 	// This is the body of the new issue or comment, not the mail body
-	body := string(markup.RenderByType(markdown.MarkupName, []byte(ctx.Content), ctx.Issue.Repo.HTMLURL(), ctx.Issue.Repo.ComposeMetas()))
+	body, err := markdown.RenderString(&markup.RenderContext{
+		URLPrefix: ctx.Issue.Repo.HTMLURL(),
+		Metas:     ctx.Issue.Repo.ComposeMetas(),
+	}, ctx.Content)
+	if err != nil {
+		return nil, err
+	}
+
 	actType, actName, tplName := actionToTemplate(ctx.Issue, ctx.ActionType, commentType, reviewType)
 
 	if actName != "new" {
@@ -240,14 +246,13 @@ func composeIssueCommentMessages(ctx *mailCommentContext, lang string, tos []str
 	// TODO: i18n templates?
 	if err := subjectTemplates.ExecuteTemplate(&mailSubject, string(tplName), mailMeta); err == nil {
 		subject = sanitizeSubject(mailSubject.String())
+		if subject == "" {
+			subject = fallback
+		}
 	} else {
 		log.Error("ExecuteTemplate [%s]: %v", tplName+"/subject", err)
 	}
 
-	if subject == "" {
-		subject = fallback
-	}
-
 	subject = emoji.ReplaceAliases(subject)
 
 	mailMeta["Subject"] = subject
@@ -275,7 +280,7 @@ func composeIssueCommentMessages(ctx *mailCommentContext, lang string, tos []str
 		msgs = append(msgs, msg)
 	}
 
-	return msgs
+	return msgs, nil
 }
 
 func sanitizeSubject(subject string) string {
@@ -288,21 +293,26 @@ func sanitizeSubject(subject string) string {
 }
 
 // SendIssueAssignedMail composes and sends issue assigned email
-func SendIssueAssignedMail(issue *models.Issue, doer *models.User, content string, comment *models.Comment, recipients []*models.User) {
+func SendIssueAssignedMail(issue *models.Issue, doer *models.User, content string, comment *models.Comment, recipients []*models.User) error {
 	langMap := make(map[string][]string)
 	for _, user := range recipients {
 		langMap[user.Language] = append(langMap[user.Language], user.Email)
 	}
 
 	for lang, tos := range langMap {
-		SendAsyncs(composeIssueCommentMessages(&mailCommentContext{
+		msgs, err := composeIssueCommentMessages(&mailCommentContext{
 			Issue:      issue,
 			Doer:       doer,
 			ActionType: models.ActionType(0),
 			Content:    content,
 			Comment:    comment,
-		}, lang, tos, false, "issue assigned"))
+		}, lang, tos, false, "issue assigned")
+		if err != nil {
+			return err
+		}
+		SendAsyncs(msgs)
 	}
+	return nil
 }
 
 // actionToTemplate returns the type and name of the action facing the user

services/mailer/mail_issue.go

@@ -146,7 +146,11 @@ func mailIssueCommentBatch(ctx *mailCommentContext, users []*models.User, visite
 		// working backwards from the last (possibly) incomplete batch. If len(receivers) can be 0 this
 		// starting condition will need to be changed slightly
 		for i := ((len(receivers) - 1) / MailBatchSize) * MailBatchSize; i >= 0; i -= MailBatchSize {
-			SendAsyncs(composeIssueCommentMessages(ctx, lang, receivers[i:], fromMention, "issue comments"))
+			msgs, err := composeIssueCommentMessages(ctx, lang, receivers[i:], fromMention, "issue comments")
+			if err != nil {
+				return err
+			}
+			SendAsyncs(msgs)
 			receivers = receivers[:i]
 		}
 	}

services/mailer/mail_release.go

@@ -10,6 +10,7 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/markdown"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/translation"
@@ -48,7 +49,15 @@ func MailNewRelease(rel *models.Release) {
 func mailNewRelease(lang string, tos []string, rel *models.Release) {
 	locale := translation.NewLocale(lang)
 
-	rel.RenderedNote = markdown.RenderString(rel.Note, rel.Repo.Link(), rel.Repo.ComposeMetas())
+	var err error
+	rel.RenderedNote, err = markdown.RenderString(&markup.RenderContext{
+		URLPrefix: rel.Repo.Link(),
+		Metas:     rel.Repo.ComposeMetas(),
+	}, rel.Note)
+	if err != nil {
+		log.Error("markdown.RenderString(%d): %v", rel.RepoID, err)
+		return
+	}
 
 	subject := locale.Tr("mail.release.new.subject", rel.TagName, rel.Repo.FullName())
 	mailMeta := map[string]interface{}{

services/mailer/mail_test.go

@@ -58,8 +58,9 @@ func TestComposeIssueCommentMessage(t *testing.T) {
 	InitMailRender(stpl, btpl)
 
 	tos := []string{"test@gitea.com", "test2@gitea.com"}
-	msgs := composeIssueCommentMessages(&mailCommentContext{Issue: issue, Doer: doer, ActionType: models.ActionCommentIssue,
+	msgs, err := composeIssueCommentMessages(&mailCommentContext{Issue: issue, Doer: doer, ActionType: models.ActionCommentIssue,
 		Content: "test body", Comment: comment}, "en-US", tos, false, "issue comment")
+	assert.NoError(t, err)
 	assert.Len(t, msgs, 2)
 	gomailMsg := msgs[0].ToMessage()
 	mailto := gomailMsg.GetHeader("To")
@@ -92,8 +93,9 @@ func TestComposeIssueMessage(t *testing.T) {
 	InitMailRender(stpl, btpl)
 
 	tos := []string{"test@gitea.com", "test2@gitea.com"}
-	msgs := composeIssueCommentMessages(&mailCommentContext{Issue: issue, Doer: doer, ActionType: models.ActionCreateIssue,
+	msgs, err := composeIssueCommentMessages(&mailCommentContext{Issue: issue, Doer: doer, ActionType: models.ActionCreateIssue,
 		Content: "test body"}, "en-US", tos, false, "issue create")
+	assert.NoError(t, err)
 	assert.Len(t, msgs, 2)
 
 	gomailMsg := msgs[0].ToMessage()
@@ -218,7 +220,8 @@ func TestTemplateServices(t *testing.T) {
 }
 
 func testComposeIssueCommentMessage(t *testing.T, ctx *mailCommentContext, tos []string, fromMention bool, info string) *Message {
-	msgs := composeIssueCommentMessages(ctx, "en-US", tos, fromMention, info)
+	msgs, err := composeIssueCommentMessages(ctx, "en-US", tos, fromMention, info)
+	assert.NoError(t, err)
 	assert.Len(t, msgs, 1)
 	return msgs[0]
 }