From 91dd3b19f69bfbeefb22c66f75ea3321f2f97bb8 Mon Sep 17 00:00:00 2001 From: Christine Dodrill Date: Wed, 13 Dec 2017 10:43:58 -0800 Subject: convert to go buildpack --- vendor/github.com/russross/blackfriday/.gitignore | 8 + vendor/github.com/russross/blackfriday/.travis.yml | 18 + vendor/github.com/russross/blackfriday/LICENSE.txt | 29 + vendor/github.com/russross/blackfriday/README.md | 283 ++++ vendor/github.com/russross/blackfriday/block.go | 787 +++++---- .../github.com/russross/blackfriday/block_test.go | 1691 ++++++++++++++++++++ vendor/github.com/russross/blackfriday/doc.go | 40 +- vendor/github.com/russross/blackfriday/esc.go | 34 + vendor/github.com/russross/blackfriday/esc_test.go | 48 + .../russross/blackfriday/helpers_test.go | 186 +++ vendor/github.com/russross/blackfriday/html.go | 1519 +++++++++--------- vendor/github.com/russross/blackfriday/inline.go | 548 ++++--- .../github.com/russross/blackfriday/inline_test.go | 1176 ++++++++++++++ vendor/github.com/russross/blackfriday/latex.go | 332 ---- vendor/github.com/russross/blackfriday/markdown.go | 756 ++++----- .../russross/blackfriday/markdown_test.go | 38 + vendor/github.com/russross/blackfriday/node.go | 354 ++++ vendor/github.com/russross/blackfriday/ref_test.go | 124 ++ .../github.com/russross/blackfriday/smartypants.go | 157 +- .../testdata/Amps and angle encoding.html | 17 + .../testdata/Amps and angle encoding.text | 21 + .../russross/blackfriday/testdata/Auto links.html | 18 + .../russross/blackfriday/testdata/Auto links.text | 13 + .../blackfriday/testdata/Backslash escapes.html | 123 ++ .../blackfriday/testdata/Backslash escapes.text | 126 ++ .../testdata/Blockquotes with code blocks.html | 15 + .../testdata/Blockquotes with code blocks.text | 11 + .../russross/blackfriday/testdata/Code Blocks.html | 18 + .../russross/blackfriday/testdata/Code Blocks.text | 14 + .../russross/blackfriday/testdata/Code Spans.html | 5 + .../russross/blackfriday/testdata/Code Spans.text | 6 + ...list-like lines no empty line before block.html | 14 + ...list-like lines no empty line before block.text | 8 + ...rd-wrapped paragraphs with list-like lines.html | 8 + ...rd-wrapped paragraphs with list-like lines.text | 8 + .../blackfriday/testdata/Horizontal rules.html | 71 + .../blackfriday/testdata/Horizontal rules.text | 67 + .../testdata/Inline HTML (Advanced).html | 15 + .../testdata/Inline HTML (Advanced).text | 15 + .../blackfriday/testdata/Inline HTML (Simple).html | 72 + .../blackfriday/testdata/Inline HTML (Simple).text | 69 + .../blackfriday/testdata/Inline HTML comments.html | 13 + .../blackfriday/testdata/Inline HTML comments.text | 13 + .../blackfriday/testdata/Links, inline style.html | 11 + .../blackfriday/testdata/Links, inline style.text | 12 + .../testdata/Links, reference style.html | 52 + .../testdata/Links, reference style.text | 71 + .../testdata/Links, shortcut references.html | 9 + .../testdata/Links, shortcut references.text | 20 + .../testdata/Literal quotes in titles.html | 3 + .../testdata/Literal quotes in titles.text | 7 + .../testdata/Markdown Documentation - Basics.html | 314 ++++ .../testdata/Markdown Documentation - Basics.text | 306 ++++ .../testdata/Markdown Documentation - Syntax.html | 946 +++++++++++ .../testdata/Markdown Documentation - Syntax.text | 888 ++++++++++ .../blackfriday/testdata/Nested blockquotes.html | 9 + .../blackfriday/testdata/Nested blockquotes.text | 5 + .../testdata/Ordered and unordered lists.html | 166 ++ .../testdata/Ordered and unordered lists.text | 131 ++ .../testdata/Strong and em together.html | 7 + .../testdata/Strong and em together.text | 7 + .../russross/blackfriday/testdata/Tabs.html | 26 + .../russross/blackfriday/testdata/Tabs.text | 21 + .../russross/blackfriday/testdata/Tidyness.html | 9 + .../russross/blackfriday/testdata/Tidyness.text | 5 + 65 files changed, 9783 insertions(+), 2130 deletions(-) create mode 100644 vendor/github.com/russross/blackfriday/.gitignore create mode 100644 vendor/github.com/russross/blackfriday/.travis.yml create mode 100644 vendor/github.com/russross/blackfriday/LICENSE.txt create mode 100644 vendor/github.com/russross/blackfriday/README.md create mode 100644 vendor/github.com/russross/blackfriday/block_test.go create mode 100644 vendor/github.com/russross/blackfriday/esc.go create mode 100644 vendor/github.com/russross/blackfriday/esc_test.go create mode 100644 vendor/github.com/russross/blackfriday/helpers_test.go create mode 100644 vendor/github.com/russross/blackfriday/inline_test.go delete mode 100644 vendor/github.com/russross/blackfriday/latex.go create mode 100644 vendor/github.com/russross/blackfriday/markdown_test.go create mode 100644 vendor/github.com/russross/blackfriday/node.go create mode 100644 vendor/github.com/russross/blackfriday/ref_test.go create mode 100644 vendor/github.com/russross/blackfriday/testdata/Amps and angle encoding.html create mode 100644 vendor/github.com/russross/blackfriday/testdata/Amps and angle encoding.text create mode 100644 vendor/github.com/russross/blackfriday/testdata/Auto links.html create mode 100644 vendor/github.com/russross/blackfriday/testdata/Auto links.text create mode 100644 vendor/github.com/russross/blackfriday/testdata/Backslash escapes.html create mode 100644 vendor/github.com/russross/blackfriday/testdata/Backslash escapes.text create mode 100644 vendor/github.com/russross/blackfriday/testdata/Blockquotes with code blocks.html create mode 100644 vendor/github.com/russross/blackfriday/testdata/Blockquotes with code blocks.text create mode 100644 vendor/github.com/russross/blackfriday/testdata/Code Blocks.html create mode 100644 vendor/github.com/russross/blackfriday/testdata/Code Blocks.text create mode 100644 vendor/github.com/russross/blackfriday/testdata/Code Spans.html create mode 100644 vendor/github.com/russross/blackfriday/testdata/Code Spans.text create mode 100644 vendor/github.com/russross/blackfriday/testdata/Hard-wrapped paragraphs with list-like lines no empty line before block.html create mode 100644 vendor/github.com/russross/blackfriday/testdata/Hard-wrapped paragraphs with list-like lines no empty line before block.text create mode 100644 vendor/github.com/russross/blackfriday/testdata/Hard-wrapped paragraphs with list-like lines.html create mode 100644 vendor/github.com/russross/blackfriday/testdata/Hard-wrapped paragraphs with list-like lines.text create mode 100644 vendor/github.com/russross/blackfriday/testdata/Horizontal rules.html create mode 100644 vendor/github.com/russross/blackfriday/testdata/Horizontal rules.text create mode 100644 vendor/github.com/russross/blackfriday/testdata/Inline HTML (Advanced).html create mode 100644 vendor/github.com/russross/blackfriday/testdata/Inline HTML (Advanced).text create mode 100644 vendor/github.com/russross/blackfriday/testdata/Inline HTML (Simple).html create mode 100644 vendor/github.com/russross/blackfriday/testdata/Inline HTML (Simple).text create mode 100644 vendor/github.com/russross/blackfriday/testdata/Inline HTML comments.html create mode 100644 vendor/github.com/russross/blackfriday/testdata/Inline HTML comments.text create mode 100644 vendor/github.com/russross/blackfriday/testdata/Links, inline style.html create mode 100644 vendor/github.com/russross/blackfriday/testdata/Links, inline style.text create mode 100644 vendor/github.com/russross/blackfriday/testdata/Links, reference style.html create mode 100644 vendor/github.com/russross/blackfriday/testdata/Links, reference style.text create mode 100644 vendor/github.com/russross/blackfriday/testdata/Links, shortcut references.html create mode 100644 vendor/github.com/russross/blackfriday/testdata/Links, shortcut references.text create mode 100644 vendor/github.com/russross/blackfriday/testdata/Literal quotes in titles.html create mode 100644 vendor/github.com/russross/blackfriday/testdata/Literal quotes in titles.text create mode 100644 vendor/github.com/russross/blackfriday/testdata/Markdown Documentation - Basics.html create mode 100644 vendor/github.com/russross/blackfriday/testdata/Markdown Documentation - Basics.text create mode 100644 vendor/github.com/russross/blackfriday/testdata/Markdown Documentation - Syntax.html create mode 100644 vendor/github.com/russross/blackfriday/testdata/Markdown Documentation - Syntax.text create mode 100644 vendor/github.com/russross/blackfriday/testdata/Nested blockquotes.html create mode 100644 vendor/github.com/russross/blackfriday/testdata/Nested blockquotes.text create mode 100644 vendor/github.com/russross/blackfriday/testdata/Ordered and unordered lists.html create mode 100644 vendor/github.com/russross/blackfriday/testdata/Ordered and unordered lists.text create mode 100644 vendor/github.com/russross/blackfriday/testdata/Strong and em together.html create mode 100644 vendor/github.com/russross/blackfriday/testdata/Strong and em together.text create mode 100644 vendor/github.com/russross/blackfriday/testdata/Tabs.html create mode 100644 vendor/github.com/russross/blackfriday/testdata/Tabs.text create mode 100644 vendor/github.com/russross/blackfriday/testdata/Tidyness.html create mode 100644 vendor/github.com/russross/blackfriday/testdata/Tidyness.text (limited to 'vendor/github.com/russross') diff --git a/vendor/github.com/russross/blackfriday/.gitignore b/vendor/github.com/russross/blackfriday/.gitignore new file mode 100644 index 0000000..75623dc --- /dev/null +++ b/vendor/github.com/russross/blackfriday/.gitignore @@ -0,0 +1,8 @@ +*.out +*.swp +*.8 +*.6 +_obj +_test* +markdown +tags diff --git a/vendor/github.com/russross/blackfriday/.travis.yml b/vendor/github.com/russross/blackfriday/.travis.yml new file mode 100644 index 0000000..a4eb257 --- /dev/null +++ b/vendor/github.com/russross/blackfriday/.travis.yml @@ -0,0 +1,18 @@ +# Travis CI (http://travis-ci.org/) is a continuous integration service for +# open source projects. This file configures it to run unit tests for +# blackfriday. + +language: go + +go: + - 1.5 + - 1.6 + - 1.7 + +install: + - go get -d -t -v ./... + - go build -v ./... + +script: + - go test -v ./... + - go test -run=^$ -bench=BenchmarkReference -benchmem diff --git a/vendor/github.com/russross/blackfriday/LICENSE.txt b/vendor/github.com/russross/blackfriday/LICENSE.txt new file mode 100644 index 0000000..2885af3 --- /dev/null +++ b/vendor/github.com/russross/blackfriday/LICENSE.txt @@ -0,0 +1,29 @@ +Blackfriday is distributed under the Simplified BSD License: + +> Copyright © 2011 Russ Ross +> All rights reserved. +> +> Redistribution and use in source and binary forms, with or without +> modification, are permitted provided that the following conditions +> are met: +> +> 1. Redistributions of source code must retain the above copyright +> notice, this list of conditions and the following disclaimer. +> +> 2. Redistributions in binary form must reproduce the above +> copyright notice, this list of conditions and the following +> disclaimer in the documentation and/or other materials provided with +> the distribution. +> +> THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +> "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +> LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +> FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +> COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +> INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +> BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +> LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +> CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +> LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +> ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +> POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/russross/blackfriday/README.md b/vendor/github.com/russross/blackfriday/README.md new file mode 100644 index 0000000..2e0db35 --- /dev/null +++ b/vendor/github.com/russross/blackfriday/README.md @@ -0,0 +1,283 @@ +Blackfriday [![Build Status](https://travis-ci.org/russross/blackfriday.svg?branch=master)](https://travis-ci.org/russross/blackfriday) +=========== + +Blackfriday is a [Markdown][1] processor implemented in [Go][2]. It +is paranoid about its input (so you can safely feed it user-supplied +data), it is fast, it supports common extensions (tables, smart +punctuation substitutions, etc.), and it is safe for all utf-8 +(unicode) input. + +HTML output is currently supported, along with Smartypants +extensions. + +It started as a translation from C of [Sundown][3]. + + +Installation +------------ + +Blackfriday is compatible with any modern Go release. With Go 1.7 and git +installed: + + go get gopkg.in/russross/blackfriday.v2 + +will download, compile, and install the package into your `$GOPATH` +directory hierarchy. Alternatively, you can achieve the same if you +import it into a project: + + import "gopkg.in/russross/blackfriday.v2" + +and `go get` without parameters. + + +Versions +-------- + +Currently maintained and recommended version of Blackfriday is `v2`. It's being +developed on its own branch: https://github.com/russross/blackfriday/v2. You +should install and import it via [gopkg.in][6] at +`gopkg.in/russross/blackfriday.v2`. + +Version 2 offers a number of improvements over v1: + +* Cleaned up API +* A separate call to [`Parse`][4], which produces an abstract syntax tree for + the document +* Latest bug fixes +* Flexibility to easily add your own rendering extensions + +Potential drawbacks: + +* Our benchmarks show v2 to be slightly slower than v1. Currently in the + ballpark of around 15%. +* API breakage. If you can't afford modifying your code to adhere to the new API + and don't care too much about the new features, v2 is probably not for you. +* Several bug fixes are trailing behind and still need to be forward-ported to + v2. See issue [#348](https://github.com/russross/blackfriday/issues/348) for + tracking. + +Usage +----- + +For the most sensible markdown processing, it is as simple as getting your input +into a byte slice and calling: + +```go +output := blackfriday.Run(input) +``` + +Your input will be parsed and the output rendered with a set of most popular +extensions enabled. If you want the most basic feature set, corresponding with +the bare Markdown specification, use: + +```go +output := blackfriday.Run(input, blackfriday.WithNoExtensions()) +``` + +### Sanitize untrusted content + +Blackfriday itself does nothing to protect against malicious content. If you are +dealing with user-supplied markdown, we recommend running Blackfriday's output +through HTML sanitizer such as [Bluemonday][5]. + +Here's an example of simple usage of Blackfriday together with Bluemonday: + +```go +import ( + "github.com/microcosm-cc/bluemonday" + "github.com/russross/blackfriday" +) + +// ... +unsafe := blackfriday.Run(input) +html := bluemonday.UGCPolicy().SanitizeBytes(unsafe) +``` + +### Custom options + +If you want to customize the set of options, use `blackfriday.WithExtensions`, +`blackfriday.WithRenderer` and `blackfriday.WithRefOverride`. + +You can also check out `blackfriday-tool` for a more complete example +of how to use it. Download and install it using: + + go get github.com/russross/blackfriday-tool + +This is a simple command-line tool that allows you to process a +markdown file using a standalone program. You can also browse the +source directly on github if you are just looking for some example +code: + +* + +Note that if you have not already done so, installing +`blackfriday-tool` will be sufficient to download and install +blackfriday in addition to the tool itself. The tool binary will be +installed in `$GOPATH/bin`. This is a statically-linked binary that +can be copied to wherever you need it without worrying about +dependencies and library versions. + + +Features +-------- + +All features of Sundown are supported, including: + +* **Compatibility**. The Markdown v1.0.3 test suite passes with + the `--tidy` option. Without `--tidy`, the differences are + mostly in whitespace and entity escaping, where blackfriday is + more consistent and cleaner. + +* **Common extensions**, including table support, fenced code + blocks, autolinks, strikethroughs, non-strict emphasis, etc. + +* **Safety**. Blackfriday is paranoid when parsing, making it safe + to feed untrusted user input without fear of bad things + happening. The test suite stress tests this and there are no + known inputs that make it crash. If you find one, please let me + know and send me the input that does it. + + NOTE: "safety" in this context means *runtime safety only*. In order to + protect yourself against JavaScript injection in untrusted content, see + [this example](https://github.com/russross/blackfriday#sanitize-untrusted-content). + +* **Fast processing**. It is fast enough to render on-demand in + most web applications without having to cache the output. + +* **Thread safety**. You can run multiple parsers in different + goroutines without ill effect. There is no dependence on global + shared state. + +* **Minimal dependencies**. Blackfriday only depends on standard + library packages in Go. The source code is pretty + self-contained, so it is easy to add to any project, including + Google App Engine projects. + +* **Standards compliant**. Output successfully validates using the + W3C validation tool for HTML 4.01 and XHTML 1.0 Transitional. + + +Extensions +---------- + +In addition to the standard markdown syntax, this package +implements the following extensions: + +* **Intra-word emphasis supression**. The `_` character is + commonly used inside words when discussing code, so having + markdown interpret it as an emphasis command is usually the + wrong thing. Blackfriday lets you treat all emphasis markers as + normal characters when they occur inside a word. + +* **Tables**. Tables can be created by drawing them in the input + using a simple syntax: + + ``` + Name | Age + --------|------ + Bob | 27 + Alice | 23 + ``` + +* **Fenced code blocks**. In addition to the normal 4-space + indentation to mark code blocks, you can explicitly mark them + and supply a language (to make syntax highlighting simple). Just + mark it like this: + + ```go + func getTrue() bool { + return true + } + ``` + + You can use 3 or more backticks to mark the beginning of the + block, and the same number to mark the end of the block. + +* **Definition lists**. A simple definition list is made of a single-line + term followed by a colon and the definition for that term. + + Cat + : Fluffy animal everyone likes + + Internet + : Vector of transmission for pictures of cats + + Terms must be separated from the previous definition by a blank line. + +* **Footnotes**. A marker in the text that will become a superscript number; + a footnote definition that will be placed in a list of footnotes at the + end of the document. A footnote looks like this: + + This is a footnote.[^1] + + [^1]: the footnote text. + +* **Autolinking**. Blackfriday can find URLs that have not been + explicitly marked as links and turn them into links. + +* **Strikethrough**. Use two tildes (`~~`) to mark text that + should be crossed out. + +* **Hard line breaks**. With this extension enabled newlines in the input + translate into line breaks in the output. This extension is off by default. + +* **Smart quotes**. Smartypants-style punctuation substitution is + supported, turning normal double- and single-quote marks into + curly quotes, etc. + +* **LaTeX-style dash parsing** is an additional option, where `--` + is translated into `–`, and `---` is translated into + `—`. This differs from most smartypants processors, which + turn a single hyphen into an ndash and a double hyphen into an + mdash. + +* **Smart fractions**, where anything that looks like a fraction + is translated into suitable HTML (instead of just a few special + cases like most smartypant processors). For example, `4/5` + becomes `4⁄5`, which renders as + 4⁄5. + + +Other renderers +--------------- + +Blackfriday is structured to allow alternative rendering engines. Here +are a few of note: + +* [github_flavored_markdown](https://godoc.org/github.com/shurcooL/github_flavored_markdown): + provides a GitHub Flavored Markdown renderer with fenced code block + highlighting, clickable heading anchor links. + + It's not customizable, and its goal is to produce HTML output + equivalent to the [GitHub Markdown API endpoint](https://developer.github.com/v3/markdown/#render-a-markdown-document-in-raw-mode), + except the rendering is performed locally. + +* [markdownfmt](https://github.com/shurcooL/markdownfmt): like gofmt, + but for markdown. + +* [LaTeX output](https://bitbucket.org/ambrevar/blackfriday-latex): + renders output as LaTeX. + + +Todo +---- + +* More unit testing +* Improve unicode support. It does not understand all unicode + rules (about what constitutes a letter, a punctuation symbol, + etc.), so it may fail to detect word boundaries correctly in + some instances. It is safe on all utf-8 input. + + +License +------- + +[Blackfriday is distributed under the Simplified BSD License](LICENSE.txt) + + + [1]: https://daringfireball.net/projects/markdown/ "Markdown" + [2]: https://golang.org/ "Go Language" + [3]: https://github.com/vmg/sundown "Sundown" + [4]: https://godoc.org/gopkg.in/russross/blackfriday.v2#Parse "Parse func" + [5]: https://github.com/microcosm-cc/bluemonday "Bluemonday" + [6]: https://labix.org/gopkg.in "gopkg.in" diff --git a/vendor/github.com/russross/blackfriday/block.go b/vendor/github.com/russross/blackfriday/block.go index 7fc731d..d7da33f 100644 --- a/vendor/github.com/russross/blackfriday/block.go +++ b/vendor/github.com/russross/blackfriday/block.go @@ -15,17 +15,26 @@ package blackfriday import ( "bytes" - "unicode" + "html" + "regexp" + + "github.com/shurcooL/sanitized_anchor_name" +) + +const ( + charEntity = "&(?:#x[a-f0-9]{1,8}|#[0-9]{1,8}|[a-z][a-z0-9]{1,31});" + escapable = "[!\"#$%&'()*+,./:;<=>?@[\\\\\\]^_`{|}~-]" +) + +var ( + reBackslashOrAmp = regexp.MustCompile("[\\&]") + reEntityOrEscapedChar = regexp.MustCompile("(?i)\\\\" + escapable + "|" + charEntity) ) // Parse block-level data. // Note: this function and many that it calls assume that // the input buffer ends with a newline. -func (p *parser) block(out *bytes.Buffer, data []byte) { - if len(data) == 0 || data[len(data)-1] != '\n' { - panic("block input is missing terminating newline") - } - +func (p *Markdown) block(data []byte) { // this is called recursively: enforce a maximum depth if p.nesting >= p.maxNesting { return @@ -34,14 +43,14 @@ func (p *parser) block(out *bytes.Buffer, data []byte) { // parse out one block-level construct at a time for len(data) > 0 { - // prefixed header: + // prefixed heading: // - // # Header 1 - // ## Header 2 + // # Heading 1 + // ## Heading 2 // ... - // ###### Header 6 - if p.isPrefixHeader(data) { - data = data[p.prefixHeader(out, data):] + // ###### Heading 6 + if p.isPrefixHeading(data) { + data = data[p.prefixHeading(data):] continue } @@ -51,7 +60,7 @@ func (p *parser) block(out *bytes.Buffer, data []byte) { // ... // if data[0] == '<' { - if i := p.html(out, data, true); i > 0 { + if i := p.html(data, true); i > 0 { data = data[i:] continue } @@ -62,9 +71,9 @@ func (p *parser) block(out *bytes.Buffer, data []byte) { // % stuff // % more stuff // % even more stuff - if p.flags&EXTENSION_TITLEBLOCK != 0 { + if p.extensions&Titleblock != 0 { if data[0] == '%' { - if i := p.titleBlock(out, data, true); i > 0 { + if i := p.titleBlock(data, true); i > 0 { data = data[i:] continue } @@ -86,7 +95,7 @@ func (p *parser) block(out *bytes.Buffer, data []byte) { // return b // } if p.codePrefix(data) > 0 { - data = data[p.code(out, data):] + data = data[p.code(data):] continue } @@ -100,8 +109,8 @@ func (p *parser) block(out *bytes.Buffer, data []byte) { // return n * fact(n-1) // } // ``` - if p.flags&EXTENSION_FENCED_CODE != 0 { - if i := p.fencedCodeBlock(out, data, true); i > 0 { + if p.extensions&FencedCode != 0 { + if i := p.fencedCodeBlock(data, true); i > 0 { data = data[i:] continue } @@ -115,9 +124,9 @@ func (p *parser) block(out *bytes.Buffer, data []byte) { // or // ______ if p.isHRule(data) { - p.r.HRule(out) + p.addBlock(HorizontalRule, nil) var i int - for i = 0; data[i] != '\n'; i++ { + for i = 0; i < len(data) && data[i] != '\n'; i++ { } data = data[i:] continue @@ -128,7 +137,7 @@ func (p *parser) block(out *bytes.Buffer, data []byte) { // > A big quote I found somewhere // > on the web if p.quotePrefix(data) > 0 { - data = data[p.quote(out, data):] + data = data[p.quote(data):] continue } @@ -138,8 +147,8 @@ func (p *parser) block(out *bytes.Buffer, data []byte) { // ------|-----|--------- // Bob | 31 | 555-1234 // Alice | 27 | 555-4321 - if p.flags&EXTENSION_TABLES != 0 { - if i := p.table(out, data); i > 0 { + if p.extensions&Tables != 0 { + if i := p.table(data); i > 0 { data = data[i:] continue } @@ -152,7 +161,7 @@ func (p *parser) block(out *bytes.Buffer, data []byte) { // // also works with + or - if p.uliPrefix(data) > 0 { - data = data[p.list(out, data, 0):] + data = data[p.list(data, 0):] continue } @@ -161,7 +170,7 @@ func (p *parser) block(out *bytes.Buffer, data []byte) { // 1. Item 1 // 2. Item 2 if p.oliPrefix(data) > 0 { - data = data[p.list(out, data, LIST_TYPE_ORDERED):] + data = data[p.list(data, ListTypeOrdered):] continue } @@ -173,55 +182,62 @@ func (p *parser) block(out *bytes.Buffer, data []byte) { // // Term 2 // : Definition c - if p.flags&EXTENSION_DEFINITION_LISTS != 0 { + if p.extensions&DefinitionLists != 0 { if p.dliPrefix(data) > 0 { - data = data[p.list(out, data, LIST_TYPE_DEFINITION):] + data = data[p.list(data, ListTypeDefinition):] continue } } // anything else must look like a normal paragraph - // note: this finds underlined headers, too - data = data[p.paragraph(out, data):] + // note: this finds underlined headings, too + data = data[p.paragraph(data):] } p.nesting-- } -func (p *parser) isPrefixHeader(data []byte) bool { +func (p *Markdown) addBlock(typ NodeType, content []byte) *Node { + p.closeUnmatchedBlocks() + container := p.addChild(typ, 0) + container.content = content + return container +} + +func (p *Markdown) isPrefixHeading(data []byte) bool { if data[0] != '#' { return false } - if p.flags&EXTENSION_SPACE_HEADERS != 0 { + if p.extensions&SpaceHeadings != 0 { level := 0 - for level < 6 && data[level] == '#' { + for level < 6 && level < len(data) && data[level] == '#' { level++ } - if data[level] != ' ' { + if level == len(data) || data[level] != ' ' { return false } } return true } -func (p *parser) prefixHeader(out *bytes.Buffer, data []byte) int { +func (p *Markdown) prefixHeading(data []byte) int { level := 0 - for level < 6 && data[level] == '#' { + for level < 6 && level < len(data) && data[level] == '#' { level++ } i := skipChar(data, level, ' ') end := skipUntilChar(data, i, '\n') skip := end id := "" - if p.flags&EXTENSION_HEADER_IDS != 0 { + if p.extensions&HeadingIDs != 0 { j, k := 0, 0 - // find start/end of header id + // find start/end of heading id for j = i; j < end-1 && (data[j] != '{' || data[j+1] != '#'); j++ { } for k = j + 1; k < end && data[k] != '}'; k++ { } - // extract header id iff found + // extract heading id iff found if j < end && k < end { id = string(data[j+2 : k]) end = j @@ -241,45 +257,41 @@ func (p *parser) prefixHeader(out *bytes.Buffer, data []byte) int { end-- } if end > i { - if id == "" && p.flags&EXTENSION_AUTO_HEADER_IDS != 0 { - id = SanitizedAnchorName(string(data[i:end])) + if id == "" && p.extensions&AutoHeadingIDs != 0 { + id = sanitized_anchor_name.Create(string(data[i:end])) } - work := func() bool { - p.inline(out, data[i:end]) - return true - } - p.r.Header(out, work, level, id) + block := p.addBlock(Heading, data[i:end]) + block.HeadingID = id + block.Level = level } return skip } -func (p *parser) isUnderlinedHeader(data []byte) int { - // test of level 1 header +func (p *Markdown) isUnderlinedHeading(data []byte) int { + // test of level 1 heading if data[0] == '=' { i := skipChar(data, 1, '=') i = skipChar(data, i, ' ') - if data[i] == '\n' { + if i < len(data) && data[i] == '\n' { return 1 - } else { - return 0 } + return 0 } - // test of level 2 header + // test of level 2 heading if data[0] == '-' { i := skipChar(data, 1, '-') i = skipChar(data, i, ' ') - if data[i] == '\n' { + if i < len(data) && data[i] == '\n' { return 2 - } else { - return 0 } + return 0 } return 0 } -func (p *parser) titleBlock(out *bytes.Buffer, data []byte, doRender bool) int { +func (p *Markdown) titleBlock(data []byte, doRender bool) int { if data[0] != '%' { return 0 } @@ -293,12 +305,17 @@ func (p *parser) titleBlock(out *bytes.Buffer, data []byte, doRender bool) int { } data = bytes.Join(splitData[0:i], []byte("\n")) - p.r.TitleBlock(out, data) - - return len(data) + consumed := len(data) + data = bytes.TrimPrefix(data, []byte("% ")) + data = bytes.Replace(data, []byte("\n% "), []byte("\n"), -1) + block := p.addBlock(Heading, data) + block.Level = 1 + block.IsTitleblock = true + + return consumed } -func (p *parser) html(out *bytes.Buffer, data []byte, doRender bool) int { +func (p *Markdown) html(data []byte, doRender bool) int { var i, j int // identify the opening tag @@ -310,17 +327,12 @@ func (p *parser) html(out *bytes.Buffer, data []byte, doRender bool) int { // handle special cases if !tagfound { // check for an HTML comment - if size := p.htmlComment(out, data, doRender); size > 0 { + if size := p.htmlComment(data, doRender); size > 0 { return size } // check for an
tag - if size := p.htmlHr(out, data, doRender); size > 0 { - return size - } - - // check for HTML CDATA - if size := p.htmlCDATA(out, data, doRender); size > 0 { + if size := p.htmlHr(data, doRender); size > 0 { return size } @@ -395,60 +407,42 @@ func (p *parser) html(out *bytes.Buffer, data []byte, doRender bool) int { for end > 0 && data[end-1] == '\n' { end-- } - p.r.BlockHtml(out, data[:end]) + finalizeHTMLBlock(p.addBlock(HTMLBlock, data[:end])) } return i } -func (p *parser) renderHTMLBlock(out *bytes.Buffer, data []byte, start int, doRender bool) int { - // html block needs to end with a blank line - if i := p.isEmpty(data[start:]); i > 0 { - size := start + i +func finalizeHTMLBlock(block *Node) { + block.Literal = block.content + block.content = nil +} + +// HTML comment, lax form +func (p *Markdown) htmlComment(data []byte, doRender bool) int { + i := p.inlineHTMLComment(data) + // needs to end with a blank line + if j := p.isEmpty(data[i:]); j > 0 { + size := i + j if doRender { // trim trailing newlines end := size for end > 0 && data[end-1] == '\n' { end-- } - p.r.BlockHtml(out, data[:end]) + block := p.addBlock(HTMLBlock, data[:end]) + finalizeHTMLBlock(block) } return size } return 0 } -// HTML comment, lax form -func (p *parser) htmlComment(out *bytes.Buffer, data []byte, doRender bool) int { - i := p.inlineHTMLComment(out, data) - return p.renderHTMLBlock(out, data, i, doRender) -} - -// HTML CDATA section -func (p *parser) htmlCDATA(out *bytes.Buffer, data []byte, doRender bool) int { - const cdataTag = "') { - i++ - } - i++ - // no end-of-comment marker - if i >= len(data) { +// HR, which is the only self-closing block tag considered +func (p *Markdown) htmlHr(data []byte, doRender bool) int { + if len(data) < 4 { return 0 } - return p.renderHTMLBlock(out, data, i, doRender) -} - -// HR, which is the only self-closing block tag considered -func (p *parser) htmlHr(out *bytes.Buffer, data []byte, doRender bool) int { if data[0] != '<' || (data[1] != 'h' && data[1] != 'H') || (data[2] != 'r' && data[2] != 'R') { return 0 } @@ -456,22 +450,31 @@ func (p *parser) htmlHr(out *bytes.Buffer, data []byte, doRender bool) int { // not an
tag after all; at least not a valid one return 0 } - i := 3 - for data[i] != '>' && data[i] != '\n' { + for i < len(data) && data[i] != '>' && data[i] != '\n' { i++ } - - if data[i] == '>' { - return p.renderHTMLBlock(out, data, i+1, doRender) + if i < len(data) && data[i] == '>' { + i++ + if j := p.isEmpty(data[i:]); j > 0 { + size := i + j + if doRender { + // trim newlines + end := size + for end > 0 && data[end-1] == '\n' { + end-- + } + finalizeHTMLBlock(p.addBlock(HTMLBlock, data[:end])) + } + return size + } } - return 0 } -func (p *parser) htmlFindTag(data []byte) (string, bool) { +func (p *Markdown) htmlFindTag(data []byte) (string, bool) { i := 0 - for isalnum(data[i]) { + for i < len(data) && isalnum(data[i]) { i++ } key := string(data[:i]) @@ -481,9 +484,11 @@ func (p *parser) htmlFindTag(data []byte) (string, bool) { return "", false } -func (p *parser) htmlFindEnd(tag string, data []byte) int { +func (p *Markdown) htmlFindEnd(tag string, data []byte) int { // assume data[0] == '<' && data[1] == '/' already tested - + if tag == "hr" { + return 2 + } // check if tag is a match closetag := []byte("") if !bytes.HasPrefix(data, closetag) { @@ -503,7 +508,7 @@ func (p *parser) htmlFindEnd(tag string, data []byte) int { return i } - if p.flags&EXTENSION_LAX_HTML_BLOCKS != 0 { + if p.extensions&LaxHTMLBlocks != 0 { return i } if skip = p.isEmpty(data[i:]); skip == 0 { @@ -514,7 +519,7 @@ func (p *parser) htmlFindEnd(tag string, data []byte) int { return i + skip } -func (*parser) isEmpty(data []byte) int { +func (*Markdown) isEmpty(data []byte) int { // it is okay to call isEmpty on an empty buffer if len(data) == 0 { return 0 @@ -526,10 +531,13 @@ func (*parser) isEmpty(data []byte) int { return 0 } } - return i + 1 + if i < len(data) && data[i] == '\n' { + i++ + } + return i } -func (*parser) isHRule(data []byte) bool { +func (*Markdown) isHRule(data []byte) bool { i := 0 // skip up to three spaces @@ -545,7 +553,7 @@ func (*parser) isHRule(data []byte) bool { // the whole line must be the char or whitespace n := 0 - for data[i] != '\n' { + for i < len(data) && data[i] != '\n' { switch { case data[i] == c: n++ @@ -561,8 +569,7 @@ func (*parser) isHRule(data []byte) bool { // isFenceLine checks if there's a fence line (e.g., ``` or ``` go) at the beginning of data, // and returns the end index if so, or 0 otherwise. It also returns the marker found. // If syntax is not nil, it gets set to the syntax specified in the fence line. -// A final newline is mandatory to recognize the fence line, unless newlineOptional is true. -func isFenceLine(data []byte, syntax *string, oldmarker string, newlineOptional bool) (end int, marker string) { +func isFenceLine(data []byte, syntax *string, oldmarker string) (end int, marker string) { i, size := 0, 0 // skip up to three spaces @@ -604,7 +611,7 @@ func isFenceLine(data []byte, syntax *string, oldmarker string, newlineOptional i = skipChar(data, i, ' ') if i >= len(data) { - if newlineOptional && i == len(data) { + if i == len(data) { return i, marker } return 0, "" @@ -649,33 +656,33 @@ func isFenceLine(data []byte, syntax *string, oldmarker string, newlineOptional i = skipChar(data, i, ' ') if i >= len(data) || data[i] != '\n' { - if newlineOptional && i == len(data) { + if i == len(data) { return i, marker } return 0, "" } - return i + 1, marker // Take newline into account. } // fencedCodeBlock returns the end index if data contains a fenced code block at the beginning, // or 0 otherwise. It writes to out if doRender is true, otherwise it has no side effects. // If doRender is true, a final newline is mandatory to recognize the fenced code block. -func (p *parser) fencedCodeBlock(out *bytes.Buffer, data []byte, doRender bool) int { +func (p *Markdown) fencedCodeBlock(data []byte, doRender bool) int { var syntax string - beg, marker := isFenceLine(data, &syntax, "", false) + beg, marker := isFenceLine(data, &syntax, "") if beg == 0 || beg >= len(data) { return 0 } var work bytes.Buffer + work.Write([]byte(syntax)) + work.WriteByte('\n') for { // safe to assume beg < len(data) // check for the end of the code block - newlineOptional := !doRender - fenceEnd, _ := isFenceLine(data[beg:], nil, marker, newlineOptional) + fenceEnd, _ := isFenceLine(data[beg:], nil, marker) if fenceEnd != 0 { beg += fenceEnd break @@ -697,24 +704,55 @@ func (p *parser) fencedCodeBlock(out *bytes.Buffer, data []byte, doRender bool) } if doRender { - p.r.BlockCode(out, work.Bytes(), syntax) + block := p.addBlock(CodeBlock, work.Bytes()) // TODO: get rid of temp buffer + block.IsFenced = true + finalizeCodeBlock(block) } return beg } -func (p *parser) table(out *bytes.Buffer, data []byte) int { - var header bytes.Buffer - i, columns := p.tableHeader(&header, data) +func unescapeChar(str []byte) []byte { + if str[0] == '\\' { + return []byte{str[1]} + } + return []byte(html.UnescapeString(string(str))) +} + +func unescapeString(str []byte) []byte { + if reBackslashOrAmp.Match(str) { + return reEntityOrEscapedChar.ReplaceAllFunc(str, unescapeChar) + } + return str +} + +func finalizeCodeBlock(block *Node) { + if block.IsFenced { + newlinePos := bytes.IndexByte(block.content, '\n') + firstLine := block.content[:newlinePos] + rest := block.content[newlinePos+1:] + block.Info = unescapeString(bytes.Trim(firstLine, "\n")) + block.Literal = rest + } else { + block.Literal = block.content + } + block.content = nil +} + +func (p *Markdown) table(data []byte) int { + table := p.addBlock(Table, nil) + i, columns := p.tableHeader(data) if i == 0 { + p.tip = table.Parent + table.Unlink() return 0 } - var body bytes.Buffer + p.addBlock(TableBody, nil) for i < len(data) { pipes, rowStart := 0, i - for ; data[i] != '\n'; i++ { + for ; i < len(data) && data[i] != '\n'; i++ { if data[i] == '|' { pipes++ } @@ -726,12 +764,12 @@ func (p *parser) table(out *bytes.Buffer, data []byte) int { } // include the newline in data sent to tableRow - i++ - p.tableRow(&body, data[rowStart:i], columns, false) + if i < len(data) && data[i] == '\n' { + i++ + } + p.tableRow(data[rowStart:i], columns, false) } - p.r.Table(out, header.Bytes(), body.Bytes(), columns) - return i } @@ -744,10 +782,10 @@ func isBackslashEscaped(data []byte, i int) bool { return backslashes&1 == 1 } -func (p *parser) tableHeader(out *bytes.Buffer, data []byte) (size int, columns []int) { +func (p *Markdown) tableHeader(data []byte) (size int, columns []CellAlignFlags) { i := 0 colCount := 1 - for i = 0; data[i] != '\n'; i++ { + for i = 0; i < len(data) && data[i] != '\n'; i++ { if data[i] == '|' && !isBackslashEscaped(data, i) { colCount++ } @@ -759,7 +797,11 @@ func (p *parser) tableHeader(out *bytes.Buffer, data []byte) (size int, columns } // include the newline in the data sent to tableRow - header := data[:i+1] + j := i + if j < len(data) && data[j] == '\n' { + j++ + } + header := data[:j] // column count ignores pipes at beginning or end of line if data[0] == '|' { @@ -769,7 +811,7 @@ func (p *parser) tableHeader(out *bytes.Buffer, data []byte) (size int, columns colCount-- } - columns = make([]int, colCount) + columns = make([]CellAlignFlags, colCount) // move on to the header underline i++ @@ -785,27 +827,29 @@ func (p *parser) tableHeader(out *bytes.Buffer, data []byte) (size int, columns // each column header is of form: / *:?-+:? *|/ with # dashes + # colons >= 3 // and trailing | optional on last column col := 0 - for data[i] != '\n' { + for i < len(data) && data[i] != '\n' { dashes := 0 if data[i] == ':' { i++ - columns[col] |= TABLE_ALIGNMENT_LEFT + columns[col] |= TableAlignmentLeft dashes++ } - for data[i] == '-' { + for i < len(data) && data[i] == '-' { i++ dashes++ } - if data[i] == ':' { + if i < len(data) && data[i] == ':' { i++ - columns[col] |= TABLE_ALIGNMENT_RIGHT + columns[col] |= TableAlignmentRight dashes++ } - for data[i] == ' ' { + for i < len(data) && data[i] == ' ' { i++ } - + if i == len(data) { + return + } // end of column test is messy switch { case dashes < 3: @@ -816,12 +860,12 @@ func (p *parser) tableHeader(out *bytes.Buffer, data []byte) (size int, columns // marker found, now skip past trailing whitespace col++ i++ - for data[i] == ' ' { + for i < len(data) && data[i] == ' ' { i++ } // trailing junk found after last column - if col >= colCount && data[i] != '\n' { + if col >= colCount && i < len(data) && data[i] != '\n' { return } @@ -842,27 +886,31 @@ func (p *parser) tableHeader(out *bytes.Buffer, data []byte) (size int, columns return } - p.tableRow(out, header, columns, true) - size = i + 1 + p.addBlock(TableHead, nil) + p.tableRow(header, columns, true) + size = i + if size < len(data) && data[size] == '\n' { + size++ + } return } -func (p *parser) tableRow(out *bytes.Buffer, data []byte, columns []int, header bool) { +func (p *Markdown) tableRow(data []byte, columns []CellAlignFlags, header bool) { + p.addBlock(TableRow, nil) i, col := 0, 0 - var rowWork bytes.Buffer if data[i] == '|' && !isBackslashEscaped(data, i) { i++ } for col = 0; col < len(columns) && i < len(data); col++ { - for data[i] == ' ' { + for i < len(data) && data[i] == ' ' { i++ } cellStart := i - for (data[i] != '|' || isBackslashEscaped(data, i)) && data[i] != '\n' { + for i < len(data) && (data[i] != '|' || isBackslashEscaped(data, i)) && data[i] != '\n' { i++ } @@ -871,42 +919,33 @@ func (p *parser) tableRow(out *bytes.Buffer, data []byte, columns []int, header // skip the end-of-cell marker, possibly taking us past end of buffer i++ - for cellEnd > cellStart && data[cellEnd-1] == ' ' { + for cellEnd > cellStart && cellEnd-1 < len(data) && data[cellEnd-1] == ' ' { cellEnd-- } - var cellWork bytes.Buffer - p.inline(&cellWork, data[cellStart:cellEnd]) - - if header { - p.r.TableHeaderCell(&rowWork, cellWork.Bytes(), columns[col]) - } else { - p.r.TableCell(&rowWork, cellWork.Bytes(), columns[col]) - } + cell := p.addBlock(TableCell, data[cellStart:cellEnd]) + cell.IsHeader = header + cell.Align = columns[col] } // pad it out with empty columns to get the right number for ; col < len(columns); col++ { - if header { - p.r.TableHeaderCell(&rowWork, nil, columns[col]) - } else { - p.r.TableCell(&rowWork, nil, columns[col]) - } + cell := p.addBlock(TableCell, nil) + cell.IsHeader = header + cell.Align = columns[col] } // silently ignore rows with too many cells - - p.r.TableRow(out, rowWork.Bytes()) } // returns blockquote prefix length -func (p *parser) quotePrefix(data []byte) int { +func (p *Markdown) quotePrefix(data []byte) int { i := 0 - for i < 3 && data[i] == ' ' { + for i < 3 && i < len(data) && data[i] == ' ' { i++ } - if data[i] == '>' { - if data[i+1] == ' ' { + if i < len(data) && data[i] == '>' { + if i+1 < len(data) && data[i+1] == ' ' { return i + 2 } return i + 1 @@ -916,7 +955,7 @@ func (p *parser) quotePrefix(data []byte) int { // blockquote ends with at least one blank line // followed by something without a blockquote prefix -func (p *parser) terminateBlockquote(data []byte, beg, end int) bool { +func (p *Markdown) terminateBlockquote(data []byte, beg, end int) bool { if p.isEmpty(data[beg:]) <= 0 { return false } @@ -927,7 +966,8 @@ func (p *parser) terminateBlockquote(data []byte, beg, end int) bool { } // parse a blockquote fragment -func (p *parser) quote(out *bytes.Buffer, data []byte) int { +func (p *Markdown) quote(data []byte) int { + block := p.addBlock(BlockQuote, nil) var raw bytes.Buffer beg, end := 0, 0 for beg < len(data) { @@ -935,9 +975,9 @@ func (p *parser) quote(out *bytes.Buffer, data []byte) int { // Step over whole lines, collecting them. While doing that, check for // fenced code and if one's found, incorporate it altogether, // irregardless of any contents inside it - for data[end] != '\n' { - if p.flags&EXTENSION_FENCED_CODE != 0 { - if i := p.fencedCodeBlock(out, data[end:], false); i > 0 { + for end < len(data) && data[end] != '\n' { + if p.extensions&FencedCode != 0 { + if i := p.fencedCodeBlock(data[end:], false); i > 0 { // -1 to compensate for the extra end++ after the loop: end += i - 1 break @@ -945,44 +985,47 @@ func (p *parser) quote(out *bytes.Buffer, data []byte) int { } end++ } - end++ - + if end < len(data) && data[end] == '\n' { + end++ + } if pre := p.quotePrefix(data[beg:]); pre > 0 { // skip the prefix beg += pre } else if p.terminateBlockquote(data, beg, end) { break } - // this line is part of the blockquote raw.Write(data[beg:end]) beg = end } - - var cooked bytes.Buffer - p.block(&cooked, raw.Bytes()) - p.r.BlockQuote(out, cooked.Bytes()) + p.block(raw.Bytes()) + p.finalize(block) return end } // returns prefix length for block code -func (p *parser) codePrefix(data []byte) int { - if data[0] == ' ' && data[1] == ' ' && data[2] == ' ' && data[3] == ' ' { +func (p *Markdown) codePrefix(data []byte) int { + if len(data) >= 1 && data[0] == '\t' { + return 1 + } + if len(data) >= 4 && data[0] == ' ' && data[1] == ' ' && data[2] == ' ' && data[3] == ' ' { return 4 } return 0 } -func (p *parser) code(out *bytes.Buffer, data []byte) int { +func (p *Markdown) code(data []byte) int { var work bytes.Buffer i := 0 for i < len(data) { beg := i - for data[i] != '\n' { + for i < len(data) && data[i] != '\n' { + i++ + } + if i < len(data) && data[i] == '\n' { i++ } - i++ blankline := p.isEmpty(data[beg:i]) > 0 if pre := p.codePrefix(data[beg:i]); pre > 0 { @@ -993,7 +1036,7 @@ func (p *parser) code(out *bytes.Buffer, data []byte) int { break } - // verbatim copy to the working buffeu + // verbatim copy to the working buffer if blankline { work.WriteByte('\n') } else { @@ -1013,122 +1056,183 @@ func (p *parser) code(out *bytes.Buffer, data []byte) int { work.WriteByte('\n') - p.r.BlockCode(out, work.Bytes(), "") + block := p.addBlock(CodeBlock, work.Bytes()) // TODO: get rid of temp buffer + block.IsFenced = false + finalizeCodeBlock(block) return i } // returns unordered list item prefix -func (p *parser) uliPrefix(data []byte) int { +func (p *Markdown) uliPrefix(data []byte) int { i := 0 - // start with up to 3 spaces - for i < 3 && data[i] == ' ' { + for i < len(data) && i < 3 && data[i] == ' ' { i++ } - - // need a *, +, or - followed by a space + if i >= len(data)-1 { + return 0 + } + // need one of {'*', '+', '-'} followed by a space or a tab if (data[i] != '*' && data[i] != '+' && data[i] != '-') || - data[i+1] != ' ' { + (data[i+1] != ' ' && data[i+1] != '\t') { return 0 } return i + 2 } // returns ordered list item prefix -func (p *parser) oliPrefix(data []byte) int { +func (p *Markdown) oliPrefix(data []byte) int { i := 0 // start with up to 3 spaces - for i < 3 && data[i] == ' ' { + for i < 3 && i < len(data) && data[i] == ' ' { i++ } // count the digits start := i - for data[i] >= '0' && data[i] <= '9' { + for i < len(data) && data[i] >= '0' && data[i] <= '9' { i++ } + if start == i || i >= len(data)-1 { + return 0 + } - // we need >= 1 digits followed by a dot and a space - if start == i || data[i] != '.' || data[i+1] != ' ' { + // we need >= 1 digits followed by a dot and a space or a tab + if data[i] != '.' || !(data[i+1] == ' ' || data[i+1] == '\t') { return 0 } return i + 2 } // returns definition list item prefix -func (p *parser) dliPrefix(data []byte) int { +func (p *Markdown) dliPrefix(data []byte) int { + if len(data) < 2 { + return 0 + } i := 0 - - // need a : followed by a spaces - if data[i] != ':' || data[i+1] != ' ' { + // need a ':' followed by a space or a tab + if data[i] != ':' || !(data[i+1] == ' ' || data[i+1] == '\t') { return 0 } - for data[i] == ' ' { + for i < len(data) && data[i] == ' ' { i++ } return i + 2 } // parse ordered or unordered list block -func (p *parser) list(out *bytes.Buffer, data []byte, flags int) int { +func (p *Markdown) list(data []byte, flags ListType) int { i := 0 - flags |= LIST_ITEM_BEGINNING_OF_LIST - work := func() bool { - for i < len(data) { - skip := p.listItem(out, data[i:], &flags) - i += skip + flags |= ListItemBeginningOfList + block := p.addBlock(List, nil) + block.ListFlags = flags + block.Tight = true - if skip == 0 || flags&LIST_ITEM_END_OF_LIST != 0 { - break - } - flags &= ^LIST_ITEM_BEGINNING_OF_LIST + for i < len(data) { + skip := p.listItem(data[i:], &flags) + if flags&ListItemContainsBlock != 0 { + block.ListData.Tight = false } - return true + i += skip + if skip == 0 || flags&ListItemEndOfList != 0 { + break + } + flags &= ^ListItemBeginningOfList } - p.r.List(out, work, flags) + above := block.Parent + finalizeList(block) + p.tip = above return i } +// Returns true if block ends with a blank line, descending if needed +// into lists and sublists. +func endsWithBlankLine(block *Node) bool { + // TODO: figure this out. Always false now. + for block != nil { + //if block.lastLineBlank { + //return true + //} + t := block.Type + if t == List || t == Item { + block = block.LastChild + } else { + break + } + } + return false +} + +func finalizeList(block *Node) { + block.open = false + item := block.FirstChild + for item != nil { + // check for non-final list item ending with blank line: + if endsWithBlankLine(item) && item.Next != nil { + block.ListData.Tight = false + break + } + // recurse into children of list item, to see if there are spaces + // between any of them: + subItem := item.FirstChild + for subItem != nil { + if endsWithBlankLine(subItem) && (item.Next != nil || subItem.Next != nil) { + block.ListData.Tight = false + break + } + subItem = subItem.Next + } + item = item.Next + } +} + // Parse a single list item. // Assumes initial prefix is already removed if this is a sublist. -func (p *parser) listItem(out *bytes.Buffer, data []byte, flags *int) int { +func (p *Markdown) listItem(data []byte, flags *ListType) int { // keep track of the indentation of the first line itemIndent := 0 - for itemIndent < 3 && data[itemIndent] == ' ' { - itemIndent++ + if data[0] == '\t' { + itemIndent += 4 + } else { + for itemIndent < 3 && data[itemIndent] == ' ' { + itemIndent++ + } } + var bulletChar byte = '*' i := p.uliPrefix(data) if i == 0 { i = p.oliPrefix(data) + } else { + bulletChar = data[i-2] } if i == 0 { i = p.dliPrefix(data) // reset definition term flag if i > 0 { - *flags &= ^LIST_TYPE_TERM + *flags &= ^ListTypeTerm } } if i == 0 { - // if in defnition list, set term flag and continue - if *flags&LIST_TYPE_DEFINITION != 0 { - *flags |= LIST_TYPE_TERM + // if in definition list, set term flag and continue + if *flags&ListTypeDefinition != 0 { + *flags |= ListTypeTerm } else { return 0 } } // skip leading whitespace on first line - for data[i] == ' ' { + for i < len(data) && data[i] == ' ' { i++ } // find the end of the line line := i - for i > 0 && data[i-1] != '\n' { + for i > 0 && i < len(data) && data[i-1] != '\n' { i++ } @@ -1148,7 +1252,7 @@ gatherlines: i++ // find the end of this line - for data[i-1] != '\n' { + for i < len(data) && data[i-1] != '\n' { i++ } @@ -1156,18 +1260,24 @@ gatherlines: // and move on to the next line if p.isEmpty(data[line:i]) > 0 { containsBlankLine = true - raw.Write(data[line:i]) line = i continue } // calculate the indentation indent := 0 - for indent < 4 && line+indent < i && data[line+indent] == ' ' { - indent++ + indentIndex := 0 + if data[line] == '\t' { + indentIndex++ + indent += 4 + } else { + for indent < 4 && line+indent < i && data[line+indent] == ' ' { + indent++ + indentIndex++ + } } - chunk := data[line+indent : i] + chunk := data[line+indentIndex : i] // evaluate how this line fits in switch { @@ -1177,15 +1287,7 @@ gatherlines: p.dliPrefix(chunk) > 0: if containsBlankLine { - // end the list if the type changed after a blank line - if indent <= itemIndent && - ((*flags&LIST_TYPE_ORDERED != 0 && p.uliPrefix(chunk) > 0) || - (*flags&LIST_TYPE_ORDERED == 0 && p.oliPrefix(chunk) > 0)) { - - *flags |= LIST_ITEM_END_OF_LIST - break gatherlines - } - *flags |= LIST_ITEM_CONTAINS_BLOCK + *flags |= ListItemContainsBlock } // to be a nested list, it must be indented more @@ -1199,93 +1301,89 @@ gatherlines: sublist = raw.Len() } - // is this a nested prefix header? - case p.isPrefixHeader(chunk): - // if the header is not indented, it is not nested in the list + // is this a nested prefix heading? + case p.isPrefixHeading(chunk): + // if the heading is not indented, it is not nested in the list // and thus ends the list if containsBlankLine && indent < 4 { - *flags |= LIST_ITEM_END_OF_LIST + *flags |= ListItemEndOfList break gatherlines } - *flags |= LIST_ITEM_CONTAINS_BLOCK + *flags |= ListItemContainsBlock // anything following an empty line is only part // of this item if it is indented 4 spaces // (regardless of the indentation of the beginning of the item) case containsBlankLine && indent < 4: - if *flags&LIST_TYPE_DEFINITION != 0 && i < len(data)-1 { + if *flags&ListTypeDefinition != 0 && i < len(data)-1 { // is the next item still a part of this list? next := i - for data[next] != '\n' { + for next < len(data) && data[next] != '\n' { next++ } for next < len(data)-1 && data[next] == '\n' { next++ } if i < len(data)-1 && data[i] != ':' && data[next] != ':' { - *flags |= LIST_ITEM_END_OF_LIST + *flags |= ListItemEndOfList } } else { - *flags |= LIST_ITEM_END_OF_LIST + *flags |= ListItemEndOfList } break gatherlines // a blank line means this should be parsed as a block case containsBlankLine: - *flags |= LIST_ITEM_CONTAINS_BLOCK + raw.WriteByte('\n') + *flags |= ListItemContainsBlock } - containsBlankLine = false + // if this line was preceded by one or more blanks, + // re-introduce the blank into the buffer + if containsBlankLine { + containsBlankLine = false + raw.WriteByte('\n') + } // add the line into the working buffer without prefix - raw.Write(data[line+indent : i]) + raw.Write(data[line+indentIndex : i]) line = i } - // If reached end of data, the Renderer.ListItem call we're going to make below - // is definitely the last in the list. - if line >= len(data) { - *flags |= LIST_ITEM_END_OF_LIST - } - rawBytes := raw.Bytes() + block := p.addBlock(Item, nil) + block.ListFlags = *flags + block.Tight = false + block.BulletChar = bulletChar + block.Delimiter = '.' // Only '.' is possible in Markdown, but ')' will also be possible in CommonMark + // render the contents of the list item - var cooked bytes.Buffer - if *flags&LIST_ITEM_CONTAINS_BLOCK != 0 && *flags&LIST_TYPE_TERM == 0 { + if *flags&ListItemContainsBlock != 0 && *flags&ListTypeTerm == 0 { // intermediate render of block item, except for definition term if sublist > 0 { - p.block(&cooked, rawBytes[:sublist]) - p.block(&cooked, rawBytes[sublist:]) + p.block(rawBytes[:sublist]) + p.block(rawBytes[sublist:]) } else { - p.block(&cooked, rawBytes) + p.block(rawBytes) } } else { // intermediate render of inline item if sublist > 0 { - p.inline(&cooked, rawBytes[:sublist]) - p.block(&cooked, rawBytes[sublist:]) + child := p.addChild(Paragraph, 0) + child.content = rawBytes[:sublist] + p.block(rawBytes[sublist:]) } else { - p.inline(&cooked, rawBytes) + child := p.addChild(Paragraph, 0) + child.content = rawBytes } } - - // render the actual list item - cookedBytes := cooked.Bytes() - parsedEnd := len(cookedBytes) - - // strip trailing newlines - for parsedEnd > 0 && cookedBytes[parsedEnd-1] == '\n' { - parsedEnd-- - } - p.r.ListItem(out, cookedBytes[:parsedEnd], *flags) - return line } // render a single paragraph that has already been parsed out -func (p *parser) renderParagraph(out *bytes.Buffer, data []byte) { +func (p *Markdown) renderParagraph(data []byte) { if len(data) == 0 { return } @@ -1296,27 +1394,29 @@ func (p *parser) renderParagraph(out *bytes.Buffer, data []byte) { beg++ } + end := len(data) // trim trailing newline - end := len(data) - 1 + if data[len(data)-1] == '\n' { + end-- + } // trim trailing spaces for end > beg && data[end-1] == ' ' { end-- } - work := func() bool { - p.inline(out, data[beg:end]) - return true - } - p.r.Paragraph(out, work) + p.addBlock(Paragraph, data[beg:end]) } -func (p *parser) paragraph(out *bytes.Buffer, data []byte) int { +func (p *Markdown) paragraph(data []byte) int { // prev: index of 1st char of previous line // line: index of 1st char of current line // i: index of cursor/end of current line var prev, line, i int - + tabSize := TabSizeDefault + if p.extensions&TabSizeEight != 0 { + tabSize = TabSizeDouble + } // keep going until we find something to mark the end of the paragraph for i < len(data) { // mark the beginning of the current line @@ -1324,24 +1424,32 @@ func (p *parser) paragraph(out *bytes.Buffer, data []byte) int { current := data[i:] line = i + // did we find a reference or a footnote? If so, end a paragraph + // preceding it and report that we have consumed up to the end of that + // reference: + if refEnd := isReference(p, current, tabSize); refEnd > 0 { + p.renderParagraph(data[:i]) + return i + refEnd + } + // did we find a blank line marking the end of the paragraph? if n := p.isEmpty(current); n > 0 { // did this blank line followed by a definition list item? - if p.flags&EXTENSION_DEFINITION_LISTS != 0 { + if p.extensions&DefinitionLists != 0 { if i < len(data)-1 && data[i+1] == ':' { - return p.list(out, data[prev:], LIST_TYPE_DEFINITION) + return p.list(data[prev:], ListTypeDefinition) } } - p.renderParagraph(out, data[:i]) + p.renderParagraph(data[:i]) return i + n } - // an underline under some text marks a header, so our paragraph ended on prev line + // an underline under some text marks a heading, so our paragraph ended on prev line if i > 0 { - if level := p.isUnderlinedHeader(current); level > 0 { + if level := p.isUnderlinedHeading(current); level > 0 { // render the paragraph - p.renderParagraph(out, data[:prev]) + p.renderParagraph(data[:prev]) // ignore leading and trailing whitespace eol := i - 1 @@ -1352,24 +1460,17 @@ func (p *parser) paragraph(out *bytes.Buffer, data []byte) int { eol-- } - // render the header - // this ugly double closure avoids forcing variables onto the heap - work := func(o *bytes.Buffer, pp *parser, d []byte) func() bool { - return func() bool { - pp.inline(o, d) - return true - } - }(out, p, data[prev:eol]) - id := "" - if p.flags&EXTENSION_AUTO_HEADER_IDS != 0 { - id = SanitizedAnchorName(string(data[prev:eol])) + if p.extensions&AutoHeadingIDs != 0 { + id = sanitized_anchor_name.Create(string(data[prev:eol])) } - p.r.Header(out, work, level, id) + block := p.addBlock(Heading, data[prev:eol]) + block.Level = level + block.HeadingID = id // find the end of the underline - for data[i] != '\n' { + for i < len(data) && data[i] != '\n' { i++ } return i @@ -1377,74 +1478,72 @@ func (p *parser) paragraph(out *bytes.Buffer, data []byte) int { } // if the next line starts a block of HTML, then the paragraph ends here - if p.flags&EXTENSION_LAX_HTML_BLOCKS != 0 { - if data[i] == '<' && p.html(out, current, false) > 0 { + if p.extensions&LaxHTMLBlocks != 0 { + if data[i] == '<' && p.html(current, false) > 0 { // rewind to before the HTML block - p.renderParagraph(out, data[:i]) + p.renderParagraph(data[:i]) return i } } - // if there's a prefixed header or a horizontal rule after this, paragraph is over - if p.isPrefixHeader(current) || p.isHRule(current) { - p.renderParagraph(out, data[:i]) + // if there's a prefixed heading or a horizontal rule after this, paragraph is over + if p.isPrefixHeading(current) || p.isHRule(current) { + p.renderParagraph(data[:i]) return i } // if there's a fenced code block, paragraph is over - if p.flags&EXTENSION_FENCED_CODE != 0 { - if p.fencedCodeBlock(out, current, false) > 0 { - p.renderParagraph(out, data[:i]) + if p.extensions&FencedCode != 0 { + if p.fencedCodeBlock(current, false) > 0 { + p.renderParagraph(data[:i]) return i } } // if there's a definition list item, prev line is a definition term - if p.flags&EXTENSION_DEFINITION_LISTS != 0 { + if p.extensions&DefinitionLists != 0 { if p.dliPrefix(current) != 0 { - return p.list(out, data[prev:], LIST_TYPE_DEFINITION) + ret := p.list(data[prev:], ListTypeDefinition) + return ret } } // if there's a list after this, paragraph is over - if p.flags&EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK != 0 { + if p.extensions&NoEmptyLineBeforeBlock != 0 { if p.uliPrefix(current) != 0 || p.oliPrefix(current) != 0 || p.quotePrefix(current) != 0 || p.codePrefix(current) != 0 { - p.renderParagraph(out, data[:i]) + p.renderParagraph(data[:i]) return i } } // otherwise, scan to the beginning of the next line - for data[i] != '\n' { - i++ + nl := bytes.IndexByte(data[i:], '\n') + if nl >= 0 { + i += nl + 1 + } else { + i += len(data[i:]) } - i++ } - p.renderParagraph(out, data[:i]) + p.renderParagraph(data[:i]) return i } -// SanitizedAnchorName returns a sanitized anchor name for the given text. -// -// It implements the algorithm specified in the package comment. -func SanitizedAnchorName(text string) string { - var anchorName []rune - futureDash := false - for _, r := range text { - switch { - case unicode.IsLetter(r) || unicode.IsNumber(r): - if futureDash && len(anchorName) > 0 { - anchorName = append(anchorName, '-') - } - futureDash = false - anchorName = append(anchorName, unicode.ToLower(r)) - default: - futureDash = true - } +func skipChar(data []byte, start int, char byte) int { + i := start + for i < len(data) && data[i] == char { + i++ + } + return i +} + +func skipUntilChar(text []byte, start int, cha