WikiNote3/wikinote/markup.py

#   WikiNote3
#   Copyright © 2020  Lee Yingtong Li (RunasSudo)
#
#   This program is free software: you can redistribute it and/or modify
#   it under the terms of the GNU Affero General Public License as published by
#   the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU Affero General Public License for more details.
#
#   You should have received a copy of the GNU Affero General Public License
#   along with this program.  If not, see <https://www.gnu.org/licenses/>.

import markdown
import markdown.extensions.extra, markdown.extensions.footnotes, markdown.extensions.attr_list

import re
import xml.etree.ElementTree as ET

from .mdx_urlize import UrlizeExtension

directives = {}
roles = {}

class WNMarkdown(markdown.Markdown):
	def __init__(self, *args, **kwargs):
		super().__init__(*args, **kwargs)
		
		self.registerExtensions([FootnoteExtension(), UrlizeExtension(), 'toc', 'tables'], {})
		
		self.meta = {}
		
		# Markdown in HTML
		self.preprocessors['html_block'].markdown_in_raw = True
		self.parser.blockprocessors.register(markdown.extensions.extra.MarkdownInHtmlProcessor(self.parser), 'markdown_block', 105)
		self.parser.blockprocessors.tag_counter = -1
		self.parser.blockprocessors.contain_span_tags = re.compile(r'^(p|h[1-6]|li|dd|dt|td|th|legend|address)$', re.IGNORECASE)
		
		# Override default Markdown processors
		self.preprocessors.register(NormalizeWhitespace(self), 'normalize_whitespace', 30)
		self.parser.blockprocessors.register(HashHeaderProcessor(self.parser), 'hashheader', 70)
		self.treeprocessors.register(AttrListTreeprocessor(self), 'attr_list', 8)
		
		# Our own processors
		self.parser.blockprocessors.register(DirectiveProcessor(self.parser), 'directive', 95)
		self.parser.blockprocessors.register(AdmonitionProcessor(self.parser), 'admonition', 105)
		self.inlinePatterns.register(BlueProcessor(self.parser), 'blue_em', 65)
		self.inlinePatterns.register(RoleProcessor(self.parser), 'role', 500)
		self.treeprocessors.register(WrapSectionProcessor(self), 'wrap_sections', 100)
	
	# Override
	def reset(self):
		super().reset()
		self.meta = {}
		return self
	
	# Based on Markdown.convert
	def parse(self, source):
		if not source.strip():
			return ''
		self.lines = source.split('\n')
		for prep in self.preprocessors:
			self.lines = prep.run(self.lines)
		
		root = self.parser.parseDocument(self.lines).getroot()
		
		for treeprocessor in self.treeprocessors:
			newRoot = treeprocessor.run(root)
			if newRoot is not None:
				root = newRoot
		
		return root
	
	# Based on Markdown.convert
	def serialise(self, root):
		output = self.serializer(root)
		
		if self.stripTopLevelTags:
			try:
				start = output.index('<{}>'.format(self.doc_tag)) + len(self.doc_tag) + 2
				end = output.rindex('</{}>'.format(self.doc_tag))
				output = output[start:end].strip()
			except ValueError:
				if output.strip().endswith('<{} />'.format(self.doc_tag)):
					output = ''
		
		for pp in self.postprocessors:
			output = pp.run(output)
		
		return output.strip()
	
	# Put it together
	def convert(self, source):
		root = self.parse(source)
		return self.serialise(root)
	
	def detab(self, text):
		newtext = []
		lines = text.split('\n')
		for line in lines:
			if line.startswith(' '*self.tab_length):
				newtext.append(line[self.tab_length:])
			elif line.startswith('\t'):
				newtext.append(line[1:])
			elif not line.strip():
				newtext.append('')
			else:
				break
		return '\n'.join(newtext), '\n'.join(lines[len(newtext):])

class HashHeaderProcessor(markdown.blockprocessors.HashHeaderProcessor):
	# Override to add 1 to level
	def run(self, parent, blocks):
		block = blocks.pop(0)
		m = self.RE.search(block)
		before = block[:m.start()]
		after = block[m.end():]
		if before:
			self.parser.parseBlocks(parent, [before])
		h = ET.SubElement(parent, 'h{}'.format(len(m.group('level')) + 1)) # Add 1 to level
		h.text = m.group('header').strip()
		if after:
			blocks.insert(0, after)

class NormalizeWhitespace(markdown.preprocessors.Preprocessor):
	# Override to retain tabs
	def run(self, lines):
		source = '\n'.join(lines)
		source = source.replace(markdown.util.STX, "").replace(markdown.util.ETX, "")
		source = source.replace("\r\n", "\n").replace("\r", "\n") + "\n\n"
		source = re.sub(r'(?<=\n) +\n', '\n', source)
		return source.split('\n')

class DirectiveProcessor(markdown.blockprocessors.BlockProcessor):
	RE = re.compile(r'^.. +(?P<name>[a-zA-Z0-9_-]+?)::(?: +(?P<arg>.*?))?(?:\n|$)')
	
	def test(self, parent, block):
		return bool(self.RE.search(block))
	
	def run(self, parent, blocks):
		block = blocks.pop(0)
		m = self.RE.search(block)
		
		# Get directive content
		if '\n' in block:
			content = block[block.index('\n') + 1:]
		else:
			content = ''
		for b in blocks[:]:
			if b.startswith('\t'):
				blocks.pop(0)
				content += b
		
		content, theRest = self.parser.md.detab(content)
		
		directive = directives[m.group('name')](self.parser.md, arg=m.group('arg') or '', content=content)
		el = directive.render()
		el.directive = directive
		parent.append(el)
		
		if theRest:
			blocks.insert(0, theRest)

class RoleProcessor(markdown.inlinepatterns.InlineProcessor):
	def __init__(self, md):
		super().__init__(r':(?P<name>[^:]+?):`(?P<content>[^`]+?)`', md)
	
	def handleMatch(self, m, data):
		role = roles[m.group('name')](self.md, m.group('content'))
		el = role.render()
		el.role = role
		return el, m.start(0), m.end(0)

class BlueProcessor(markdown.inlinepatterns.InlineProcessor):
	def __init__(self, md):
		super().__init__(r'!!(.+?)!!', md)
	
	def handleMatch(self, m, data):
		el = ET.Element('span')
		el.text = m.group(1)
		el.set('class', 'blue')
		return el, m.start(0), m.end(0)

class WrapSectionProcessor(markdown.treeprocessors.Treeprocessor):
	def run(self, root):
		section = ET.Element('section')
		
		for child in list(root):
			if child.tag in ('h1', 'h2', 'h3'):
				if len(section) > 0:
					root.insert(list(root).index(child), section)
					section = ET.Element('section')
			else:
				section.append(child)
				root.remove(child)
		
		if len(section) > 0:
			root.append(section)

# Adapted from Python-Markdown
# Allow tabs
class AdmonitionProcessor(markdown.blockprocessors.BlockProcessor):
	CLASSNAME = 'admonition'
	CLASSNAME_TITLE = 'admonition-title'
	RE = re.compile(r'(?:^|\n)!!! ?([\w\-]+(?: +[\w\-]+)*)(?: +"(.*?)")? *(?:\n|$)')
	RE_SPACES = re.compile('  +|\t+')
	
	def test(self, parent, block):
		sibling = self.lastChild(parent)
		return self.RE.search(block) or \
			((block.startswith(' ' * self.tab_length) or block.startswith('\t')) and sibling is not None and
			sibling.get('class', '').find(self.CLASSNAME) != -1)
	
	def run(self, parent, blocks):
		sibling = self.lastChild(parent)
		block = blocks.pop(0)
		m = self.RE.search(block)
		
		if m:
			block = block[m.end():]  # removes the first line
		
		block, theRest = self.parser.md.detab(block)
		
		if m:
			klass, title = self.get_class_and_title(m)
			div = ET.SubElement(parent, 'div')
			div.set('class', '{} {}'.format(self.CLASSNAME, klass))
			if title:
				p = ET.SubElement(div, 'p')
				p.text = title
				p.set('class', self.CLASSNAME_TITLE)
		else:
			div = sibling
		
		self.parser.parseChunk(div, block)
		
		if theRest:
			# This block contained unindented line(s) after the first indented
			# line. Insert these lines as the first block of the master blocks
			# list for future processing.
			blocks.insert(0, theRest)
	
	def get_class_and_title(self, match):
		klass, title = match.group(1).lower(), match.group(2)
		klass = self.RE_SPACES.sub(' ', klass)
		if title is None:
			# no title was provided, use the capitalized classname as title
			# e.g.: `!!! note` will render
			# `<p class="admonition-title">Note</p>`
			title = klass.split(' ', 1)[0].capitalize()
		elif title == '':
			# an explicit blank title should not be rendered
			# e.g.: `!!! warning ""` will *not* render `p` with a title
			title = None
		return klass, title

# Adapted from Python-Markdown
# Fix for tables
class AttrListTreeprocessor(markdown.treeprocessors.Treeprocessor):
	BASE_RE = r'\{\:?([^\}\n]*)\}'
	HEADER_RE = re.compile(r'[ ]+%s[ ]*$' % BASE_RE)
	BLOCK_RE = re.compile(r'\n[ ]*%s[ ]*$' % BASE_RE)
	INLINE_RE = re.compile(r'^%s' % BASE_RE)
	NAME_RE = re.compile(r'[^A-Z_a-z\u00c0-\u00d6\u00d8-\u00f6\u00f8-\u02ff'
						r'\u0370-\u037d\u037f-\u1fff\u200c-\u200d'
						r'\u2070-\u218f\u2c00-\u2fef\u3001-\ud7ff'
						r'\uf900-\ufdcf\ufdf0-\ufffd'
						r'\:\-\.0-9\u00b7\u0300-\u036f\u203f-\u2040]+')
	
	def run(self, doc):
		for elem in doc.iter():
			if self.md.is_block_level(elem.tag):
				# Block level: check for attrs on last line of text
				RE = self.BLOCK_RE
				if markdown.extensions.attr_list.isheader(elem) or elem.tag == 'dt':
					# header or def-term: check for attrs at end of line
					RE = self.HEADER_RE
				if len(elem) and elem.tag == 'li':
					# special case list items. children may include a ul or ol.
					pos = None
					# find the ul or ol position
					for i, child in enumerate(elem):
						if child.tag in ['ul', 'ol']:
							pos = i
							break
					if pos is None and elem[-1].tail:
						# use tail of last child. no ul or ol.
						m = RE.search(elem[-1].tail)
						if m:
							self.assign_attrs(elem, m.group(1))
							elem[-1].tail = elem[-1].tail[:m.start()]
					elif pos is not None and pos > 0 and elem[pos-1].tail:
						# use tail of last child before ul or ol
						m = RE.search(elem[pos-1].tail)
						if m:
							self.assign_attrs(elem, m.group(1))
							elem[pos-1].tail = elem[pos-1].tail[:m.start()]
					elif elem.text:
						# use text. ul is first child.
						m = RE.search(elem.text)
						if m:
							self.assign_attrs(elem, m.group(1))
							elem.text = elem.text[:m.start()]
				elif len(elem) and elem.tag == 'table' and len(elem[-1]) and len(elem[-1][-1]) and elem[-1][-1][0].text:
					# SPECIAL CASE table, use last row
					RE = self.INLINE_RE
					m = RE.search(elem[-1][-1][0].text) # tbody -> tr -> td
					if m:
						self.assign_attrs(elem, m.group(1))
						# Remove last row
						elem[-1].remove(elem[-1][-1]) # tbody -> tr
				elif len(elem) and elem[-1].tail:
					# has children. Get from tail of last child
					m = RE.search(elem[-1].tail)
					if m:
						self.assign_attrs(elem, m.group(1))
						elem[-1].tail = elem[-1].tail[:m.start()]
						if markdown.extensions.attr_list.isheader(elem):
							# clean up trailing #s
							elem[-1].tail = elem[-1].tail.rstrip('#').rstrip()
				elif elem.text:
					# no children. Get from text.
					m = RE.search(elem.text)
					if not m and elem.tag == 'td':
						m = re.search(self.BASE_RE, elem.text)
					if m:
						self.assign_attrs(elem, m.group(1))
						elem.text = elem.text[:m.start()]
						if markdown.extensions.attr_list.isheader(elem):
							# clean up trailing #s
							elem.text = elem.text.rstrip('#').rstrip()
			else:
				# inline: check for attrs at start of tail
				if elem.tail:
					m = self.INLINE_RE.match(elem.tail)
					if m:
						self.assign_attrs(elem, m.group(1))
						elem.tail = elem.tail[m.end():]
	
	def assign_attrs(self, elem, attrs):
		""" Assign attrs to element. """
		for k, v in markdown.extensions.attr_list.get_attrs(attrs):
			if k == '.':
				# add to class
				cls = elem.get('class')
				if cls:
					elem.set('class', '{} {}'.format(cls, v))
				else:
					elem.set('class', v)
			else:
				# assign attr k with v
				elem.set(self.sanitize_name(k), v)
	
	def sanitize_name(self, name):
		"""
		Sanitize name as 'an XML Name, minus the ":"'.
		See https://www.w3.org/TR/REC-xml-names/#NT-NCName
		"""
		return self.NAME_RE.sub('_', name)

# Footnotes

class FootnoteExtension(markdown.extensions.footnotes.FootnoteExtension):
	# Override
	def extendMarkdown(self, md):
		md.registerExtension(self)
		self.parser = md.parser
		self.md = md
		md.preprocessors.register(markdown.extensions.footnotes.FootnotePreprocessor(self), 'footnote', 15)
		FOOTNOTE_RE = r'\[\^([^\]]*)\]'  # blah blah [^1] blah
		md.inlinePatterns.register(FootnoteInlineProcessor(FOOTNOTE_RE, self), 'footnote', 175)
		md.treeprocessors.register(markdown.extensions.footnotes.FootnoteTreeprocessor(self), 'footnote', 50)
	
	# Override to omit backlinks
	def makeFootnotesDiv(self, root):
		if not list(self.footnotes.keys()):
			return None
		
		div = ET.Element("div")
		div.set('class', 'footnote')
		ol = ET.SubElement(div, "ol")
		surrogate_parent = ET.Element("div")
		
		for index, id in enumerate(self.footnotes.keys(), start=1):
			li = ET.SubElement(ol, "li")
			li.set("id", self.makeFootnoteId(id))
			self.parser.parseChunk(surrogate_parent, self.footnotes[id])
			for el in list(surrogate_parent):
				li.append(el)
				surrogate_parent.remove(el)
		return div

class FootnoteInlineProcessor(markdown.extensions.footnotes.FootnoteInlineProcessor):
	# Override to handle commas
	def handleMatch(self, m, data):
		id = m.group(1).rstrip(',')
		if id in self.footnotes.footnotes.keys():
			sup = ET.Element("sup")
			sup.set('class', 'footnote-ref')
			a = ET.SubElement(sup, "a")
			sup.set('id', self.footnotes.makeFootnoteRefId(id, found=True))
			a.set('href', '#' + self.footnotes.makeFootnoteId(id))
			a.text = str(list(self.footnotes.footnotes.keys()).index(id) + 1)
			if m.group(1).endswith(','):
				a.tail = ','
			return sup, m.start(0), m.end(0)
		else:
			return None, None, None

# Custom directives and roles
from . import markup_custom
directives.update(markup_custom.directives)
roles.update(markup_custom.roles)

try:
	from . import markup_custom2
	directives.update(markup_custom2.directives)
	roles.update(markup_custom2.roles)
except ImportError:
	pass