cortav  Diff

--  ~ lexi hale <lexi@hale.su>
--  © AGPLv3
--  ? reference implementation of the cortav document language

local ss = require 'sirsem'
-- aliases for commonly used sirsem funcs
local startswith = ss.str.begins
local eachcode = ss.str.enc.utf8.each
local dump = ss.dump
local declare = ss.declare

-- make this module available to require() when linked into a lua bytecode program with luac
local ct = ss.namespace 'cortav'
ct.info = {
	version = ss.version {0,1; 'devel'};
................................................................................
	end);
	cli = ss.exnkind 'command line parse error';
	mode = ss.exnkind('bad mode', function(msg, ...)
		return string.format("mode “%s” "..msg, ...)
	end);
	unimpl = ss.exnkind 'feature not implemented';
	ext = ss.exnkind 'extension error';



}

ct.ctx = declare {
	mk = function(src) return {src = src} end;
	ident = 'context';
	cast = {
		string = function(me)
................................................................................
			table.insert(self.sec.blocks,block)
			return block
		end;
		ref = function(self,id)
			if not id:find'%.' then
				local rid = self.sec.refs[id]
				if self.sec.refs[id] then
					return self.sec.refs[id]
				else self:fail("no such ref %s in current section", id or '') end
			else
				local sec, ref = string.match(id, "(.-)%.(.+)")
				local s = self.doc.sections[sec]
				if s then
					if s.refs[ref] then
						return s.refs[ref]
					else self:fail("no such ref %s in section %s", ref, sec) end
				else self:fail("no such section %s", sec) end
			end
		end
	};
}

................................................................................
		meta = {};
		vars = {};
		ext = {
			inhibit = {};
			need = {};
			use = {};
		};

	} end;
	construct = function(me)
		me.docjob = ct.ext.job('doc', me, nil)
	end;
}

-- FP helper functions
................................................................................

-- renderer engines
function ct.render.html(doc, opts)
	local doctitle = opts['title']
	local f = string.format
	local ids = {}
	local canonicalID = {}
	local function getSafeID(obj)

		if canonicalID[obj] then
			return canonicalID[obj]
		elseif obj.id and ids[obj.id] then

			local newid
			local i = 1
			repeat newid = obj.id .. string.format('-%x', i)
				i = i + 1 until not ids[newid]
			ids[newid] = obj
			canonicalID[obj] = newid
			return newid
		else
			local cid = obj.id
			if not cid then
				local i = 1
				repeat cid = string.format('x-%x', i)
					i = i + 1 until not ids[cid]
			end
			ids[cid] = obj
			canonicalID[obj] = cid
			return cid
		end
	end




	local langsused = {}
	local langpairs = {
		lua = { color = 0x9377ff };
		terra = { color = 0xff77c8 };
		c = { name = 'C', color = 0x77ffe8 };
		html = { color = 0xfff877 };
		scheme = { color = 0x77ff88 };
		lisp = { color = 0x77ff88 };
		fortran = { color = 0xff779a };
		python = { color = 0xffd277 };
		python = { color = 0xcdd6ff };
	}

	local stylesets = {









































































		header = [[

			h1,h2,h3,h4,h5,h6 { border-bottom: 1px solid @tone(0.7); }
			h1 { font-size: 200%; border-bottom-style: double !important; border-bottom-width: 3px !important; margin: 0em -1em; }
			h2 { font-size: 130%; margin: 0em -0.7em; }
			h3 { font-size: 110%; margin: 0em -0.5em; }
			h4 { font-size: 100%; font-weight: normal; margin: 0em -0.2em; }
			h5 { font-size: 90%; font-weight: normal; }
			h6 { font-size: 80%; font-weight: normal; }
................................................................................
			section:target > :is(h1,h2,h3,h4,h5,h6) {

			}
		]];
		paragraph = [[
			p {
				margin: 0.7em 0;

			}
			section {
				margin: 1.2em 0;
			}
			section:first-child { margin-top: 0; }
		]];
		accent = [[

			body { background: @bg; color: @fg }
			a[href] {
				color: @tone(0.7 30);
				text-decoration-color: @tone/0.4(0.7 30);
			}
			a[href]:hover {
				color: @tone(0.9 30);
				text-decoration-color: @tone/0.7(0.7 30);
			}
			h1 { color: @tone(2); }
			h2 { color: @tone(1.5); }
			h3 { color: @tone(1.2); }
			h4 { color: @tone(1); }
			h5,h6 { color: @tone(0.8); }











		]];


















		code = [[
			code {

				background: @fg;
				color: @bg;
				font-family: monospace;
				font-size: 90%;
				padding: 3px 5px;
			}



















		]];
		abbr = [[
			abbr[title] { cursor: help; }
		]];
		editors_markup = [[]];
		block_code_listing = [[
			section > figure.listing {
				font-family: monospace;
				background: @tone(0.05);
				color: @fg;
				padding: 0;
				margin: 0.3em 0;
				counter-reset: line-number;
				position: relative;
				border: 1px solid @fg;
			}
			section > figure.listing>div {
				white-space: pre-wrap;


				counter-increment: line-number;
				text-indent: -2.3em;
				margin-left: 2.3em;
			}
			section > figure.listing>:is(div,hr)::before {
				width: 1.0em;
				padding: 0.2em 0.4em;
				text-align: right;
				display: inline-block;
				background-color: @tone(0.2);
				border-right: 1px solid @fg;
				content: counter(line-number);
				margin-right: 0.3em;
			}
			section > figure.listing>hr::before {
				color: transparent;
				padding-top: 0;
				padding-bottom: 0;
			}
			section > figure.listing>div::before {
				color: @fg;
			}
			section > figure.listing>div:last-child::before {
				padding-bottom: 0.5em;
			}
			section > figure.listing>figcaption:first-child {
				border: none;
				border-bottom: 1px solid @fg;
			}
			section > figure.listing>figcaption::after {
				display: block;
				float: right;
				font-weight: normal;
				font-style: italic;
				font-size: 70%;
				padding-top: 0.3em;
			}
			section > figure.listing>figcaption {
				font-family: sans-serif;
				font-size: 120%;
				padding: 0.2em 0.4em;
				border: none;
				color: @tone(2);
			}
			section > figure.listing > hr {
				border: none;
				margin: 0;
				height: 0.7em;
				counter-increment: line-number;
			}
		]];
	}
................................................................................
		stylesets = stylesets;
		stylesets_active = stylesNeeded;
		obj_htmlid = getSafeID;
		-- remaining fields added later
	}

	local renderJob = doc:job('render_html', nil, render_state_handle)


	local runhook = function(h, ...)
		return renderJob:hook(h, render_state_handle, ...)
	end









































	local function getSpanRenderers(procs)
		local tag, elt, catenate = procs.tag, procs.elt, procs.catenate
		local htmlDoc = function(title, head, body)
			return [[<!doctype html>]] .. tag('html',nil,
				tag('head', nil,
					elt('meta',{charset = 'utf-8'}) ..
					(title and tag('title', nil, title) or '') ..
					(head or '')) ..
				tag('body', nil, body or ''))
		end

		local span_renderers = {}
		local function htmlSpan(spans, block, sec)
			local text = {}
			for k,v in pairs(spans) do
				if type(v) == 'string' then
					table.insert(text,(v:gsub('[<>&"]',
						function(x)
							return string.format('&#%02u;', string.byte(x))
						end)))




				else
					table.insert(text, span_renderers[v.kind](v, block, sec))
				end
			end
			return table.concat(text)
		end
















		function span_renderers.format(sp,...)
			local tags = { strong = 'strong', emph = 'em', strike = 'del', insert = 'ins', literal = 'code' }
			if sp.style == 'literal' and not opts['fossil-uv'] then
				stylesNeeded.code = true
			end
			if sp.style == 'del' or sp.style == 'ins' then
				stylesNeeded.editors_markup = true


			end
			return tag(tags[sp.style],nil,htmlSpan(sp.spans,...))
		end

		function span_renderers.term(t,b,s)
			local r = b.origin:ref(t.ref)
			local name = t.ref
			if name:find'%.' then name = name:match '^[^.]*%.(.+)$' end
			if type(r) ~= 'string' then
				b.origin:fail('%s is an object, not a reference', t.ref)
			end
			stylesNeeded.abbr = true
			return tag('abbr',{title=r},next(t.spans) and htmlSpan(t.spans,b,s) or name)
		end







		function span_renderers.macro(m,b,s)
			local r = b.origin:ref(m.macro)
			if type(r) ~= 'string' then
				b.origin:fail('%s is an object, not a reference', t.ref)




			end
			local mctx = b.origin:clone()
			mctx.invocation = m
			return htmlSpan(ct.parse_span(r, mctx),b,s)










		end

		function span_renderers.var(v,b,s)
			local val
			if v.pos then
				if not v.origin.invocation then
					v.origin:fail 'positional arguments can only be used in a macro invocation'
................................................................................
			end
			if v.raw then
				return val
			else
				return htmlSpan(ct.parse_span(val, v.origin), b, s)
			end
		end





		function span_renderers.link(sp,b,s)
			local href
			if b.origin.doc.sections[sp.ref] then
				href = '#' .. sp.ref
			else
				if sp.addr then href = sp.addr else
................................................................................
					if type(r) == 'table' then
						href = '#' .. getSafeID(r)
					else href = r end
				end
			end
			return tag('a',{href=href},next(sp.spans) and htmlSpan(sp.spans,b,s) or href)
		end


		return {
			span_renderers = span_renderers;
			htmlSpan = htmlSpan;
			htmlDoc = htmlDoc;

		}






	end





































	local function getBlockRenderers(procs, sr)
		local tag, elt, catenate = procs.tag, procs.elt, procs.catenate
		local null = function() return catenate{} end

		local block_renderers = {
			anchor = function(b,s)
................................................................................
					if #l > 0 then
						return tag('div',nil,sr.htmlSpan(l, b, s))
					else
						return elt('hr')
					end
				end, b.lines)
				if b.title then
					table.insert(nodes,1,tag('figcaption',nil,sr.htmlSpan(b.title)))
				end
				if b.lang then langsused[b.lang] = true end
				return tag('figure', {class='listing', lang=b.lang, id=b.id and getSafeID(b)}, catenate(nodes))
			end;
			aside = function(b,s)
				local bn = {}




				for _,v in pairs(b.lines) do
					table.insert(bn, tag('p', {}, sr.htmlSpan(v, b, s)))

				end
				return tag('aside', {}, bn)
			end;
			['break'] = function() --[[nop]] end;



		}
		return block_renderers;
	end

	local function getRenderers(procs)
		local r = getSpanRenderers(procs)

		r.block_renderers = getBlockRenderers(procs, r)
		return r
	end

	local tagproc do
		local elt = function(t,attrs)
			return f('<%s%s>', t,
				attrs and ss.reduce(function(a,b) return a..b end, '', 
					ss.map(function(v,k)
						if v == true
							then          return ' '..k
							elseif v then return f(' %s="%s"', k, v)
						end
					end, attrs)) or '')
		end

		tagproc = {
			toTXT = {
				tag = function(t,a,v) return v  end;
				elt = function(t,a)   return '' end;
				catenate = table.concat;
			};
			toIR = {
				tag = function(t,a,v,o) return {
					tag = t, attrs = a;
					nodes = type(v) == 'string' and {v} or v, src = o
				} end;
				
				elt = function(t,a,o) return {
					tag = t, attrs = a, src = o
				} end;

				catenate = function(...) return ... end;
			};
			toHTML = {
				elt = elt;
				tag = function(t,attrs,body)
					return f('%s%s</%s>', elt(t,attrs), body, t)
				end;
				catenate = table.concat;
			};
		}
	end

	local astproc = {
		toHTML = getRenderers(tagproc.toHTML);
		toTXT  = getRenderers(tagproc.toTXT);
		toIR   = { };
	}
................................................................................
	local ir = {}
	local dr = astproc.toHTML -- default renderers
	local plainr = astproc.toTXT
	local irBlockRdrs = astproc.toIR.block_renderers;

	render_state_handle.ir = ir;



































	runhook('ir_assemble', ir)
	for i, sec in ipairs(doc.secorder) do
		if doctitle == nil and sec.depth == 1 and sec.heading_node then
			doctitle = astproc.toTXT.htmlSpan(sec.heading_node.spans, sec.heading_node, sec)
		end
		local irs
		if sec.kind == 'ordinary' then
			if #(sec.blocks) > 0 then
				irs = {tag='section',attrs={id = getSafeID(sec)},nodes={}}

				runhook('ir_section_build', irs, sec)
				
				for i, block in ipairs(sec.blocks) do
					local rd
					if irBlockRdrs[block.kind] then
						rd = irBlockRdrs[block.kind](block,sec)
					else
						local rdr = renderJob:proc('render',block.kind,'html')
						if rdr then
							rd = rdr({
								state = render_state_handle;
								tagproc = tagproc.toIR;
								astproc = astproc.toIR;
							}, block, sec)
						end
					end
					if rd then
						if opts['heading-anchors'] and block == sec.heading_node then
							stylesNeeded.headingAnchors = true
							table.insert(rd.nodes, ' ')
							table.insert(rd.nodes, {
								tag = 'a';
								attrs = {href = '#' .. irs.attrs.id, class='anchor'};
								nodes = {type(opts['heading-anchors'])=='string' and opts['heading-anchors'] or '&sect;'};
							})
						end
						table.insert(irs.nodes, rd)
						runhook('ir_section_node_insert', rd, irs, sec)
					end
				end
			end
		elseif sec.kind == 'blockquote' then
		elseif sec.kind == 'listing' then
		elseif sec.kind == 'embed' then
		end
		if irs then table.insert(ir, irs) end
	end

















	-- restructure passes
	runhook('ir_restructure_pre', ir)
	
	---- list insertion pass
	local lists = {}
	for _, sec in pairs(ir) do
................................................................................
			local tonespan = opts.accent and .1 or 0
			local tbg = opts['dark-on-light'] and 1.0 - tonespan or tonespan
			local tfg = opts['dark-on-light'] and tonespan or 1.0 - tonespan
			if var == 'bg' then
				return tone(tbg,nil,nil,tonumber(alpha))
			elseif var == 'fg' then
				return tone(tfg,nil,nil,tonumber(alpha))


			elseif var == 'tone' then
				local l, sep, sat
				for i=1,3 do -- 🙄
					l,sep,sat = param:match('^%('..string.rep('([^%s]*)%s*',i)..'%)$')
					if l then break end
				end
				l = ss.math.lerp(tonumber(l), tbg, tfg)
................................................................................
				kind = 'var';
				pos = pos;
				raw = raw;
				var = not pos and s or nil;
				origin = c:clone();
			}
		end
	end










































	ct.spanctls = {
		{seq = '!', parse = formatter 'emph'};
		{seq = '*', parse = formatter 'strong'};
		{seq = '~', parse = formatter 'strike'};
		{seq = '+', parse = formatter 'inser'};
		{seq = '\\', parse = function(s, c) -- raw
			return s




		end};
		{seq = '$\\', parse = function(s, c) -- raw











			return {
				kind = 'format';
				style = 'literal';
				spans = {s};


				origin = c:clone();


			}
		end};
		{seq = '$', parse = formatter 'literal'};

































		{seq = '&', parse = function(s, c)
			local r, t = s:match '^([^%s]+)%s*(.-)$'
			return {
				kind = 'term';
				spans = (t and t ~= "") and ct.parse_span(t, c) or {};
				ref = r; 
				origin = c:clone();
			}
		end};
		{seq = '^', parse = function(s, c)
			local fn, t = s:match '^([^%s]+)%s*(.-)$'
................................................................................
			}
		end};
		{seq = '>', parse = insert_link};
		{seq = '→', parse = insert_link};
		{seq = '🔗', parse = insert_link};
		{seq = '##', parse = insert_var_ref(true)};
		{seq = '#', parse = insert_var_ref(false)};




	}
end

function ct.parse_span(str,ctx)
	local function delimited(start, stop, s)
		local r = { pcall(ss.str.delimit, nil, start, stop, s) }
		if r[1] then return table.unpack(r, 2) end
		ctx:fail(tostring(r[2]))
	end
	local buf = ""
	local spans = {}
	local function flush()
		if buf ~= "" then



			table.insert(spans, buf)
			buf = ""
		end
	end
	local skip = false
	for c,p in eachcode(str) do
		if skip == true then
			skip = false
			buf = buf .. c
		elseif c == '\\' then
			skip = true





		elseif c == '{' then
			flush()
			local substr, following = delimited('{','}',str:sub(p.byte))
			local splitstart, splitstop = substr:find'%s+'
			local id, argstr
			if splitstart then
				id, argstr = substr:sub(1,splitstart-1), substr:sub(splitstop+1)
................................................................................
				local i = 1
				while i <= #argstr do
					while i<=#argstr and (argstr:sub(i,i) ~= '|' or argstr:sub(i-1,i) == '\\|') do
						i = i + 1
					end
					local arg = argstr:sub(start, i == #argstr and i or i-1)
					start = i+1

					table.insert(o.args, arg)
					i = i + 1
				end
			end

			p.next.byte = p.next.byte + following - 1
			table.insert(spans,o)
................................................................................
					table.insert(spans, i.parse(substr:sub(1+#i.seq), ctx))
					break
				end
			end
			if not found then
				ctx:fail('no recognized control sequence in [%s]', substr)
			end



		else
			buf = buf .. c
		end
	end
	flush()
	return spans
end

local function
blockwrap(fn)
	return function(l,c,j)
		local block = fn(l,c,j)
		block.origin = c:clone();
		table.insert(c.sec.blocks, block);
		j:hook('block_insert', c, block, l)



	end
end

local insert_paragraph = blockwrap(function(l,c)
	if l:sub(1,1) == '.' then l = l:sub(2) end
	return {
		kind = "paragraph";
................................................................................
	if t and t ~= "" then
		local heading = {
			kind = "label";
			spans = ct.parse_span(t,c);
			origin = s.origin;
			captions = s;
		}

		table.insert(s.blocks, heading)
		s.heading_node = heading
	end
	c.sec = s

	j:hook('section_attach', c, s)
end
................................................................................
	c.doc.meta[key] = val
	j:hook('metadata_set', key, val)
end
local dextctl = function(w,c)
	local mode, exts = w(1)
	for e in exts:gmatch '([^%s]+)' do
		if mode == 'uses' then

		elseif mode == 'needs' then

		elseif mode == 'inhibits' then

		end
	end
end
local dcond = function(w,c)
	local mode, cond, exp = w(2)
	c.hide_next = mode == 'unless'
end;
................................................................................
ct.directives = {
	author = dsetmeta;
	license = dsetmeta;
	keywords = dsetmeta;
	desc = dsetmeta;
	when = dcond;
	unless = dcond;


















	expand = function(w,c)
		local _, m = w(1)
		if m ~= 'off' then
			c.expand_next = 1
		else
			c.expand_next = 0
		end
	end;
}

local function insert_table_row(l,c,j)
	local row = {}
	local buf
................................................................................
	local flush = function()
		if buf then
			buf.str = buf.str:gsub('%s+$','')
			table.insert(row, buf)
		end
		buf = { str = '' }
	end
	for c,p in eachcode(l) do
		if c == '|' or c == '+' and (p.code == 1 or l:sub(p.byte-1,p.byte-1)~='\\') then
			flush()
			buf.header = c == '+'
		elseif c == ':' then
			local lst = l:sub(p.byte-#c,p.byte-#c)
			local nxt = l:sub(p.next.byte,p.next.byte)
			if lst == '|' or lst == '+' and l:sub(p.byte-2,p.byte-2) ~= '\\' then
................................................................................
		else
			buf.str = buf.str .. c
		end
	end
	if buf.str ~= '' then flush() end 
	for _,v in pairs(row) do
		v.spans = ct.parse_span(v.str, c)

	end
	if #c.sec.blocks > 1 and c.sec.blocks[#c.sec.blocks].kind == 'table' then
		local tbl = c.sec.blocks[#c.sec.blocks]
		table.insert(tbl.rows, row)
		j:hook('block_table_attach', c, tbl, row, l)
		j:hook('block_table_row_insert', c, tbl, row, l)
	else
................................................................................
	{seq = '¶', fn = insert_paragraph};
	{seq = '❡', fn = insert_paragraph};
	{seq = '#', fn = insert_section};
	{seq = '§', fn = insert_section};
	{seq = '+', fn = insert_table_row};
	{seq = '|', fn = insert_table_row};
	{seq = '│', fn = insert_table_row};
	{seq = '!', fn = function(l,c,j) 
		local last = c.sec.blocks[#c.sec.blocks]
		local txt = l:match '^%s*!%s*(.-)$'
		if (not last) or last.kind ~= 'aside' then
			local aside = {
				kind = 'aside';
				lines = { ct.parse_span(txt, c) }

			}

			c:insert(aside)
			j:hook('block_aside_insert', c, aside, l)
			j:hook('block_aside_line_insert', c, aside, aside.lines[1], l)
			j:hook('block_insert', c, aside, l)
		else
			local sp = ct.parse_span(txt, c)

			table.insert(last.lines, sp)
			j:hook('block_aside_attach', c, last, sp, l)
			j:hook('block_aside_line_insert', c, last, sp, l)
		end
	end};
	{pred = function(s,c) return s:match'^[*:]' end, fn = blockwrap(function(l,c) -- list
		local stars = l:match '^([*:]+)'
................................................................................
		return {
			kind = 'list-item';
			depth = depth;
			ordered = ordered;
			spans = ct.parse_span(txt, c);
		}
	end)};
	{seq = '\t', fn = function(l,c,j)









		local ref, val = l:match '\t+([^:]+):%s*(.*)$'









		c.sec.refs[ref] = val

		j:hook('section_ref_attach', c, ref, val, l)






	end};
	{seq = '%', fn = function(l,c,j) -- directive
		local crit, cmdline = l:match '^%%([!%%]?)%s*(.*)$'
		local words = function(i)
			local wds = {}
			if i == 0 then return cmdline end
			for w,pos in cmdline:gmatch '([^%s]+)()' do
				table.insert(wds, w)
				i = i - 1
				if i == 0 then
					table.insert(wds,cmdline:sub(pos))
					return table.unpack(wds)
				end
			end
		end

		local cmd, rest = words(1)
		if ct.directives[cmd] then
................................................................................
			ct.directives[cmd](words,c,j)
		elseif cmd == c.doc.stage.mode['render:format'] then
			-- this is a directive for the renderer; insert it into the tree as is
			local dir = {
				kind = 'directive';
				critical = crit == '!';
				words = words;

			}
			c:insert(dir)
			j:hook('block_directive_render', j, c, dir)
		elseif c.doc:allow_ext(cmd) then -- extension directives begin with their id
			local ext = ct.ext.loaded[cmd]
			if ext.directives then
				local _, topcmd = words(2)
				if ext.directives[topcmd] then
					ext.directives[topcmd](j:delegate(ext), c, words)
................................................................................
			kind = 'code';
			listing = {
				kind = 'listing';
				lang = lang, id = id, title = title and ct.parse_span(title,c);
				lines = {};
			}
		}




		j:hook('mode_switch', c, mode)
		c.mode = mode
		if id then
			if c.sec.refs[id] then c:fail('duplicate ID %s', id) end
			c.sec.refs[id] = c.mode.listing
		end
		j:hook('block_insert', c, mode.listing, l)
		return c.mode.listing;
	end)};
	{pred = function(s,c)
		if s:match '^[%-_][*_%-%s]+' then return true end
		if startswith(s, '—') then
			for c, p in eachcode(s) do
				if ({
					['—'] = true, ['-'] = true, [' '] = true;
					['*'] = true, ['_'] = true, ['\t'] = true;
				})[c] ~= true then return false end
			end
			return true
		end
	end; fn = blockwrap(function()
		return { kind = 'horiz-rule' }
	end)};














	{fn = insert_paragraph};
}

function ct.parse(file, src, mode)




























	local function
	is_whitespace(cp)
		return cp == 0x20 or cp == 0xe390





	end




























	local ctx = ct.ctx.mk(src)
	ctx.line = 0
	ctx.doc = ct.doc.mk()
	ctx.doc.src = src
	ctx.doc.stage = {
		kind = 'parse';
		mode = mode;
	}
	ctx.sec = ctx.doc:mksec() -- toplevel section
	ctx.sec.origin = ctx:clone()









	-- create states for extension hooks
	local job = ctx.doc:job('parse',nil,ctx)
















	for full_line in file:lines() do ctx.line = ctx.line + 1
		local l
		for p, c in utf8.codes(full_line) do
			if not is_whitespace(c) then
				l = full_line:sub(p)
				break
			end
		end
		job:hook('line_read',ctx,l)

		if ctx.mode then
			if ctx.mode.kind == 'code' then
				if l and l:match '^~~~%s*$' then
					job:hook('block_listing_end',ctx,ctx.mode.listing)
					job:hook('mode_switch', c, nil)
					ctx.mode = nil
				else
					-- TODO handle formatted code
					local newline = {l}
					table.insert(ctx.mode.listing.lines, newline)
					job:hook('block_listing_newline',ctx,ctx.mode.listing,newline)
				end
			else
				ctx:fail('unimplemented syntax mode %s', ctx.mode.kind)
			end
		else
			if l then
				local function tryseqs(seqs, ...)
					for _, i in pairs(seqs) do
						if  ((not i.seq ) or startswith(l, i.seq)) and
							((not i.pred) or i.pred    (l, ctx  )) then
							i.fn(l, ctx, job, ...)
							return true
						end
					end
					return false
				end

				if not tryseqs(ct.ctlseqs) then
					local found = false
					
					for eb, ext, state in job:each('blocks') do
						if tryseqs(eb, state) then found = true break end
					end

					if not found then
						ctx:fail 'incomprehensible input line'
					end
				end
			else
				if next(ctx.sec.blocks) and ctx.sec.blocks[#ctx.sec.blocks].kind ~= 'break' then
					local brk = {kind='break'}
					job:hook('block_break', ctx, brk, l)
					table.insert(ctx.sec.blocks, brk)
				end
			end
		end
		job:hook('line_end',ctx,l)
	end

	return ctx.doc
end

--  ~ lexi hale <lexi@hale.su>
--  © AGPLv3
--  ? reference implementation of the cortav document language

local ss = require 'sirsem'
-- aliases for commonly used sirsem funcs
local startswith = ss.str.begins

local dump = ss.dump
local declare = ss.declare

-- make this module available to require() when linked into a lua bytecode program with luac
local ct = ss.namespace 'cortav'
ct.info = {
	version = ss.version {0,1; 'devel'};
................................................................................
	end);
	cli = ss.exnkind 'command line parse error';
	mode = ss.exnkind('bad mode', function(msg, ...)
		return string.format("mode “%s” "..msg, ...)
	end);
	unimpl = ss.exnkind 'feature not implemented';
	ext = ss.exnkind 'extension error';
	enc = ss.exnkind('encoding error', function(msg, ...)
		return string.format('[%s]' .. msg, ...)
	end);
}

ct.ctx = declare {
	mk = function(src) return {src = src} end;
	ident = 'context';
	cast = {
		string = function(me)
................................................................................
			table.insert(self.sec.blocks,block)
			return block
		end;
		ref = function(self,id)
			if not id:find'%.' then
				local rid = self.sec.refs[id]
				if self.sec.refs[id] then
					return self.sec.refs[id], id, self.sec
				else self:fail("no such ref %s in current section", id or '') end
			else
				local sec, ref = string.match(id, "(.-)%.(.+)")
				local s = self.doc.sections[sec]
				if s then
					if s.refs[ref] then
						return s.refs[ref], ref, sec
					else self:fail("no such ref %s in section %s", ref, sec) end
				else self:fail("no such section %s", sec) end
			end
		end
	};
}

................................................................................
		meta = {};
		vars = {};
		ext = {
			inhibit = {};
			need = {};
			use = {};
		};
		enc = ss.str.enc.utf8;
	} end;
	construct = function(me)
		me.docjob = ct.ext.job('doc', me, nil)
	end;
}

-- FP helper functions
................................................................................

-- renderer engines
function ct.render.html(doc, opts)
	local doctitle = opts['title']
	local f = string.format
	local ids = {}
	local canonicalID = {}
	local function getSafeID(obj,pfx)
		pfx = pfx or ''
		if canonicalID[obj] then
			return canonicalID[obj]
		elseif obj.id and ids[pfx .. obj.id] then
			local objid = pfx .. obj.id
			local newid
			local i = 1
			repeat newid = objid .. string.format('-%x', i)
				i = i + 1 until not ids[newid]
			ids[newid] = obj
			canonicalID[obj] = newid
			return newid
		else
			local cid = obj.id
			if not cid then
				local i = 1
				repeat cid = string.format('%sx-%x', pfx, i)
					i = i + 1 until not ids[cid]
			end
			ids[cid] = obj
			canonicalID[obj] = cid
			return cid
		end
	end

	local footnotes = {}
	local footnotecount = 0

	local langsused = {}
	local langpairs = {
		lua = { color = 0x9377ff };
		terra = { color = 0xff77c8 };
		c = { name = 'C', color = 0x77ffe8 };
		html = { color = 0xfff877 };
		scheme = { color = 0x77ff88 };
		lisp = { color = 0x77ff88 };
		fortran = { color = 0xff779a };
		python = { color = 0xffd277 };
		ruby = { color = 0xcdd6ff };
	}

	local stylesets = {
		footnote = [[
			div.footnote {
			font-family: 90%;
				display: none;
				grid-template-columns: 1em 1fr min-content;
				grid-template-rows: 1fr min-content;
				position: fixed;
				padding: 1em;
				background: @tone(0.05);
				border: black;
				margin:auto;
			}
			div.footnote:target { display:grid; }
			@media screen {
				div.footnote {
					left: 10em;
					right: 10em;
					max-width: calc(@width + 2em);
					max-height: 30vw;
					bottom: 1em;
				}
			}
			@media print {
				div.footnote {
					position: relative;
				}
				div.footnote:first-of-type {
					border-top: 1px solid black;
				}
			}

			div.footnote > a[href="#0"]{
				grid-row: 2/3;
				grid-column: 3/4;
				display: block;
				padding: 0.2em 0.7em;
				text-align: center;
				text-decoration: none;
				background: @tone(0.2);
				color: @tone(1);
				border: 1px solid black;
				margin-top: 0.6em;
				-webkit-user-select: none;
				-ms-user-select: none;
				user-select: none;
				-webkit-user-drag: none;
				user-drag: none;
			}
			div.footnote > a[href="#0"]:hover {
				background: @tone(0.3);
				color: @tone(2);
			}
			div.footnote > a[href="#0"]:active {
				background: @tone(0.05);
				color: @tone(0.4);
			}
			@media print {
				div.footnote > a[href="#0"]{
					display:none;
				}
			}
			div.footnote > div.number {
				text-align:right;
				grid-row: 1/2;
				grid-column: 1/2;
			}
			div.footnote > div.text {
				grid-row: 1/2;
				grid-column: 2/4;
				padding-left: 1em;
				overflow-y: scroll;
			}
		]];
		header = [[
			body { padding: 0 2.5em !important }
			h1,h2,h3,h4,h5,h6 { border-bottom: 1px solid @tone(0.7); }
			h1 { font-size: 200%; border-bottom-style: double !important; border-bottom-width: 3px !important; margin: 0em -1em; }
			h2 { font-size: 130%; margin: 0em -0.7em; }
			h3 { font-size: 110%; margin: 0em -0.5em; }
			h4 { font-size: 100%; font-weight: normal; margin: 0em -0.2em; }
			h5 { font-size: 90%; font-weight: normal; }
			h6 { font-size: 80%; font-weight: normal; }
................................................................................
			section:target > :is(h1,h2,h3,h4,h5,h6) {

			}
		]];
		paragraph = [[
			p {
				margin: 0.7em 0;
				text-align: justify;
			}
			section {
				margin: 1.2em 0;
			}
			section:first-child { margin-top: 0; }
		]];
		accent = [[
			@media screen {
				body { background: @bg; color: @fg }
				a[href] {
					color: @tone(0.7 30);
					text-decoration-color: @tone/0.4(0.7 30);
				}
				a[href]:hover {
					color: @tone(0.9 30);
					text-decoration-color: @tone/0.7(0.7 30);
				}
				h1 { color: @tone(2); }
				h2 { color: @tone(1.5); }
				h3 { color: @tone(1.2); }
				h4 { color: @tone(1); }
				h5,h6 { color: @tone(0.8); }
			}
			@media print {
				a[href] {
					text-decoration: none;
					color: black;
					font-weight: bold;
				}
				h1,h2,h3,h4,h5,h6 {
					border-bottom: 1px black;
				}
			}
		]];
		aside = [[
			section > aside {
				text-align: justify;
				margin: 0 1.5em;
				padding: 0.5em 0.8em;
				background: @tone(0.05);
				font-size: 90%;
				border-left: 5px solid @tone(0.2 15);
				border-right: 5px solid @tone(0.2 15);
			}
			section > aside p {
				margin: 0;
				margin-top: 0.6em;
			}
			section > aside p:first-child {
				margin: 0;
			}
      ]];
		code = [[
			code {
				display: inline-block;
				background: @tone(0.9);
				color: @bg;
				font-family: monospace;
				font-size: 90%;
				padding: 3px 5px;
			}
		]];
		var = [[
			var {
				font-style: italic;
				font-family: monospace;
				color: @tone(0.7);
			}
			code var {
				color: @tone(0.25);
			}
		]];
		math = [[
			span.equation {
				display: inline-block;
				background: @tone(0.08);
				color: @tone(2);
				padding: 0.1em 0.3em;
				border: 1px solid @tone(0.5);
			}
		]];
		abbr = [[
			abbr[title] { cursor: help; }
		]];
		editors_markup = [[]];
		block_code_listing = [[
			figure.listing {
				font-family: monospace;
				background: @tone(0.05);
				color: @fg;
				padding: 0;
				margin: 0.3em 0;
				counter-reset: line-number;
				position: relative;
				border: 1px solid @fg;
			}
			figure.listing>div {
				white-space: pre-wrap;
				tab-size: 3;
				-moz-tab-size: 3;
				counter-increment: line-number;
				text-indent: -2.3em;
				margin-left: 2.3em;
			}
			figure.listing>:is(div,hr)::before {
				width: 1.0em;
				padding: 0.2em 0.4em;
				text-align: right;
				display: inline-block;
				background-color: @tone(0.2);
				border-right: 1px solid @fg;
				content: counter(line-number);
				margin-right: 0.3em;
			}
			figure.listing>hr::before {
				color: transparent;
				padding-top: 0;
				padding-bottom: 0;
			}
			figure.listing>div::before {
				color: @fg;
			}
			figure.listing>div:last-child::before {
				padding-bottom: 0.5em;
			}
			figure.listing>figcaption:first-child {
				border: none;
				border-bottom: 1px solid @fg;
			}
			figure.listing>figcaption::after {
				display: block;
				float: right;
				font-weight: normal;
				font-style: italic;
				font-size: 70%;
				padding-top: 0.3em;
			}
			figure.listing>figcaption {
				font-family: sans-serif;
				font-size: 120%;
				padding: 0.2em 0.4em;
				border: none;
				color: @tone(2);
			}
			figure.listing > hr {
				border: none;
				margin: 0;
				height: 0.7em;
				counter-increment: line-number;
			}
		]];
	}
................................................................................
		stylesets = stylesets;
		stylesets_active = stylesNeeded;
		obj_htmlid = getSafeID;
		-- remaining fields added later
	}

	local renderJob = doc:job('render_html', nil, render_state_handle)
	doc.stage.job = renderJob;

	local runhook = function(h, ...)
		return renderJob:hook(h, render_state_handle, ...)
	end

	local tagproc do
		local elt = function(t,attrs)
			return f('<%s%s>', t,
				attrs and ss.reduce(function(a,b) return a..b end, '',
					ss.map(function(v,k)
						if v == true
							then          return ' '..k
							elseif v then return f(' %s="%s"', k, v)
						end
					end, attrs)) or '')
		end

		tagproc = {
			toTXT = {
				tag = function(t,a,v) return v  end;
				elt = function(t,a)   return '' end;
				catenate = table.concat;
			};
			toIR = {
				tag = function(t,a,v,o) return {
					tag = t, attrs = a;
					nodes = type(v) == 'string' and {v} or v, src = o
				} end;

				elt = function(t,a,o) return {
					tag = t, attrs = a, src = o
				} end;

				catenate = function(...) return ... end;
			};
			toHTML = {
				elt = elt;
				tag = function(t,attrs,body)
					return f('%s%s</%s>', elt(t,attrs), body, t)
				end;
				catenate = table.concat;
			};
		}
	end

	local function getBaseRenderers(procs, span_renderers)
		local tag, elt, catenate = procs.tag, procs.elt, procs.catenate
		local htmlDoc = function(title, head, body)
			return [[<!doctype html>]] .. tag('html',nil,
				tag('head', nil,
					elt('meta',{charset = 'utf-8'}) ..
					(title and tag('title', nil, title) or '') ..
					(head or '')) ..
				tag('body', nil, body or ''))
		end


		local function htmlSpan(spans, block, sec)
			local text = {}
			for k,v in pairs(spans) do
				if type(v) == 'string' then

					v=v:gsub('[<>&"]', function(x)
							return string.format('&#%02u;', string.byte(x))
						end)
					for fn, ext in renderJob:each('hook','render_html_sanitize') do
						v = fn(renderJob:delegate(ext), v)
					end
					table.insert(text,v)
				else
					table.insert(text, (span_renderers[v.kind](v, block, sec)))
				end
			end
			return table.concat(text)
		end
		return {htmlDoc=htmlDoc, htmlSpan=htmlSpan}
	end

	local spanparse = function(...)
		local s = ct.parse_span(...)
		doc.docjob:hook('meddle_span', s)
		return s
	end

	local cssRulesFor = {}
	local function getSpanRenderers(procs)
		local tag, elt, catenate = procs.tag, procs.elt, procs.catenate
		local span_renderers = {}
		local plainrdr = getBaseRenderers(tagproc.toTXT, span_renderers)
		local htmlSpan = getBaseRenderers(procs, span_renderers).htmlSpan

		function span_renderers.format(sp,...)
			local tags = { strong = 'strong', emph = 'em', strike = 'del', insert = 'ins', literal = 'code', variable = 'var'}
			if sp.style == 'literal' and not opts['fossil-uv'] then
				stylesNeeded.code = true

			elseif sp.style == 'strike' or sp.style == 'insert' then
				stylesNeeded.editors_markup = true
			elseif sp.style == 'variable' then
				stylesNeeded.var = true
			end
			return tag(tags[sp.style],nil,htmlSpan(sp.spans,...))
		end

		function span_renderers.deref(t,b,s)
			local r = b.origin:ref(t.ref)
			local name = t.ref
			if name:find'%.' then name = name:match '^[^.]*%.(.+)$' end
			if type(r) == 'string' then


				stylesNeeded.abbr = true
				return tag('abbr',{title=r},next(t.spans) and htmlSpan(t.spans,b,s) or name)
			end
			if r.kind == 'resource' then
				local rid = getSafeID(r, 'res-')
				if r.class == 'image' then
					if not cssRulesFor[r] then
						local css = prepcss(string.format([[
							section p > .%s {
							}




						]], rid))
						stylesets[r] = css
						cssRulesFor[r] = css
						stylesNeeded[r] = true
					end



					return tag('div',{class=rid},catenate{'blaah'})
				elseif r.class == 'video' then
					local vid = {}
					return tag('video',nil,vid)
				elseif r.class == 'font' then
					b.origin:fail('fonts cannot be instantiated, use %font directive instead')
				end
			else
				b.origin:fail('%s is not an object that can be embedded', t.ref)
			end
		end

		function span_renderers.var(v,b,s)
			local val
			if v.pos then
				if not v.origin.invocation then
					v.origin:fail 'positional arguments can only be used in a macro invocation'
................................................................................
			end
			if v.raw then
				return val
			else
				return htmlSpan(ct.parse_span(val, v.origin), b, s)
			end
		end

		function span_renderers.raw(v,b,s)
			return htmlSpan(v.spans, b, s)
		end

		function span_renderers.link(sp,b,s)
			local href
			if b.origin.doc.sections[sp.ref] then
				href = '#' .. sp.ref
			else
				if sp.addr then href = sp.addr else
................................................................................
					if type(r) == 'table' then
						href = '#' .. getSafeID(r)
					else href = r end
				end
			end
			return tag('a',{href=href},next(sp.spans) and htmlSpan(sp.spans,b,s) or href)
		end

		span_renderers['line-break'] = function(sp,b,s)
			return elt('br')



		end

		function span_renderers.macro(m,b,s)
			local macroname = plainrdr.htmlSpan(
				ct.parse_span(m.macro, b.origin), b,s)
			local r = b.origin:ref(macroname)
			if type(r) ~= 'string' then
				b.origin:fail('%s is an object, not a reference', t.ref)
			end
			local mctx = b.origin:clone()
			mctx.invocation = m
			return htmlSpan(ct.parse_span(r, mctx),b,s)
		end
		function span_renderers.math(m,b,s)
			stylesNeeded.math = true
			return tag('span',{class='equation'},htmlSpan(m.spans, b, s))
		end;
		function span_renderers.directive(d,b,s)
			if d.ext == 'html' then
			elseif b.origin.doc:allow_ext(d.ext) then
			elseif d.crit then
				b.origin:fail('critical extension %s unavailable', d.ext)
			elseif d.failthru then
				return htmlSpan(d.spans, b, s)
			end
		end
		function span_renderers.footnote(f,b,s)
			stylesNeeded.footnote = true
			local source, sid, ssec = b.origin:ref(f.ref)
			local cnc = getSafeID(ssec) .. ' ' .. sid
			local fn
			if footnotes[cnc] then
				fn = footnotes[cnc]
			else
				footnotecount = footnotecount + 1
				fn = {num = footnotecount, origin = b.origin, fnid=cnc, source = source}
				fn.id = getSafeID(fn)
				footnotes[cnc] = fn
			end
			return tag('a', {href='#'..fn.id}, htmlSpan(f.spans) ..
						tag('sup',nil, fn.num))
		end

		return span_renderers
	end

	local function getBlockRenderers(procs, sr)
		local tag, elt, catenate = procs.tag, procs.elt, procs.catenate
		local null = function() return catenate{} end

		local block_renderers = {
			anchor = function(b,s)
................................................................................
					if #l > 0 then
						return tag('div',nil,sr.htmlSpan(l, b, s))
					else
						return elt('hr')
					end
				end, b.lines)
				if b.title then
					table.insert(nodes,1, tag('figcaption',nil,sr.htmlSpan(b.title)))
				end
				if b.lang then langsused[b.lang] = true end
				return tag('figure', {class='listing', lang=b.lang, id=b.id and getSafeID(b)}, catenate(nodes))
			end;
			aside = function(b,s)
				local bn = {}
				stylesNeeded.aside = true
				if #b.lines == 1 then
					bn[1] = sr.htmlSpan(b.lines[1], b, s)
				else
					for _,v in pairs(b.lines) do
						table.insert(bn, tag('p', {}, sr.htmlSpan(v, b, s)))
					end
				end
				return tag('aside', {}, bn)
			end;
			['break'] = function() -- HACK
				-- lists need to be rewritten to work like asides
				return '';
			end;
		}
		return block_renderers;
	end

	local function getRenderers(procs)
		local span_renderers = getSpanRenderers(procs)
		local r = getBaseRenderers(procs,span_renderers)
		r.block_renderers = getBlockRenderers(procs, r)
		return r








































	end

	local astproc = {
		toHTML = getRenderers(tagproc.toHTML);
		toTXT  = getRenderers(tagproc.toTXT);
		toIR   = { };
	}
................................................................................
	local ir = {}
	local dr = astproc.toHTML -- default renderers
	local plainr = astproc.toTXT
	local irBlockRdrs = astproc.toIR.block_renderers;

	render_state_handle.ir = ir;

	local function renderBlocks(blocks, irs)
		for i, block in ipairs(blocks) do
			local rd
			if irBlockRdrs[block.kind] then
				rd = irBlockRdrs[block.kind](block,sec)
			else
				local rdr = renderJob:proc('render',block.kind,'html')
				if rdr then
					rd = rdr({
						state = render_state_handle;
						tagproc = tagproc.toIR;
						astproc = astproc.toIR;
					}, block, sec)
				end
			end
			if rd then
				if opts['heading-anchors'] and block == sec.heading_node then
					stylesNeeded.headingAnchors = true
					table.insert(rd.nodes, ' ')
					table.insert(rd.nodes, {
						tag = 'a';
						attrs = {href = '#' .. irs.attrs.id, class='anchor'};
						nodes = {type(opts['heading-anchors'])=='string' and opts['heading-anchors'] or '&sect;'};
					})
				end
				if rd.src and rd.src.origin.lang then
					if not rd.attrs then rd.attrs = {} end
					rd.attrs.lang = rd.src.origin.lang
				end
				table.insert(irs.nodes, rd)
				runhook('ir_section_node_insert', rd, irs, sec)
			end
		end
	end
	runhook('ir_assemble', ir)
	for i, sec in ipairs(doc.secorder) do
		if doctitle == nil and sec.depth == 1 and sec.heading_node then
			doctitle = astproc.toTXT.htmlSpan(sec.heading_node.spans, sec.heading_node, sec)
		end
		local irs
		if sec.kind == 'ordinary' then
			if #(sec.blocks) > 0 then
				irs = {tag='section',attrs={id = getSafeID(sec)},nodes={}}

				runhook('ir_section_build', irs, sec)

				renderBlocks(sec.blocks, irs)



























			end
		elseif sec.kind == 'blockquote' then
		elseif sec.kind == 'listing' then
		elseif sec.kind == 'embed' then
		end
		if irs then table.insert(ir, irs) end
	end

	for _, fn in pairs(footnotes) do
		local tag = tagproc.toIR.tag
		local body = {nodes={}}
		local ftir = {}
		for l in fn.source:gmatch('([^\n]*)') do
			ct.parse_line(l, fn.origin, ftir)
		end
		renderBlocks(ftir,body)
		local note = tag('div',{class='footnote',id=fn.id}, {
			tag('div',{class='number'}, tostring(fn.num)),
			tag('div',{class='text'}, body.nodes),
			tag('a',{href='#0'},'close')
		})
		table.insert(ir, note)
	end

	-- restructure passes
	runhook('ir_restructure_pre', ir)
	
	---- list insertion pass
	local lists = {}
	for _, sec in pairs(ir) do
................................................................................
			local tonespan = opts.accent and .1 or 0
			local tbg = opts['dark-on-light'] and 1.0 - tonespan or tonespan
			local tfg = opts['dark-on-light'] and tonespan or 1.0 - tonespan
			if var == 'bg' then
				return tone(tbg,nil,nil,tonumber(alpha))
			elseif var == 'fg' then
				return tone(tfg,nil,nil,tonumber(alpha))
			elseif var == 'width' then
				return opts['width'] or '100vw'
			elseif var == 'tone' then
				local l, sep, sat
				for i=1,3 do -- 🙄
					l,sep,sat = param:match('^%('..string.rep('([^%s]*)%s*',i)..'%)$')
					if l then break end
				end
				l = ss.math.lerp(tonumber(l), tbg, tfg)
................................................................................
				kind = 'var';
				pos = pos;
				raw = raw;
				var = not pos and s or nil;
				origin = c:clone();
			}
		end
	end
	local function insert_span_directive(crit, failthru)
		return function(s,c)
			local args = ss.str.breakwords(d.doc.enc, s, 1)
			local brksyms = map(enc.encodeUCS, {
				'.', ',', ':', ';', '!', '$', '&', '^',
				'/', '?', '@', '='
			})
			local brkhash = {} for _,s in pairs(brksyms) do
				brkhash[s] = true
			end

			local extname = ''
			local sym
			local cmd = ''
			for ch,p in ss.str.each(c.doc.enc, args[1]) do
				if sym == nil then
					if brkhash[ch] then
						sym = ch
					else
						extname = extname .. ch
					end
				elseif brkhash[ch] then
					sym = sym + ch
				else
					cmd = cmd + ch
				end
			end
			if cmd == '' then cmd = nil end
			local spans if failthru then
				spans = ct.parse_span(args[2], c)
			end
			return {
				kind = 'directive';
				ext = extname;
				cmd = cmd;
				args = args;
				crit = crit;
				failthru = failthru;
				spans = spans;
			}
		end
	end
	ct.spanctls = {
		{seq = '!', parse = formatter 'emph'};
		{seq = '*', parse = formatter 'strong'};
		{seq = '~', parse = formatter 'strike'};
		{seq = '+', parse = formatter 'insert'};
		{seq = '\\', parse = function(s, c) -- raw
			return {
				kind = 'raw';
				spans = {s};
				origin = c:clone();
			}
		end};
		{seq = '`\\', parse = function(s, c) -- raw
			local o = c:clone();
			local str = ''
			for c, p in ss.str.each(c.doc.enc, s) do
				local q = p:esc()
				if q then
					str = str ..  q
					p.next.byte = p.next.byte + #q
				else
					str = str .. c
				end
			end
			return {
				kind = 'format';
				style = 'literal';
				spans = {{
					kind = 'raw';
					spans = {str};
					origin = o;
				}};
				origin = o;
			}
		end};
		{seq = '`', parse = formatter 'literal'};
		{seq = '$', parse = formatter 'variable'};
		{seq = '^', parse = function(s,c) --footnotes
			local r, t = s:match '^([^%s]+)%s*(.-)$'
			return {
				kind = 'footnote';
				ref = r;
				spans = ct.parse_span(t, c);
				origin = c:clone();
			}
		-- TODO support for footnote sections
		end};
		{seq = '=', parse = function(s,c) --math mode
			local tx = {
				['%*'] = '×';
				['/'] = '÷';
			}
			for k,v in pairs(tx) do s = s:gsub(k,v) end
			s=s:gsub('%^([0-9]+)', function(num)
				local sup = {'⁰','¹','²','³','⁴','⁵','⁶','⁷','⁸','⁹'};
				local r = ''
				for i=1,#num do
					r = r .. sup[1 + (num:byte(i) - 0x30)]
				end
				return r
			end)
			local m = {s} --TODO
			return {
				kind = 'math';
				original = s;
				spans = m;
				origin = c:clone();
			};
		end};
		{seq = '&', parse = function(s, c)
			local r, t = s:match '^([^%s]+)%s*(.-)$'
			return {
				kind = 'deref';
				spans = (t and t ~= "") and ct.parse_span(t, c) or {};
				ref = r; 
				origin = c:clone();
			}
		end};
		{seq = '^', parse = function(s, c)
			local fn, t = s:match '^([^%s]+)%s*(.-)$'
................................................................................
			}
		end};
		{seq = '>', parse = insert_link};
		{seq = '→', parse = insert_link};
		{seq = '🔗', parse = insert_link};
		{seq = '##', parse = insert_var_ref(true)};
		{seq = '#', parse = insert_var_ref(false)};
		{seq = '%%', parse = function() --[[NOP]] end};
		{seq = '%!', parse = insert_span_directive(true,false)};
		{seq = '%:', parse = insert_span_directive(false,true)};
		{seq = '%', parse = insert_span_directive(false,false)};
	}
end

function ct.parse_span(str,ctx)
	local function delimited(start, stop, s)
		local r = { pcall(ss.str.delimit, nil, start, stop, s) }
		if r[1] then return table.unpack(r, 2) end
		ctx:fail(tostring(r[2]))
	end
	local buf = ""
	local spans = {}
	local function flush()
		if buf ~= "" then
	-- 			for fn, ext in ctx.doc.docjob:each('hook','doc_meddle_string') do
	-- 				buf = fn(ctx.doc.docjob:delegate(ext), ctx, buf)
	-- 			end
			table.insert(spans, buf)
			buf = ""
		end
	end
	local skip = false
	for c,p in ss.str.each(ctx.doc.enc,str) do
		local ba, ca, es = ctx.doc.enc.parse_escape(str:sub(p.byte))
		if es then
			flush()
			table.insert(spans, {
				kind = 'raw';
				spans = {es};
				origin = ctx:clone()
			})
			p.next.byte = p.next.byte + ba;
			p.next.code = p.next.code + ca;
		elseif c == '{' then
			flush()
			local substr, following = delimited('{','}',str:sub(p.byte))
			local splitstart, splitstop = substr:find'%s+'
			local id, argstr
			if splitstart then
				id, argstr = substr:sub(1,splitstart-1), substr:sub(splitstop+1)
................................................................................
				local i = 1
				while i <= #argstr do
					while i<=#argstr and (argstr:sub(i,i) ~= '|' or argstr:sub(i-1,i) == '\\|') do
						i = i + 1
					end
					local arg = argstr:sub(start, i == #argstr and i or i-1)
					start = i+1
					arg=arg:gsub('\\|','|')
					table.insert(o.args, arg)
					i = i + 1
				end
			end

			p.next.byte = p.next.byte + following - 1
			table.insert(spans,o)
................................................................................
					table.insert(spans, i.parse(substr:sub(1+#i.seq), ctx))
					break
				end
			end
			if not found then
				ctx:fail('no recognized control sequence in [%s]', substr)
			end
		elseif c == '\n' then
			flush()
			table.insert(spans,{kind='line-break',origin=ctx:clone()})
		else
			buf = buf .. c
		end
	end
	flush()
	return spans
end

local function
blockwrap(fn)
	return function(l,c,j,d)
		local block = fn(l,c,j,d)
		block.origin = c:clone();
		table.insert(d, block);
		j:hook('block_insert', c, block, l)
		if block.spans then
			c.doc.docjob:hook('meddle_span', block.spans, block)
		end
	end
end

local insert_paragraph = blockwrap(function(l,c)
	if l:sub(1,1) == '.' then l = l:sub(2) end
	return {
		kind = "paragraph";
................................................................................
	if t and t ~= "" then
		local heading = {
			kind = "label";
			spans = ct.parse_span(t,c);
			origin = s.origin;
			captions = s;
		}
		c.doc.docjob:hook('meddle_span', heading.spans, heading)
		table.insert(s.blocks, heading)
		s.heading_node = heading
	end
	c.sec = s

	j:hook('section_attach', c, s)
end
................................................................................
	c.doc.meta[key] = val
	j:hook('metadata_set', key, val)
end
local dextctl = function(w,c)
	local mode, exts = w(1)
	for e in exts:gmatch '([^%s]+)' do
		if mode == 'uses' then
			c.doc.ext.use[e] = true
		elseif mode == 'needs' then
			c.doc.ext.need[e] = true
		elseif mode == 'inhibits' then
			c.doc.ext.inhibit[e] = true
		end
	end
end
local dcond = function(w,c)
	local mode, cond, exp = w(2)
	c.hide_next = mode == 'unless'
end;
................................................................................
ct.directives = {
	author = dsetmeta;
	license = dsetmeta;
	keywords = dsetmeta;
	desc = dsetmeta;
	when = dcond;
	unless = dcond;
	pragma = function(w,c)
	end;
	lang = function(w,c)
		local _, op, l = w(2)
		local langstack = c.doc.stage.langstack
		if op == 'is' then
			langstack[math.max(1, #langstack)] = l
		elseif op == 'push' then
			table.insert(langstack, l)
		elseif op == 'pop' then
			if next(langstack) then
				langstack[#langstack] = nil
			end
		elseif op == 'sec' then
			c.sec.lang = l
		else c:fail('bad language directive “%s”', op) end
		c.lang = langstack[#langstack]
	end;
	expand = function(w,c)
		local _, m = w(1)
		if m ~= 'off' then
			c.doc.stage.expand_next = 1
		else
			c.doc.stage.expand_next = 0
		end
	end;
}

local function insert_table_row(l,c,j)
	local row = {}
	local buf
................................................................................
	local flush = function()
		if buf then
			buf.str = buf.str:gsub('%s+$','')
			table.insert(row, buf)
		end
		buf = { str = '' }
	end
	for c,p in ss.str.each(c.doc.enc,l) do
		if c == '|' or c == '+' and (p.code == 1 or l:sub(p.byte-1,p.byte-1)~='\\') then
			flush()
			buf.header = c == '+'
		elseif c == ':' then
			local lst = l:sub(p.byte-#c,p.byte-#c)
			local nxt = l:sub(p.next.byte,p.next.byte)
			if lst == '|' or lst == '+' and l:sub(p.byte-2,p.byte-2) ~= '\\' then
................................................................................
		else
			buf.str = buf.str .. c
		end
	end
	if buf.str ~= '' then flush() end 
	for _,v in pairs(row) do
		v.spans = ct.parse_span(v.str, c)
		c.doc.docjob:hook('meddle_span', v.spans, v)
	end
	if #c.sec.blocks > 1 and c.sec.blocks[#c.sec.blocks].kind == 'table' then
		local tbl = c.sec.blocks[#c.sec.blocks]
		table.insert(tbl.rows, row)
		j:hook('block_table_attach', c, tbl, row, l)
		j:hook('block_table_row_insert', c, tbl, row, l)
	else
................................................................................
	{seq = '¶', fn = insert_paragraph};
	{seq = '❡', fn = insert_paragraph};
	{seq = '#', fn = insert_section};
	{seq = '§', fn = insert_section};
	{seq = '+', fn = insert_table_row};
	{seq = '|', fn = insert_table_row};
	{seq = '│', fn = insert_table_row};
	{seq = '!', fn = function(l,c,j,d)
		local last = d[#d]
		local txt = l:match '^%s*!%s*(.-)$'
		if (not last) or last.kind ~= 'aside' then
			local aside = {
				kind = 'aside';
				lines = { ct.parse_span(txt, c) };
				origin = c:clone();
			}
			c.doc.docjob:hook('meddle_span', aside.lines[1], aside)
			table.insert(d,aside)
			j:hook('block_aside_insert', c, aside, l)
			j:hook('block_aside_line_insert', c, aside, aside.lines[1], l)
			j:hook('block_insert', c, aside, l)
		else
			local sp = ct.parse_span(txt, c)
			c.doc.docjob:hook('meddle_span', sp, last)
			table.insert(last.lines, sp)
			j:hook('block_aside_attach', c, last, sp, l)
			j:hook('block_aside_line_insert', c, last, sp, l)
		end
	end};
	{pred = function(s,c) return s:match'^[*:]' end, fn = blockwrap(function(l,c) -- list
		local stars = l:match '^([*:]+)'
................................................................................
		return {
			kind = 'list-item';
			depth = depth;
			ordered = ordered;
			spans = ct.parse_span(txt, c);
		}
	end)};
	{seq = '\t\t', fn = function(l,c,j,d)
		local last = d[#d]
		if (not last) or (last.kind ~= 'reference') then
			c:fail('reference continuations must immediately follow a reference')
		end
		local str = l:match '^\t\t(.-)%s*$'
		last.val = last.val .. '\n' .. str
		c.sec.refs[last.key] = last.val
	end};
	{seq = '\t', fn = blockwrap(function(l,c,j,d)
		local ref, val = l:match '\t+([^:]+):%s*(.*)$'
		local last = d[#d]
		local rsrc
		if last and last.kind == 'resource' then
			last.props[ref] = val
			rsrc = last
		elseif last and last.kind == 'reference' and last.rsrc then
			last.rsrc.props[ref] = val
			rsrc = last.rsrc
		else
			c.sec.refs[ref] = val
		end
		j:hook('section_ref_attach', c, ref, val, l)
		return {
			kind = 'reference';
			rsrc = rsrc;
			key = ref;
			val = val;
		}
	end)};
	{seq = '%', fn = function(l,c,j,d) -- directive
		local crit, cmdline = l:match '^%%([!%%]?)%s*(.*)$'
		local words = function(i)
			local wds = {}
			if i == 0 then return cmdline end
			for w,pos in cmdline:gmatch '([^%s]+)()' do
				table.insert(wds, w)
				i = i - 1
				if i == 0 then
					table.insert(wds,(cmdline:sub(pos):match('^%s*(.-)%s*$')))
					return table.unpack(wds)
				end
			end
		end

		local cmd, rest = words(1)
		if ct.directives[cmd] then
................................................................................
			ct.directives[cmd](words,c,j)
		elseif cmd == c.doc.stage.mode['render:format'] then
			-- this is a directive for the renderer; insert it into the tree as is
			local dir = {
				kind = 'directive';
				critical = crit == '!';
				words = words;
				origin = c;
			}
			table.insert(d, dir)
			j:hook('block_directive_render', j, c, dir)
		elseif c.doc:allow_ext(cmd) then -- extension directives begin with their id
			local ext = ct.ext.loaded[cmd]
			if ext.directives then
				local _, topcmd = words(2)
				if ext.directives[topcmd] then
					ext.directives[topcmd](j:delegate(ext), c, words)
................................................................................
			kind = 'code';
			listing = {
				kind = 'listing';
				lang = lang, id = id, title = title and ct.parse_span(title,c);
				lines = {};
			}
		}
		if c.doc.stage.expand_next and c.doc.stage.expand_next > 0 then
			c.doc.stage.expand_next = c.doc.stage.expand_next - 1
			mode.expand = true
		end
		j:hook('mode_switch', c, mode)
		c.mode = mode
		if id then
			if c.sec.refs[id] then c:fail('duplicate ID %s', id) end
			c.sec.refs[id] = c.mode.listing
		end
		j:hook('block_insert', c, mode.listing, l)
		return c.mode.listing;
	end)};
	{pred = function(s,c)
		if s:match '^[%-_][*_%-%s]+' then return true end
		if startswith(s, '—') then
			for c, p in ss.str.each(c.doc.enc,s) do
				if ({
					['—'] = true, ['-'] = true, [' '] = true;
					['*'] = true, ['_'] = true, ['\t'] = true;
				})[c] ~= true then return false end
			end
			return true
		end
	end; fn = blockwrap(function()
		return { kind = 'horiz-rule' }
	end)};
	{seq='@', fn=blockwrap(function(s,c)
		local id = s:match '^@%s*(.-)%s*$'
		local rsrc = {
			kind = 'resource';
			props = {};
			id = id;
		}
		if c.sec.refs[id] then
			c:fail('an object with id “%s” already exists in that section',id)
		else
			c.sec.refs[id] = rsrc
		end
		return rsrc
	end)};
	{fn = insert_paragraph};
}

function ct.parse_line(l, ctx, dest)
	local newspan
	local job = ctx.doc.stage.job
	job:hook('line_read',ctx,l)
	if ctx.mode then
		if ctx.mode.kind == 'code' then
			if l and l:match '^~~~%s*$' then
				job:hook('block_listing_end',ctx,ctx.mode.listing)
				job:hook('mode_switch', c, nil)
				ctx.mode = nil
			else
				-- TODO handle formatted code
				local newline
				if ctx.mode.expand
					then newline = ct.parse_span(l, ctx)
					else newline = {l}
				end
				table.insert(ctx.mode.listing.lines, newline)
				job:hook('block_listing_newline',ctx,ctx.mode.listing,newline)
			end
	  else
			local mf = job:proc('modes', ctx.mode.kind)
			if not mf then
				ctx:fail('unimplemented syntax mode %s', ctx.mode.kind)
			end
			mf(job, ctx, l, dest) --NOTE: you are responsible for triggering the appropriate hooks if you insert anything!
		end
	else
		if l then
			local function tryseqs(seqs, ...)


				for _, i in pairs(seqs) do
					if ((not i.seq ) or startswith(l, i.seq)) and
					   ((not i.pred) or i.pred    (l, ctx  )) then
						i.fn(l, ctx, job, dest, ...)
						return true
					end
				end
				return false
			end

			if not tryseqs(ct.ctlseqs) then
				local found = false

				for eb, ext, state in job:each('blocks') do
					if tryseqs(eb, state) then found = true break end
				end

				if not found then
					ctx:fail 'incomprehensible input line'
				end
			end
		else
			if next(dest) and dest[#dest].kind ~= 'break' then
				local brk = {kind='break', origin = ctx:clone()}
				job:hook('block_break', ctx, brk, l)
				table.insert(dest, brk)
			end
		end
	end
	job:hook('line_end',ctx,l)
end

function ct.parse(file, src, mode, setup)

	local ctx = ct.ctx.mk(src)
	ctx.line = 0
	ctx.doc = ct.doc.mk()
	ctx.doc.src = src




	ctx.sec = ctx.doc:mksec() -- toplevel section
	ctx.sec.origin = ctx:clone()
	ctx.lang = mode['meta:lang']
	if mode['parse:enc'] then
		local e = ss.str.enc[mode['parse:enc']]
		if not e then
			ct.exns.enc('requested encoding not supported',mode['parse:enc']):throw()
		end
		ctx.doc.enc = e
	end

	-- create states for extension hooks
	local job = ctx.doc:job('parse',nil,ctx)
	ctx.doc.stage = {
		kind = 'parse';
		mode = mode;
		job = job;
		langstack = {ctx.lang};
		fontstack = {};
	}

	local function
	is_whitespace(cp)
		return ctx.doc.enc.iswhitespace(cp)
	end

	if setup then setup(ctx) end


	for full_line in file:lines() do ctx.line = ctx.line + 1
		local l
		for p, c in utf8.codes(full_line) do
			if not is_whitespace(c) then
				l = full_line:sub(p)
				break
			end
		end
		ct.parse_line(l, ctx, ctx.sec.blocks)
	end

	for i, sec in ipairs(ctx.doc.secorder) do
		for refid, r in ipairs(sec.refs) do
			if type(r) == 'table' and r.kind == 'resource' and r.props.src then
				local lines = ss.str.breaklines(ctx.doc.enc, r.props.src)
				local srcs = {}
				for i,l in ipairs(lines) do
					local args = ss.str.breakwords(ctx.doc.enc, l, 2, {escape=true})
					if #args < 3 then
						r.origin:fail('invalid syntax for resource %s', t.ref)
					end
					local mimebreak = function(s)
						local wds = ss.str.split(ctx.doc.enc, s, '/', 1, {escape=true})
						return wds
					end
					local mime = mimebreak(args[2]);
					local mimeclasses = {
						['application/svg+xml'] = 'image';
					}
					local class = mimeclasses[mime]
					table.insert(srcs, {
						mode = args[1];
						mime = mime;
						uri = args[3];
						class = class or mime[1];
					})
				end
				 --ideally move this into its own mimetype lib
				local kind = r.props.as or srcs[1].class
				r.class = kind
				r.srcs = srcs
			end
		end
	end
	ctx.doc.stage = nil
	ctx.doc.docjob:hook('meddle_ast')
	return ctx.doc
end