@@ -5,9 +5,8 @@ local ss = require 'sirsem' -- aliases for commonly used sirsem funcs local startswith = ss.str.begins -local eachcode = ss.str.enc.utf8.each local dump = ss.dump local declare = ss.declare -- make this module available to require() when linked into a lua bytecode program with luac @@ -84,8 +83,11 @@ return string.format("mode ā€œ%sā€ "..msg, ...) end); unimpl = ss.exnkind 'feature not implemented'; ext = ss.exnkind 'extension error'; + enc = ss.exnkind('encoding error', function(msg, ...) + return string.format('[%s]' .. msg, ...) + end); } ct.ctx = declare { mk = function(src) return {src = src} end; @@ -115,16 +117,16 @@ ref = function(self,id) if not id:find'%.' then local rid = self.sec.refs[id] if self.sec.refs[id] then - return self.sec.refs[id] + return self.sec.refs[id], id, self.sec else self:fail("no such ref %s in current section", id or '') end else local sec, ref = string.match(id, "(.-)%.(.+)") local s = self.doc.sections[sec] if s then if s.refs[ref] then - return s.refs[ref] + return s.refs[ref], ref, sec else self:fail("no such ref %s in section %s", ref, sec) end else self:fail("no such section %s", sec) end end end @@ -220,8 +222,9 @@ inhibit = {}; need = {}; use = {}; }; + enc = ss.str.enc.utf8; } end; construct = function(me) me.docjob = ct.ext.job('doc', me, nil) end; @@ -400,15 +403,17 @@ local doctitle = opts['title'] local f = string.format local ids = {} local canonicalID = {} - local function getSafeID(obj) + local function getSafeID(obj,pfx) + pfx = pfx or '' if canonicalID[obj] then return canonicalID[obj] - elseif obj.id and ids[obj.id] then + elseif obj.id and ids[pfx .. obj.id] then + local objid = pfx .. obj.id local newid local i = 1 - repeat newid = obj.id .. string.format('-%x', i) + repeat newid = objid .. string.format('-%x', i) i = i + 1 until not ids[newid] ids[newid] = obj canonicalID[obj] = newid return newid @@ -415,9 +420,9 @@ else local cid = obj.id if not cid then local i = 1 - repeat cid = string.format('x-%x', i) + repeat cid = string.format('%sx-%x', pfx, i) i = i + 1 until not ids[cid] end ids[cid] = obj canonicalID[obj] = cid @@ -424,8 +429,11 @@ return cid end end + local footnotes = {} + local footnotecount = 0 + local langsused = {} local langpairs = { lua = { color = 0x9377ff }; terra = { color = 0xff77c8 }; @@ -434,13 +442,87 @@ scheme = { color = 0x77ff88 }; lisp = { color = 0x77ff88 }; fortran = { color = 0xff779a }; python = { color = 0xffd277 }; - python = { color = 0xcdd6ff }; + ruby = { color = 0xcdd6ff }; } local stylesets = { + footnote = [[ + div.footnote { + font-family: 90%; + display: none; + grid-template-columns: 1em 1fr min-content; + grid-template-rows: 1fr min-content; + position: fixed; + padding: 1em; + background: @tone(0.05); + border: black; + margin:auto; + } + div.footnote:target { display:grid; } + @media screen { + div.footnote { + left: 10em; + right: 10em; + max-width: calc(@width + 2em); + max-height: 30vw; + bottom: 1em; + } + } + @media print { + div.footnote { + position: relative; + } + div.footnote:first-of-type { + border-top: 1px solid black; + } + } + + div.footnote > a[href="#0"]{ + grid-row: 2/3; + grid-column: 3/4; + display: block; + padding: 0.2em 0.7em; + text-align: center; + text-decoration: none; + background: @tone(0.2); + color: @tone(1); + border: 1px solid black; + margin-top: 0.6em; + -webkit-user-select: none; + -ms-user-select: none; + user-select: none; + -webkit-user-drag: none; + user-drag: none; + } + div.footnote > a[href="#0"]:hover { + background: @tone(0.3); + color: @tone(2); + } + div.footnote > a[href="#0"]:active { + background: @tone(0.05); + color: @tone(0.4); + } + @media print { + div.footnote > a[href="#0"]{ + display:none; + } + } + div.footnote > div.number { + text-align:right; + grid-row: 1/2; + grid-column: 1/2; + } + div.footnote > div.text { + grid-row: 1/2; + grid-column: 2/4; + padding-left: 1em; + overflow-y: scroll; + } + ]]; header = [[ + body { padding: 0 2.5em !important } h1,h2,h3,h4,h5,h6 { border-bottom: 1px solid @tone(0.7); } h1 { font-size: 200%; border-bottom-style: double !important; border-bottom-width: 3px !important; margin: 0em -1em; } h2 { font-size: 130%; margin: 0em -0.7em; } h3 { font-size: 110%; margin: 0em -0.5em; } @@ -493,45 +575,96 @@ ]]; paragraph = [[ p { margin: 0.7em 0; + text-align: justify; } section { margin: 1.2em 0; } section:first-child { margin-top: 0; } ]]; accent = [[ - body { background: @bg; color: @fg } - a[href] { - color: @tone(0.7 30); - text-decoration-color: @tone/0.4(0.7 30); + @media screen { + body { background: @bg; color: @fg } + a[href] { + color: @tone(0.7 30); + text-decoration-color: @tone/0.4(0.7 30); + } + a[href]:hover { + color: @tone(0.9 30); + text-decoration-color: @tone/0.7(0.7 30); + } + h1 { color: @tone(2); } + h2 { color: @tone(1.5); } + h3 { color: @tone(1.2); } + h4 { color: @tone(1); } + h5,h6 { color: @tone(0.8); } } - a[href]:hover { - color: @tone(0.9 30); - text-decoration-color: @tone/0.7(0.7 30); + @media print { + a[href] { + text-decoration: none; + color: black; + font-weight: bold; + } + h1,h2,h3,h4,h5,h6 { + border-bottom: 1px black; + } } - h1 { color: @tone(2); } - h2 { color: @tone(1.5); } - h3 { color: @tone(1.2); } - h4 { color: @tone(1); } - h5,h6 { color: @tone(0.8); } ]]; + aside = [[ + section > aside { + text-align: justify; + margin: 0 1.5em; + padding: 0.5em 0.8em; + background: @tone(0.05); + font-size: 90%; + border-left: 5px solid @tone(0.2 15); + border-right: 5px solid @tone(0.2 15); + } + section > aside p { + margin: 0; + margin-top: 0.6em; + } + section > aside p:first-child { + margin: 0; + } + ]]; code = [[ code { - background: @fg; + display: inline-block; + background: @tone(0.9); color: @bg; font-family: monospace; font-size: 90%; padding: 3px 5px; } + ]]; + var = [[ + var { + font-style: italic; + font-family: monospace; + color: @tone(0.7); + } + code var { + color: @tone(0.25); + } + ]]; + math = [[ + span.equation { + display: inline-block; + background: @tone(0.08); + color: @tone(2); + padding: 0.1em 0.3em; + border: 1px solid @tone(0.5); + } ]]; abbr = [[ abbr[title] { cursor: help; } ]]; editors_markup = [[]]; block_code_listing = [[ - section > figure.listing { + figure.listing { font-family: monospace; background: @tone(0.05); color: @fg; padding: 0; @@ -539,15 +672,17 @@ counter-reset: line-number; position: relative; border: 1px solid @fg; } - section > figure.listing>div { + figure.listing>div { white-space: pre-wrap; + tab-size: 3; + -moz-tab-size: 3; counter-increment: line-number; text-indent: -2.3em; margin-left: 2.3em; } - section > figure.listing>:is(div,hr)::before { + figure.listing>:is(div,hr)::before { width: 1.0em; padding: 0.2em 0.4em; text-align: right; display: inline-block; @@ -555,39 +690,39 @@ border-right: 1px solid @fg; content: counter(line-number); margin-right: 0.3em; } - section > figure.listing>hr::before { + figure.listing>hr::before { color: transparent; padding-top: 0; padding-bottom: 0; } - section > figure.listing>div::before { + figure.listing>div::before { color: @fg; } - section > figure.listing>div:last-child::before { + figure.listing>div:last-child::before { padding-bottom: 0.5em; } - section > figure.listing>figcaption:first-child { + figure.listing>figcaption:first-child { border: none; border-bottom: 1px solid @fg; } - section > figure.listing>figcaption::after { + figure.listing>figcaption::after { display: block; float: right; font-weight: normal; font-style: italic; font-size: 70%; padding-top: 0.3em; } - section > figure.listing>figcaption { + figure.listing>figcaption { font-family: sans-serif; font-size: 120%; padding: 0.2em 0.4em; border: none; color: @tone(2); } - section > figure.listing > hr { + figure.listing > hr { border: none; margin: 0; height: 0.7em; counter-increment: line-number; @@ -607,14 +742,55 @@ -- remaining fields added later } local renderJob = doc:job('render_html', nil, render_state_handle) + doc.stage.job = renderJob; local runhook = function(h, ...) return renderJob:hook(h, render_state_handle, ...) end - local function getSpanRenderers(procs) + local tagproc do + local elt = function(t,attrs) + return f('<%s%s>', t, + attrs and ss.reduce(function(a,b) return a..b end, '', + ss.map(function(v,k) + if v == true + then return ' '..k + elseif v then return f(' %s="%s"', k, v) + end + end, attrs)) or '') + end + + tagproc = { + toTXT = { + tag = function(t,a,v) return v end; + elt = function(t,a) return '' end; + catenate = table.concat; + }; + toIR = { + tag = function(t,a,v,o) return { + tag = t, attrs = a; + nodes = type(v) == 'string' and {v} or v, src = o + } end; + + elt = function(t,a,o) return { + tag = t, attrs = a, src = o + } end; + + catenate = function(...) return ... end; + }; + toHTML = { + elt = elt; + tag = function(t,attrs,body) + return f('%s%s', elt(t,attrs), body, t) + end; + catenate = table.concat; + }; + } + end + + local function getBaseRenderers(procs, span_renderers) local tag, elt, catenate = procs.tag, procs.elt, procs.catenate local htmlDoc = function(title, head, body) return [[]] .. tag('html',nil, tag('head', nil, @@ -623,54 +799,83 @@ (head or '')) .. tag('body', nil, body or '')) end - local span_renderers = {} local function htmlSpan(spans, block, sec) local text = {} for k,v in pairs(spans) do if type(v) == 'string' then - table.insert(text,(v:gsub('[<>&"]', - function(x) + v=v:gsub('[<>&"]', function(x) return string.format('&#%02u;', string.byte(x)) - end))) + end) + for fn, ext in renderJob:each('hook','render_html_sanitize') do + v = fn(renderJob:delegate(ext), v) + end + table.insert(text,v) else - table.insert(text, span_renderers[v.kind](v, block, sec)) + table.insert(text, (span_renderers[v.kind](v, block, sec))) end end return table.concat(text) end + return {htmlDoc=htmlDoc, htmlSpan=htmlSpan} + end + + local spanparse = function(...) + local s = ct.parse_span(...) + doc.docjob:hook('meddle_span', s) + return s + end + + local cssRulesFor = {} + local function getSpanRenderers(procs) + local tag, elt, catenate = procs.tag, procs.elt, procs.catenate + local span_renderers = {} + local plainrdr = getBaseRenderers(tagproc.toTXT, span_renderers) + local htmlSpan = getBaseRenderers(procs, span_renderers).htmlSpan function span_renderers.format(sp,...) - local tags = { strong = 'strong', emph = 'em', strike = 'del', insert = 'ins', literal = 'code' } + local tags = { strong = 'strong', emph = 'em', strike = 'del', insert = 'ins', literal = 'code', variable = 'var'} if sp.style == 'literal' and not opts['fossil-uv'] then stylesNeeded.code = true - end - if sp.style == 'del' or sp.style == 'ins' then + elseif sp.style == 'strike' or sp.style == 'insert' then stylesNeeded.editors_markup = true + elseif sp.style == 'variable' then + stylesNeeded.var = true end return tag(tags[sp.style],nil,htmlSpan(sp.spans,...)) end - function span_renderers.term(t,b,s) + function span_renderers.deref(t,b,s) local r = b.origin:ref(t.ref) local name = t.ref if name:find'%.' then name = name:match '^[^.]*%.(.+)$' end - if type(r) ~= 'string' then - b.origin:fail('%s is an object, not a reference', t.ref) + if type(r) == 'string' then + stylesNeeded.abbr = true + return tag('abbr',{title=r},next(t.spans) and htmlSpan(t.spans,b,s) or name) end - stylesNeeded.abbr = true - return tag('abbr',{title=r},next(t.spans) and htmlSpan(t.spans,b,s) or name) - end - - function span_renderers.macro(m,b,s) - local r = b.origin:ref(m.macro) - if type(r) ~= 'string' then - b.origin:fail('%s is an object, not a reference', t.ref) + if r.kind == 'resource' then + local rid = getSafeID(r, 'res-') + if r.class == 'image' then + if not cssRulesFor[r] then + local css = prepcss(string.format([[ + section p > .%s { + } + ]], rid)) + stylesets[r] = css + cssRulesFor[r] = css + stylesNeeded[r] = true + end + return tag('div',{class=rid},catenate{'blaah'}) + elseif r.class == 'video' then + local vid = {} + return tag('video',nil,vid) + elseif r.class == 'font' then + b.origin:fail('fonts cannot be instantiated, use %font directive instead') + end + else + b.origin:fail('%s is not an object that can be embedded', t.ref) end - local mctx = b.origin:clone() - mctx.invocation = m - return htmlSpan(ct.parse_span(r, mctx),b,s) end function span_renderers.var(v,b,s) local val @@ -689,8 +894,12 @@ else return htmlSpan(ct.parse_span(val, v.origin), b, s) end end + + function span_renderers.raw(v,b,s) + return htmlSpan(v.spans, b, s) + end function span_renderers.link(sp,b,s) local href if b.origin.doc.sections[sp.ref] then @@ -704,15 +913,56 @@ end end return tag('a',{href=href},next(sp.spans) and htmlSpan(sp.spans,b,s) or href) end - return { - span_renderers = span_renderers; - htmlSpan = htmlSpan; - htmlDoc = htmlDoc; - } + + span_renderers['line-break'] = function(sp,b,s) + return elt('br') + end + + function span_renderers.macro(m,b,s) + local macroname = plainrdr.htmlSpan( + ct.parse_span(m.macro, b.origin), b,s) + local r = b.origin:ref(macroname) + if type(r) ~= 'string' then + b.origin:fail('%s is an object, not a reference', t.ref) + end + local mctx = b.origin:clone() + mctx.invocation = m + return htmlSpan(ct.parse_span(r, mctx),b,s) + end + function span_renderers.math(m,b,s) + stylesNeeded.math = true + return tag('span',{class='equation'},htmlSpan(m.spans, b, s)) + end; + function span_renderers.directive(d,b,s) + if d.ext == 'html' then + elseif b.origin.doc:allow_ext(d.ext) then + elseif d.crit then + b.origin:fail('critical extension %s unavailable', d.ext) + elseif d.failthru then + return htmlSpan(d.spans, b, s) + end + end + function span_renderers.footnote(f,b,s) + stylesNeeded.footnote = true + local source, sid, ssec = b.origin:ref(f.ref) + local cnc = getSafeID(ssec) .. ' ' .. sid + local fn + if footnotes[cnc] then + fn = footnotes[cnc] + else + footnotecount = footnotecount + 1 + fn = {num = footnotecount, origin = b.origin, fnid=cnc, source = source} + fn.id = getSafeID(fn) + footnotes[cnc] = fn + end + return tag('a', {href='#'..fn.id}, htmlSpan(f.spans) .. + tag('sup',nil, fn.num)) + end + + return span_renderers end - local function getBlockRenderers(procs, sr) local tag, elt, catenate = procs.tag, procs.elt, procs.catenate local null = function() return catenate{} end @@ -769,69 +1019,38 @@ return elt('hr') end end, b.lines) if b.title then - table.insert(nodes,1,tag('figcaption',nil,sr.htmlSpan(b.title))) + table.insert(nodes,1, tag('figcaption',nil,sr.htmlSpan(b.title))) end if b.lang then langsused[b.lang] = true end return tag('figure', {class='listing', lang=b.lang, id=b.id and getSafeID(b)}, catenate(nodes)) end; aside = function(b,s) local bn = {} - for _,v in pairs(b.lines) do - table.insert(bn, tag('p', {}, sr.htmlSpan(v, b, s))) + stylesNeeded.aside = true + if #b.lines == 1 then + bn[1] = sr.htmlSpan(b.lines[1], b, s) + else + for _,v in pairs(b.lines) do + table.insert(bn, tag('p', {}, sr.htmlSpan(v, b, s))) + end end return tag('aside', {}, bn) end; - ['break'] = function() --[[nop]] end; + ['break'] = function() -- HACK + -- lists need to be rewritten to work like asides + return ''; + end; } return block_renderers; end local function getRenderers(procs) - local r = getSpanRenderers(procs) + local span_renderers = getSpanRenderers(procs) + local r = getBaseRenderers(procs,span_renderers) r.block_renderers = getBlockRenderers(procs, r) return r - end - - local tagproc do - local elt = function(t,attrs) - return f('<%s%s>', t, - attrs and ss.reduce(function(a,b) return a..b end, '', - ss.map(function(v,k) - if v == true - then return ' '..k - elseif v then return f(' %s="%s"', k, v) - end - end, attrs)) or '') - end - - tagproc = { - toTXT = { - tag = function(t,a,v) return v end; - elt = function(t,a) return '' end; - catenate = table.concat; - }; - toIR = { - tag = function(t,a,v,o) return { - tag = t, attrs = a; - nodes = type(v) == 'string' and {v} or v, src = o - } end; - - elt = function(t,a,o) return { - tag = t, attrs = a, src = o - } end; - - catenate = function(...) return ... end; - }; - toHTML = { - elt = elt; - tag = function(t,attrs,body) - return f('%s%s', elt(t,attrs), body, t) - end; - catenate = table.concat; - }; - } end local astproc = { toHTML = getRenderers(tagproc.toHTML); @@ -856,8 +1075,42 @@ local irBlockRdrs = astproc.toIR.block_renderers; render_state_handle.ir = ir; + local function renderBlocks(blocks, irs) + for i, block in ipairs(blocks) do + local rd + if irBlockRdrs[block.kind] then + rd = irBlockRdrs[block.kind](block,sec) + else + local rdr = renderJob:proc('render',block.kind,'html') + if rdr then + rd = rdr({ + state = render_state_handle; + tagproc = tagproc.toIR; + astproc = astproc.toIR; + }, block, sec) + end + end + if rd then + if opts['heading-anchors'] and block == sec.heading_node then + stylesNeeded.headingAnchors = true + table.insert(rd.nodes, ' ') + table.insert(rd.nodes, { + tag = 'a'; + attrs = {href = '#' .. irs.attrs.id, class='anchor'}; + nodes = {type(opts['heading-anchors'])=='string' and opts['heading-anchors'] or '§'}; + }) + end + if rd.src and rd.src.origin.lang then + if not rd.attrs then rd.attrs = {} end + rd.attrs.lang = rd.src.origin.lang + end + table.insert(irs.nodes, rd) + runhook('ir_section_node_insert', rd, irs, sec) + end + end + end runhook('ir_assemble', ir) for i, sec in ipairs(doc.secorder) do if doctitle == nil and sec.depth == 1 and sec.heading_node then doctitle = astproc.toTXT.htmlSpan(sec.heading_node.spans, sec.heading_node, sec) @@ -865,46 +1118,33 @@ local irs if sec.kind == 'ordinary' then if #(sec.blocks) > 0 then irs = {tag='section',attrs={id = getSafeID(sec)},nodes={}} - runhook('ir_section_build', irs, sec) - - for i, block in ipairs(sec.blocks) do - local rd - if irBlockRdrs[block.kind] then - rd = irBlockRdrs[block.kind](block,sec) - else - local rdr = renderJob:proc('render',block.kind,'html') - if rdr then - rd = rdr({ - state = render_state_handle; - tagproc = tagproc.toIR; - astproc = astproc.toIR; - }, block, sec) - end - end - if rd then - if opts['heading-anchors'] and block == sec.heading_node then - stylesNeeded.headingAnchors = true - table.insert(rd.nodes, ' ') - table.insert(rd.nodes, { - tag = 'a'; - attrs = {href = '#' .. irs.attrs.id, class='anchor'}; - nodes = {type(opts['heading-anchors'])=='string' and opts['heading-anchors'] or '§'}; - }) - end - table.insert(irs.nodes, rd) - runhook('ir_section_node_insert', rd, irs, sec) - end - end + renderBlocks(sec.blocks, irs) end elseif sec.kind == 'blockquote' then elseif sec.kind == 'listing' then elseif sec.kind == 'embed' then end if irs then table.insert(ir, irs) end end + + for _, fn in pairs(footnotes) do + local tag = tagproc.toIR.tag + local body = {nodes={}} + local ftir = {} + for l in fn.source:gmatch('([^\n]*)') do + ct.parse_line(l, fn.origin, ftir) + end + renderBlocks(ftir,body) + local note = tag('div',{class='footnote',id=fn.id}, { + tag('div',{class='number'}, tostring(fn.num)), + tag('div',{class='text'}, body.nodes), + tag('a',{href='#0'},'close') + }) + table.insert(ir, note) + end -- restructure passes runhook('ir_restructure_pre', ir) @@ -1036,8 +1276,10 @@ if var == 'bg' then return tone(tbg,nil,nil,tonumber(alpha)) elseif var == 'fg' then return tone(tfg,nil,nil,tonumber(alpha)) + elseif var == 'width' then + return opts['width'] or '100vw' elseif var == 'tone' then local l, sep, sat for i=1,3 do -- šŸ™„ l,sep,sat = param:match('^%('..string.rep('([^%s]*)%s*',i)..'%)$') @@ -1127,30 +1369,124 @@ var = not pos and s or nil; origin = c:clone(); } end + end + local function insert_span_directive(crit, failthru) + return function(s,c) + local args = ss.str.breakwords(d.doc.enc, s, 1) + local brksyms = map(enc.encodeUCS, { + '.', ',', ':', ';', '!', '$', '&', '^', + '/', '?', '@', '=' + }) + local brkhash = {} for _,s in pairs(brksyms) do + brkhash[s] = true + end + + local extname = '' + local sym + local cmd = '' + for ch,p in ss.str.each(c.doc.enc, args[1]) do + if sym == nil then + if brkhash[ch] then + sym = ch + else + extname = extname .. ch + end + elseif brkhash[ch] then + sym = sym + ch + else + cmd = cmd + ch + end + end + if cmd == '' then cmd = nil end + local spans if failthru then + spans = ct.parse_span(args[2], c) + end + return { + kind = 'directive'; + ext = extname; + cmd = cmd; + args = args; + crit = crit; + failthru = failthru; + spans = spans; + } + end end ct.spanctls = { {seq = '!', parse = formatter 'emph'}; {seq = '*', parse = formatter 'strong'}; {seq = '~', parse = formatter 'strike'}; - {seq = '+', parse = formatter 'inser'}; + {seq = '+', parse = formatter 'insert'}; {seq = '\\', parse = function(s, c) -- raw - return s - end}; - {seq = '$\\', parse = function(s, c) -- raw return { - kind = 'format'; - style = 'literal'; + kind = 'raw'; spans = {s}; origin = c:clone(); } end}; - {seq = '$', parse = formatter 'literal'}; + {seq = '`\\', parse = function(s, c) -- raw + local o = c:clone(); + local str = '' + for c, p in ss.str.each(c.doc.enc, s) do + local q = p:esc() + if q then + str = str .. q + p.next.byte = p.next.byte + #q + else + str = str .. c + end + end + return { + kind = 'format'; + style = 'literal'; + spans = {{ + kind = 'raw'; + spans = {str}; + origin = o; + }}; + origin = o; + } + end}; + {seq = '`', parse = formatter 'literal'}; + {seq = '$', parse = formatter 'variable'}; + {seq = '^', parse = function(s,c) --footnotes + local r, t = s:match '^([^%s]+)%s*(.-)$' + return { + kind = 'footnote'; + ref = r; + spans = ct.parse_span(t, c); + origin = c:clone(); + } + -- TODO support for footnote sections + end}; + {seq = '=', parse = function(s,c) --math mode + local tx = { + ['%*'] = 'Ɨ'; + ['/'] = 'Ć·'; + } + for k,v in pairs(tx) do s = s:gsub(k,v) end + s=s:gsub('%^([0-9]+)', function(num) + local sup = {'ā°','Ā¹','Ā²','Ā³','ā“','āµ','ā¶','ā·','āø','ā¹'}; + local r = '' + for i=1,#num do + r = r .. sup[1 + (num:byte(i) - 0x30)] + end + return r + end) + local m = {s} --TODO + return { + kind = 'math'; + original = s; + spans = m; + origin = c:clone(); + }; + end}; {seq = '&', parse = function(s, c) local r, t = s:match '^([^%s]+)%s*(.-)$' return { - kind = 'term'; + kind = 'deref'; spans = (t and t ~= "") and ct.parse_span(t, c) or {}; ref = r; origin = c:clone(); } @@ -1168,8 +1504,12 @@ {seq = 'ā†’', parse = insert_link}; {seq = 'šŸ”—', parse = insert_link}; {seq = '##', parse = insert_var_ref(true)}; {seq = '#', parse = insert_var_ref(false)}; + {seq = '%%', parse = function() --[[NOP]] end}; + {seq = '%!', parse = insert_span_directive(true,false)}; + {seq = '%:', parse = insert_span_directive(false,true)}; + {seq = '%', parse = insert_span_directive(false,false)}; } end function ct.parse_span(str,ctx) @@ -1181,19 +1521,27 @@ local buf = "" local spans = {} local function flush() if buf ~= "" then + -- for fn, ext in ctx.doc.docjob:each('hook','doc_meddle_string') do + -- buf = fn(ctx.doc.docjob:delegate(ext), ctx, buf) + -- end table.insert(spans, buf) buf = "" end end local skip = false - for c,p in eachcode(str) do - if skip == true then - skip = false - buf = buf .. c - elseif c == '\\' then - skip = true + for c,p in ss.str.each(ctx.doc.enc,str) do + local ba, ca, es = ctx.doc.enc.parse_escape(str:sub(p.byte)) + if es then + flush() + table.insert(spans, { + kind = 'raw'; + spans = {es}; + origin = ctx:clone() + }) + p.next.byte = p.next.byte + ba; + p.next.code = p.next.code + ca; elseif c == '{' then flush() local substr, following = delimited('{','}',str:sub(p.byte)) local splitstart, splitstop = substr:find'%s+' @@ -1217,8 +1565,9 @@ i = i + 1 end local arg = argstr:sub(start, i == #argstr and i or i-1) start = i+1 + arg=arg:gsub('\\|','|') table.insert(o.args, arg) i = i + 1 end end @@ -1239,8 +1588,11 @@ end if not found then ctx:fail('no recognized control sequence in [%s]', substr) end + elseif c == '\n' then + flush() + table.insert(spans,{kind='line-break',origin=ctx:clone()}) else buf = buf .. c end end @@ -1249,13 +1601,16 @@ end local function blockwrap(fn) - return function(l,c,j) - local block = fn(l,c,j) + return function(l,c,j,d) + local block = fn(l,c,j,d) block.origin = c:clone(); - table.insert(c.sec.blocks, block); + table.insert(d, block); j:hook('block_insert', c, block, l) + if block.spans then + c.doc.docjob:hook('meddle_span', block.spans, block) + end end end local insert_paragraph = blockwrap(function(l,c) @@ -1285,8 +1640,9 @@ spans = ct.parse_span(t,c); origin = s.origin; captions = s; } + c.doc.docjob:hook('meddle_span', heading.spans, heading) table.insert(s.blocks, heading) s.heading_node = heading end c.sec = s @@ -1302,10 +1658,13 @@ local dextctl = function(w,c) local mode, exts = w(1) for e in exts:gmatch '([^%s]+)' do if mode == 'uses' then + c.doc.ext.use[e] = true elseif mode == 'needs' then + c.doc.ext.need[e] = true elseif mode == 'inhibits' then + c.doc.ext.inhibit[e] = true end end end local dcond = function(w,c) @@ -1318,14 +1677,32 @@ keywords = dsetmeta; desc = dsetmeta; when = dcond; unless = dcond; + pragma = function(w,c) + end; + lang = function(w,c) + local _, op, l = w(2) + local langstack = c.doc.stage.langstack + if op == 'is' then + langstack[math.max(1, #langstack)] = l + elseif op == 'push' then + table.insert(langstack, l) + elseif op == 'pop' then + if next(langstack) then + langstack[#langstack] = nil + end + elseif op == 'sec' then + c.sec.lang = l + else c:fail('bad language directive ā€œ%sā€', op) end + c.lang = langstack[#langstack] + end; expand = function(w,c) local _, m = w(1) if m ~= 'off' then - c.expand_next = 1 + c.doc.stage.expand_next = 1 else - c.expand_next = 0 + c.doc.stage.expand_next = 0 end end; } @@ -1338,9 +1715,9 @@ table.insert(row, buf) end buf = { str = '' } end - for c,p in eachcode(l) do + for c,p in ss.str.each(c.doc.enc,l) do if c == '|' or c == '+' and (p.code == 1 or l:sub(p.byte-1,p.byte-1)~='\\') then flush() buf.header = c == '+' elseif c == ':' then @@ -1374,8 +1751,9 @@ end if buf.str ~= '' then flush() end for _,v in pairs(row) do v.spans = ct.parse_span(v.str, c) + c.doc.docjob:hook('meddle_span', v.spans, v) end if #c.sec.blocks > 1 and c.sec.blocks[#c.sec.blocks].kind == 'table' then local tbl = c.sec.blocks[#c.sec.blocks] table.insert(tbl.rows, row) @@ -1401,22 +1779,25 @@ {seq = 'Ā§', fn = insert_section}; {seq = '+', fn = insert_table_row}; {seq = '|', fn = insert_table_row}; {seq = 'ā”‚', fn = insert_table_row}; - {seq = '!', fn = function(l,c,j) - local last = c.sec.blocks[#c.sec.blocks] + {seq = '!', fn = function(l,c,j,d) + local last = d[#d] local txt = l:match '^%s*!%s*(.-)$' if (not last) or last.kind ~= 'aside' then local aside = { kind = 'aside'; - lines = { ct.parse_span(txt, c) } + lines = { ct.parse_span(txt, c) }; + origin = c:clone(); } - c:insert(aside) + c.doc.docjob:hook('meddle_span', aside.lines[1], aside) + table.insert(d,aside) j:hook('block_aside_insert', c, aside, l) j:hook('block_aside_line_insert', c, aside, aside.lines[1], l) j:hook('block_insert', c, aside, l) else local sp = ct.parse_span(txt, c) + c.doc.docjob:hook('meddle_span', sp, last) table.insert(last.lines, sp) j:hook('block_aside_attach', c, last, sp, l) j:hook('block_aside_line_insert', c, last, sp, l) end @@ -1433,14 +1814,39 @@ ordered = ordered; spans = ct.parse_span(txt, c); } end)}; - {seq = '\t', fn = function(l,c,j) - local ref, val = l:match '\t+([^:]+):%s*(.*)$' - c.sec.refs[ref] = val - j:hook('section_ref_attach', c, ref, val, l) + {seq = '\t\t', fn = function(l,c,j,d) + local last = d[#d] + if (not last) or (last.kind ~= 'reference') then + c:fail('reference continuations must immediately follow a reference') + end + local str = l:match '^\t\t(.-)%s*$' + last.val = last.val .. '\n' .. str + c.sec.refs[last.key] = last.val end}; - {seq = '%', fn = function(l,c,j) -- directive + {seq = '\t', fn = blockwrap(function(l,c,j,d) + local ref, val = l:match '\t+([^:]+):%s*(.*)$' + local last = d[#d] + local rsrc + if last and last.kind == 'resource' then + last.props[ref] = val + rsrc = last + elseif last and last.kind == 'reference' and last.rsrc then + last.rsrc.props[ref] = val + rsrc = last.rsrc + else + c.sec.refs[ref] = val + end + j:hook('section_ref_attach', c, ref, val, l) + return { + kind = 'reference'; + rsrc = rsrc; + key = ref; + val = val; + } + end)}; + {seq = '%', fn = function(l,c,j,d) -- directive local crit, cmdline = l:match '^%%([!%%]?)%s*(.*)$' local words = function(i) local wds = {} if i == 0 then return cmdline end @@ -1447,9 +1853,9 @@ for w,pos in cmdline:gmatch '([^%s]+)()' do table.insert(wds, w) i = i - 1 if i == 0 then - table.insert(wds,cmdline:sub(pos)) + table.insert(wds,(cmdline:sub(pos):match('^%s*(.-)%s*$'))) return table.unpack(wds) end end end @@ -1462,10 +1868,11 @@ local dir = { kind = 'directive'; critical = crit == '!'; words = words; + origin = c; } - c:insert(dir) + table.insert(d, dir) j:hook('block_directive_render', j, c, dir) elseif c.doc:allow_ext(cmd) then -- extension directives begin with their id local ext = ct.ext.loaded[cmd] if ext.directives then @@ -1508,8 +1915,12 @@ lang = lang, id = id, title = title and ct.parse_span(title,c); lines = {}; } } + if c.doc.stage.expand_next and c.doc.stage.expand_next > 0 then + c.doc.stage.expand_next = c.doc.stage.expand_next - 1 + mode.expand = true + end j:hook('mode_switch', c, mode) c.mode = mode if id then if c.sec.refs[id] then c:fail('duplicate ID %s', id) end @@ -1520,9 +1931,9 @@ end)}; {pred = function(s,c) if s:match '^[%-_][*_%-%s]+' then return true end if startswith(s, 'ā€”') then - for c, p in eachcode(s) do + for c, p in ss.str.each(c.doc.enc,s) do if ({ ['ā€”'] = true, ['-'] = true, [' '] = true; ['*'] = true, ['_'] = true, ['\t'] = true; })[c] ~= true then return false end @@ -1530,31 +1941,122 @@ return true end end; fn = blockwrap(function() return { kind = 'horiz-rule' } + end)}; + {seq='@', fn=blockwrap(function(s,c) + local id = s:match '^@%s*(.-)%s*$' + local rsrc = { + kind = 'resource'; + props = {}; + id = id; + } + if c.sec.refs[id] then + c:fail('an object with id ā€œ%sā€ already exists in that section',id) + else + c.sec.refs[id] = rsrc + end + return rsrc end)}; {fn = insert_paragraph}; } -function ct.parse(file, src, mode) - local function - is_whitespace(cp) - return cp == 0x20 or cp == 0xe390 +function ct.parse_line(l, ctx, dest) + local newspan + local job = ctx.doc.stage.job + job:hook('line_read',ctx,l) + if ctx.mode then + if ctx.mode.kind == 'code' then + if l and l:match '^~~~%s*$' then + job:hook('block_listing_end',ctx,ctx.mode.listing) + job:hook('mode_switch', c, nil) + ctx.mode = nil + else + -- TODO handle formatted code + local newline + if ctx.mode.expand + then newline = ct.parse_span(l, ctx) + else newline = {l} + end + table.insert(ctx.mode.listing.lines, newline) + job:hook('block_listing_newline',ctx,ctx.mode.listing,newline) + end + else + local mf = job:proc('modes', ctx.mode.kind) + if not mf then + ctx:fail('unimplemented syntax mode %s', ctx.mode.kind) + end + mf(job, ctx, l, dest) --NOTE: you are responsible for triggering the appropriate hooks if you insert anything! + end + else + if l then + local function tryseqs(seqs, ...) + for _, i in pairs(seqs) do + if ((not i.seq ) or startswith(l, i.seq)) and + ((not i.pred) or i.pred (l, ctx )) then + i.fn(l, ctx, job, dest, ...) + return true + end + end + return false + end + + if not tryseqs(ct.ctlseqs) then + local found = false + + for eb, ext, state in job:each('blocks') do + if tryseqs(eb, state) then found = true break end + end + + if not found then + ctx:fail 'incomprehensible input line' + end + end + else + if next(dest) and dest[#dest].kind ~= 'break' then + local brk = {kind='break', origin = ctx:clone()} + job:hook('block_break', ctx, brk, l) + table.insert(dest, brk) + end + end end + job:hook('line_end',ctx,l) +end + +function ct.parse(file, src, mode, setup) local ctx = ct.ctx.mk(src) ctx.line = 0 ctx.doc = ct.doc.mk() ctx.doc.src = src - ctx.doc.stage = { - kind = 'parse'; - mode = mode; - } ctx.sec = ctx.doc:mksec() -- toplevel section ctx.sec.origin = ctx:clone() + ctx.lang = mode['meta:lang'] + if mode['parse:enc'] then + local e = ss.str.enc[mode['parse:enc']] + if not e then + ct.exns.enc('requested encoding not supported',mode['parse:enc']):throw() + end + ctx.doc.enc = e + end -- create states for extension hooks local job = ctx.doc:job('parse',nil,ctx) + ctx.doc.stage = { + kind = 'parse'; + mode = mode; + job = job; + langstack = {ctx.lang}; + fontstack = {}; + } + + local function + is_whitespace(cp) + return ctx.doc.enc.iswhitespace(cp) + end + + if setup then setup(ctx) end + for full_line in file:lines() do ctx.line = ctx.line + 1 local l for p, c in utf8.codes(full_line) do @@ -1562,58 +2064,44 @@ l = full_line:sub(p) break end end - job:hook('line_read',ctx,l) + ct.parse_line(l, ctx, ctx.sec.blocks) + end - if ctx.mode then - if ctx.mode.kind == 'code' then - if l and l:match '^~~~%s*$' then - job:hook('block_listing_end',ctx,ctx.mode.listing) - job:hook('mode_switch', c, nil) - ctx.mode = nil - else - -- TODO handle formatted code - local newline = {l} - table.insert(ctx.mode.listing.lines, newline) - job:hook('block_listing_newline',ctx,ctx.mode.listing,newline) - end - else - ctx:fail('unimplemented syntax mode %s', ctx.mode.kind) - end - else - if l then - local function tryseqs(seqs, ...) - for _, i in pairs(seqs) do - if ((not i.seq ) or startswith(l, i.seq)) and - ((not i.pred) or i.pred (l, ctx )) then - i.fn(l, ctx, job, ...) - return true - end + for i, sec in ipairs(ctx.doc.secorder) do + for refid, r in ipairs(sec.refs) do + if type(r) == 'table' and r.kind == 'resource' and r.props.src then + local lines = ss.str.breaklines(ctx.doc.enc, r.props.src) + local srcs = {} + for i,l in ipairs(lines) do + local args = ss.str.breakwords(ctx.doc.enc, l, 2, {escape=true}) + if #args < 3 then + r.origin:fail('invalid syntax for resource %s', t.ref) + end + local mimebreak = function(s) + local wds = ss.str.split(ctx.doc.enc, s, '/', 1, {escape=true}) + return wds end - return false + local mime = mimebreak(args[2]); + local mimeclasses = { + ['application/svg+xml'] = 'image'; + } + local class = mimeclasses[mime] + table.insert(srcs, { + mode = args[1]; + mime = mime; + uri = args[3]; + class = class or mime[1]; + }) end - - if not tryseqs(ct.ctlseqs) then - local found = false - - for eb, ext, state in job:each('blocks') do - if tryseqs(eb, state) then found = true break end - end - - if not found then - ctx:fail 'incomprehensible input line' - end - end - else - if next(ctx.sec.blocks) and ctx.sec.blocks[#ctx.sec.blocks].kind ~= 'break' then - local brk = {kind='break'} - job:hook('block_break', ctx, brk, l) - table.insert(ctx.sec.blocks, brk) - end + --ideally move this into its own mimetype lib + local kind = r.props.as or srcs[1].class + r.class = kind + r.srcs = srcs end end - job:hook('line_end',ctx,l) end - + ctx.doc.stage = nil + ctx.doc.docjob:hook('meddle_ast') return ctx.doc end