util  Artifact [2e5da05ad6]

Artifact 2e5da05ad6577e7e32ae1130385f69578705942c798de54dbb5bc899a56ae289:


-- [ʞ] parvan.lua
--  ? tool for maintaining and searching dictionaries
-- [ CONTROL CLASS: GREEN ]
-- [ CODEWORDS: - GENERAL ACCESS - ]
-- [ CONTROLLING AUTHORITY: - INTERDIRECTORIAL -
--  < Commission for Defense Communication >
--    +WCO   Worlds Culture Overdirectorate
--      SSD  Social Sciences Directorate
--       ELS External Linguistics Subdirectorate
--    +WSO   Worlds Security Overdirectorate
--      EID  External Influence Directorate ]

local function implies(a,b) return a==b or not(a) end

local ansi = {
	levels = {
		plain = 0;
		ansi = 1;
		color = 2;
		color8b = 3;
		color24b = 4;
	};
}

ansi.seqs = {
	br = {ansi.levels.ansi, "[1m", "[21m"};
	hl = {ansi.levels.ansi, "[7m", "[27m"};
	ul = {ansi.levels.ansi, "[4m", "[24m"};
	em = {ansi.levels.ansi, "[3m", "[23m"};
};

function ansi.termclass(fd) -- awkwardly emulate isatty
	if fd:seek('cur',0) then
		return ansi.levels.plain
	end

	if os.getenv('COLORTERM') == 'truecolor' then
		return ansi.levels.color24b
	end

	local term = os.getenv('TERM')
	if term then
		if term:match '-256color' then
			return ansi.levels.color8b
		elseif term:match '-color' then
			return ansi.levels.color
		else
			return ansi.levels.ansi
		end
	end

	return ansi.levels.plain
end

function ansi.formatter(fd)
	local cl = ansi.termclass(fd)
	local id = function(...) return ... end
	local esc = '\27'
	local f = {}
	for k,v in pairs(ansi.seqs) do
		local lvl, on, off = table.unpack(v)
		if lvl <= cl then
			f[k] = function(s)
				return esc..on .. s .. esc..off
			end
		else f[k] = id end
	end
	local function ftoi(r,g,b)
		return math.ceil(r*0xff),
		       math.ceil(g*0xff),
		       math.ceil(b*0xff)
	end
	local reset = "\27[39m"
	function f.color(str, n, br)
		return string.format("\27[%s%cm",
			(bg and 4 or 3) +
			(br and 6 or 0), 0x30+n)
			.. str .. reset
	end
	function f.resetLine()
		return '\27[1K\13'
	end
	if cl == ansi.levels.color24b then
		function f.rgb(str, r,g,b, bg)
			return string.format("\27[%c8;2;%u;%u;%um", bg and 0x34 or 0x33,
				ftoi(r,g,b)) .. str .. reset
		end
	elseif cl == ansi.levels.color8b then
		function f.rgb(str, r,g,b, bg)
			local code = 16 + (r * 5)*36 + (g * 5)*6 + (b * 6)
			return string.format("\27[%c8;5;%um", bg and 0x34 or 0x33, code)
				.. str .. reset
		end
	elseif cl == ansi.levels.color then
		function f.rgb(str, r,g,b, bg)
			local code = 0x30 + 1 -- TODO
			return string.format("\27[%c%cm", bg and 0x34 or 0x33, code)
				.. str .. reset
		end
	else
		function f.rgb(s) return s end
		function f.color(s) return s end
		function f.resetLine() return '' end
	end
	return f
end


local function dump(v,pfx,cyc,ismeta)
	pfx = pfx or ''
	cyc = cyc or {}
	local np = pfx .. '  '

	if type(v) == 'table' then
		if cyc[v] then return '<...>' else cyc[v] = true end
	end

	if type(v) == 'string' then
		return string.format('%q', v)
	elseif type(v) == 'table' then
		local str = ''
		for k,v in pairs(v) do
			local tkey, tval = dump(k,np,cyc), dump(v,np,cyc)
			str = str .. string.format('%s[%s] = %s\n', np, tkey,tval)
		end
		local meta = ''
		if getmetatable(v) then
			meta = dump(getmetatable(v),pfx,cyc,true) .. '::'
		end
		if ismeta then
			return string.format('%s<|\n%s%s|>',meta,str,pfx)
		else
			return meta..'{\n' .. str .. pfx .. '}\n'
		end
	else
		return string.format('%s', v)
	end
end

local struct = {
	__call = function(s,...) return s:mk(...) end;
}
function struct:mk(s)
	function s.is(o) return getmetatable(o) == s end
	return setmetatable(s, self)
end
setmetatable(struct, struct)

local stream = struct {
	__index = {
		next = function(self, f)
			local flds = {string.unpack('<'..f, self.data, self.index)}
			self.index = flds[#flds]
			flds[#flds] = nil
			return table.unpack(flds)
		end;
	};
	mk = function(self, str)
		return setmetatable({
			data = str;
			index = 1;
		}, self)
	end;
}

local fmt = {}

local userError = struct {
	__tostring = function(self) return self.msg end;
	mk = function(self, s) return setmetatable({msg=s},self) end;
}

local function id10t(...)
	error(userError(string.format(...)),0)
end

local packer,unpacker =
	function(f) return function(...) return string.pack  ("<"..f, ...) end end,
	function(f) return function( s ) return s:next       (f) end end
local qpack = function(f) return {
	encode = packer(f);
	decode = unpacker(f);
} end

local parse, marshal
fmt.string = qpack "s4"
fmt.label = qpack "s2"
fmt.tag = qpack "s1"
fmt.u8 = qpack "I1"
fmt.u16 = qpack "I2"
fmt.u24 = qpack "I3"
fmt.u32 = qpack "I4"
fmt.list = function(t,ty) ty = ty or fmt.u32
	return {
		encode = function(a)
			local vals = {marshal(ty, #a)}
			for i=1,#a do
				table.insert(vals, marshal(t, a[i]))
			end
			return table.concat(vals)
		end;
		decode = function(s)
			local n = parse(ty, s)
			local vals = {}
			for i=1,n do
				table.insert(vals, parse(t, s))
			end
			return vals
		end;
	}
end

fmt.map = function(from,to,ity)
	local ent = fmt.list({
		{'key', from},
		{'val', to}
	}, ity)
	return {
		encode = function(a)
			local m = {}
			for k,v in pairs(a) do
				table.insert(m, {key=k, val=v})
			end
			return ent.encode(m)
		end;
		decode = function(s)
			local lst = ent.decode(s)
			local m = {}
			for _,p in pairs(lst) do m[p.key] = p.val end
			return m
		end;
	}
end

fmt.form = {
	{'form', fmt.u16};
	{'text', fmt.label};
}

fmt.note = {
	{'kind', fmt.tag};
	{'paras', fmt.list(fmt.string)};
}

fmt.meaning = {
	{'lit', fmt.string};
	{'notes', fmt.list(fmt.note,fmt.u8)};
}

fmt.def = {
	{'part', fmt.u8};
	{'branch', fmt.list(fmt.label,fmt.u8)};
	{'means', fmt.list(fmt.meaning,fmt.u8)};
	{'forms', fmt.list(fmt.form,fmt.u16)};
}

fmt.word = {
	{'defs', fmt.list(fmt.def,fmt.u8)};
}

fmt.dictHeader = {
	{'lang', fmt.tag};
	{'meta', fmt.string};
	{'partsOfSpeech', fmt.list(fmt.tag,fmt.u16)};
}

fmt.synonymSet = {
	{'uid', fmt.u32};
		-- IDs are persistent random values so they can be used
		-- as reliable identifiers even when merging exports in
		-- a parvan-unaware VCS
	{'members', fmt.list({
		{'word', fmt.label}, {'def', fmt.u8};
	},fmt.u16)};
}

fmt.dict = {
	{'header', fmt.dictHeader};
	{'words', fmt.map(fmt.string,fmt.word)};
	{'synonyms', fmt.list(fmt.synonymSet)};
}

function marshal(ty, val)
	if ty.encode then
		return ty.encode(val)
	end
	local ac = {}

	for idx,fld in ipairs(ty) do
		local name, fty = table.unpack(fld)
		table.insert(ac, marshal(fty, assert(val[name])))
	end

	return table.concat(ac)
end

function parse(ty, stream)
	if ty.decode then
		return ty.decode(stream)
	end

	local obj = {}
	for idx,fld in ipairs(ty) do
		local name, fty = table.unpack(fld)
		obj[name] = parse(fty, stream)
	end
	return obj
end

local function
atomizer()
	local map = {}
	local i = 1
	return function(v)
		if map[v] then return map[v] else
			map[v] = i
			i=i+1
			return i-1
		end
	end, map
end

local function
writeDict(d)
	local atomizePoS, posMap = atomizer()
	for lit,w in pairs(d.words) do
		for j,def in ipairs(w.defs) do
			def.part = atomizePoS(def.part)
		end
	end
	d.header.partsOfSpeech = {}
	for v,i in pairs(posMap) do
		d.header.partsOfSpeech[i] = v
	end
	return 'PV0\2'..marshal(fmt.dict, d)
end

local function
readDict(file)
	local s = stream(file)
	local magic = s:next 'c4'
	if magic ~= 'PV0\2' then
		id10t 'not a parvan file'
	end
	local d = parse(fmt.dict, s)
	-- handle atoms
	for lit,w in pairs(d.words) do
		for j,def in ipairs(w.defs) do
			def.part = d.header.partsOfSpeech[def.part]
		end
	end
	return d
end

local function strwords(str) -- here be dragons
	local wds = {}
	local w = {}
	local state, d, quo, dquo = 0,0
	local function flush(n)
		if next(w) then
			table.insert(wds, utf8.char(table.unpack(w)))
			w = {}
		end
		state = n
		quo = nil
		dquo = nil
		d = 0
	end
	local function isws(c)
		return c == 0x20 or c == 0x09 or c == 0x0a
	end
	for p,cp in utf8.codes(str) do
		if state == 0 then -- begin
			if not(isws(cp)) then
				if cp == 0x22 or cp == 0x27 then
					quo = cp
				elseif cp == 0x5b then -- boxquote
					 quo = 0x5d
					dquo = 0x5b
				elseif cp == 0x7b then -- curlquote
					 quo = 0x7d
					dquo = 0x7b
				elseif cp == 0x201c then -- fancyquote
					 quo = 0x201d
					dquo = 0x201c
				end
				if quo then
					state = 2
					d = 1
				elseif cp == 0x5c then -- escape
					state = 11
				else
					state = 1
					table.insert(w, cp)
				end
			end
		elseif state == 1 then -- word
			if isws(cp) then flush(0)
			elseif cp == 0x5c then state = 11 else
				table.insert(w,cp)
			end
		elseif state == 2 then -- (nested?) quote
			if cp == 0x5c then state = 12
			elseif cp == quo then
				d = d - 1
				if d == 0 then
					flush(0)
				else
					table.insert(w,cp)
				end
			else
				if cp == dquo then d = d + 1 end
				table.insert(w,cp)
			end
		elseif state == 11 or state == 12 then -- escape
			-- 12 = quote escape, 11 = raw escape
			if cp == 0x63 then --n
				table.insert(w,0x0a)
			else
				table.insert(w,cp)
			end
			state = state - 10
		end
	end
	flush()
	return wds
end

local predicates
local function parsefilter(str)
	local f = strwords(str)
	if #f == 1 then return function(e) return predicates.lit.fn(e,f[1]) end end
	if not predicates[f[1]] then
		id10t('no such predicate %s',f[1])
	else
		local p = predicates[f[1]].fn
		return function(e)
			return p(e, table.unpack(f,2))
		end
	end
end

do
	local function p_all(e,pred,...)
		if pred == nil then return true end
		pred = parsefilter(pred)
		if not pred(e) then return false end
		return p_all(e,...)
	end;
	local function p_any(e,pred,...)
		if pred == nil then return false end
		pred = parsefilter(pred)
		if pred(e) then return true end
		return p_any(e,...)
	end;
	local function p_none(e,pred,...)
		if pred == nil then return true end
		pred = parsefilter(pred)
		if pred(e) then return false end
		return p_none(e,...)
	end;
	local function p_some(e,count,pred,...)
		if count == 0 then return true end
		if pred == nil then return false end
		pred = parsefilter(pred)
		if pred(e) then
			count = count-1
		end
		return p_some(e,count,...)
	end;

	local function prepScan(...)
		local map = {}
		local tgt = select('#',...)
		for _,v in pairs{...} do map[v] = true end
		return map,tgt
	end
	predicates = {
		all = {
			fn = p_all;
			syntax = '<pred>…';
			help = 'every sub-<pred> matches'
		};
		any = {
			fn = p_any;
			syntax = '<pred>…';
			help = 'any sub-<pred> matches'
		};
		none = {
			fn = p_none;
			syntax = '<pred>…';
			help = 'no sub-<pred> matches'
		};
		some = {
			fn = p_some;
			syntax = '<count> <pred>…';
			help = '<count> or more sub-<pred>s match'
		};
		def = {
			help = 'word has at least one definition that contains all <keyword>s';
			syntax = '<keyword>…';
			fn = function(e,...)
				local kw = {...}
				for i,d in ipairs(e.word.defs) do
					for j,m in ipairs(d.means) do
						for k,n in ipairs(kw) do
							if not string.find(m.lit, n, 1, true) then
								goto notfound
							end
						end
						do return true end
						::notfound::
					end
				end
				return false
			end;
		};
		lit = {
			help = 'word is, begins with, or ends with <word>';
			syntax = '<word> [(pfx|sfx)]';
			fn = function(e,val,op)
				if not op then
					return e.lit == val
				elseif op == 'pfx' then
					return val == string.sub(e.lit,1,#val)
				elseif op == 'sfx' then
					return val == string.sub(e.lit,(#e.lit) - #val + 1)
				else
					id10t('[lit %s %s] is not a valid filter, “%s” should be either “pfx” or “sfx”',val,op,op)
				end
			end;
		};
		form = {
			help = 'match against word\'s inflected forms';
			syntax = '(<inflect> | <form> (set | is <inflect> | (pfx|sfx|match) <affix>))';
			fn = function(e, k, op, v)
			end;
		};
		part = {
			help = 'word has definitions for every <part> of speech';
			syntax = '<part>…';
			fn = function(e,...)
				local map, tgt = prepScan(...)
				local matches = 0
				for i,d in ipairs(e.word.defs) do
					if map[d.part] then matches = matches + 1 end
				end
				return matches == tgt
			end
		};
		root = {
			help = 'match a word that derives from every <word>';
			syntax = '<word>…';
			fn = function(e,...)
				local map, tgt = prepScan(...)
				for i,d in ipairs(e.word.defs) do
					local matches = 0
					for j,r in ipairs(d.branch) do
						if map[r] then matches = matches + 1 end
					end
					if matches == tgt then return true end
				end
			end
		};
		note = {
			help = 'word has a matching note';
			syntax = '([kind <kind> [<term>]] | term <term> | (min|max|count) <n>)';
			fn = function(e, op, k, t)
				if op == 'kind' or op == 'term' then
					if op == 'term' and t then
						id10t('too many arguments for [note term <term>]')
					end
					for _,d in ipairs(e.word.defs) do
					for _,m in ipairs(d.means) do
					for _,n in ipairs(m.notes) do
						if op=='term' or n.kind == k then
							if op=='kind' and t == nil then return true end
							if string.find(table.concat(n.paras,'\n'), t or k, 1, true) ~= nil then return true end
						end
					end end end
				elseif op == 'min' or op == 'max' or op == 'count' then
					if t then
						id10t('too many arguments for [note %s <n>]',op)
					end
					local n = math.floor(tonumber(k))
					local total = 0
					for i,d in ipairs(e.word.defs) do
					for j,m in ipairs(d.means) do
						total = total + #m.notes
						if op == 'min' and total >= n then return true end
						if op == 'max' and total > n then return false end
					end end
					if op == 'count' then return total == n end
					if op == 'max'   then return total <= n end
					return false
				end
			end;
		};
	}
end

local function
safeopen(file,...)
	if type(file) == 'string' then
		local fd = io.open(file,...)
		if not fd then error(userError("cannot open file " .. file),2) end
		return fd
	else
		return file
	end
end

local function
safeNavWord(ctx, word, dn, mn, nn)
	local w = ctx.dict.words[word]
	if not w then id10t 'bad word' end
	if dn == nil then return w end

	local d = w.defs[tonumber(dn)]
	if not d then id10t('no definition #%u',dn) end
	if mn == nil then return w,d end

	local m = d.means[tonumber(mn)]
	if not m then id10t('no meaning #%u',mn) end
	if nn == nil then return w,d,m end

	local n = m.notes[tonumber(nn)]
	if not n then id10t('no note #%u',nn) end
	return w,d,m,n
end

local function copy(tab)
	local new = {}
	for k,v in pairs(tab) do new[k] = v end
	return new
end

local function parsePath(p)
	local w,dn,mn,nn = p:match('^(.+)@([0-9]+)/([0-9]+):([0-9]+)$')
	if not w then w,dn,mn = p:match('^(.+)@([0-9]+)/([0-9]+)$') end
	if not w then w,dn = p:match('^(.+)@([0-9]+)$') end
	if not w then w=p:match('^(.-)%.?$') end
	return {w = w, dn = tonumber(dn), mn = tonumber(mn), nn = tonumber(nn)}
end

local cmds = {
	create = {
		help = "initialize a new dictionary file";
		syntax = "<lang>";
		raw = true;
		exec = function(ctx, lang)
			if not lang then
				id10t 'for what language?'
			end
			local fd = safeopen(ctx.file,"wb")
			local new = {
				header = {
					lang = lang;
					meta = "";
					partsOfSpeech = {};
					branch = {};
				};
				words = {};
				synonyms = {};
			}
			local o = writeDict(new);
			fd:write(o)
			fd:close()
		end;
	};
	coin = {
		help = "add a new word";
		syntax = "<word>";
		write = true;
		exec = function(ctx,word)
			if ctx.dict.words[word] then
				id10t "word already coined"
			end
			ctx.dict.words[word] = {defs={}}
		end;
	};
	def = {
		help = "define a word";
		syntax = "<word> <part-of-speech> [<meaning> [<root>…]]";
		write = true;
		exec = function(ctx,word,part,means,...)
			local etym = {...}
			if (not word) or not part then
				id10t 'bad definition'
			end
			if not ctx.dict.words[word] then
				ctx.dict.words[word] = {defs={}}
			end
			local n = #(ctx.dict.words[word].defs)+1
			ctx.dict.words[word].defs[n] = {
				part = part;
				branch = etym;
				means = {means and {lit=means,notes={}} or nil};
				forms = {};
			}
			ctx.log('info', string.format('added definition #%u to “%s”', n, word))
		end;
	};
	mean = {
		help = "add a meaning to a definition";
		syntax = "<word> <def#> <meaning>";
		write = true;
		exec = function(ctx,word,dn,m)
			local _,d = safeNavWord(ctx,word,dn)
			table.insert(d.means, {lit=m,notes={}})
		end;
	};
	syn = {
		help = "manage synonym groups";
		syntax = {
			"(show|purge) <path>";
			"(link|drop) <word> <group#> <path>…";
			"new <path> <path>…";
			"clear <word> [<group#>]";
		};
		write = true;
		exec = function(ctx, op, tgtw, ...)
			local groups = {}
			local wp = parsePath(tgtw)
			local w,d = safeNavWord(ctx, wp.w, wp.dn)
			if not (op=='new' or op=='link' or op=='drop' or op=='clear' or op=='show' or op=='purge') then
				id10t('invalid operation “%s” for `syn`', op)
			end
			if op == 'new' then
				local links = {{word = wp.w, def = wp.dn or 1}}
				for i,l in ipairs{...} do
					local parsed = parsePath(l)
					links[i+1] = {word = parsed.w, def = parsed.dn or 1}
				end
				table.insert(ctx.dict.synonyms, {
					uid=math.random(0,0xffffFFFF);
					members=links;
				})
			else -- assemble a list of groups
				for i,ss in ipairs(ctx.dict.synonyms) do
					for j,s in ipairs(ss.members) do
						if s.word == wp.w and (wp.dn == nil or s.def == wp.dn) then
							table.insert(groups, {set = ss, mem = s})
							break
						end
					end
				end

				if op == 'show' then
					for i, g in ipairs(groups) do
						local w,d = safeNavWord(ctx, g.mem.word, g.mem.def)
						local function label(wd,defn)
							local fulldef = {}
							for i,v in ipairs(defn.means) do
								fulldef[i] = v.lit
							end
							fulldef = table.concat(fulldef, '; ')
							return string.format("%s(%s): %s",wd,defn.part,fulldef)
						end
						local others = {}
						for j, o in ipairs(g.set.members) do
							if not (o.word == g.mem.word and o.def == (wp.dn or 1)) then
								local ow, od = safeNavWord(ctx, o.word,o.def)
								table.insert(others, '      '..label(o.word,od))
							end
						end
						io.stdout:write(string.format("% 4u) %s\n%s", i, label(g.mem.word,d),table.concat(others,'\n')))
					end
				elseif op == 'link' or op == 'drop' then
					local tgtn, paths = (...), { select(2, ...) }
				end
			end
		end;
	};
	mod = {
		help = "move, merge, split, or delete words or definitions";
		syntax = {
			"<path> (drop | [move|merge|clobber] <path> | out [<part> [<root>…]])";
			"path ::= <word>[(@<def#>[/<meaning#>[:<note#>]]|.)]";
		};
		write = true;
	};
	note = {
		help = "add a note to a definition or a paragraph to a note";
		syntax = {"(<m-path> (add|for) <kind> | <m-path>:<note#>) <para>…";
			"m-path ::= <word>@<def#>/<meaning#>"};
		write = true;
		exec = function(ctx,path,...)
			local paras, mng
			local dest = parsePath(path)
			local _,_,m = safeNavWord(ctx,dest.w,dest.dn,dest.mn)
			if dest.nn then
				paras = {...}
			else
				local op, kind = ...
				paras = { select(3, ...) }
				if op == 'add' then
					dest.nn = #(m.notes) + 1
					m.notes[dest.nn] = {kind=kind, paras=paras}
					return
				elseif op == 'for' then
					for i,nn in ipairs(m.notes) do
						if nn.kind == kind then
							dest.nn = i break
						end
					end
					if not dest.nn then
						id10t('no note of kind %s in %s',kind,path)
					end
				end
			end
			local dpa = m.notes[dest.nn].paras
			local top = #dpa
			for i,p in ipairs(paras) do
				dpa[top+i] = p
			end
		end
	};
	shell = {
		help = "open an interactive prompt";
		raw = true;
	};
	help = {
		help = "show help";
		nofile = true;
		syntax = "[<command>]";
	};
	predicates = {
		help = "show available filter predicates";
		nofile = true;
		syntax = "[<predicate>]";
	};
	export = {
		help = "create a text file dump compatible with source control";
	};
	dump = {
		exec = function(ctx) print(dump(ctx.dict)) end
	};
	ls = {
		help = "list all words that meet any given <filter>";
		syntax = {"[<filter>…]";
			"filter ::= (<word>|<pred> <arg>…)";
			"arg    ::= (<atom>|'['(<string>|<pred> <arg>…)']')"};
   }
}

function cmds.predicates.exec(ctx, pred)
	local list = predicates
	if pred then list = {predicates[pred]} end
	local f = ctx.sty[io.stderr]
	for k,p in pairs(predicates) do
		if p.help then
			io.stderr:write(
				f.br('  - ' ..
					f.rgb('[',.8,.3,1) ..
					k .. ' ' ..
					(f.color(p.syntax,5) or '…') ..
					f.rgb(']',.8,.3,1)) .. ': ' ..
				f.color(p.help,4,true) .. '\n')
		end
	end
end

function cmds.ls.exec(ctx,...)
	local filter = nil
	local out = {}
	for i,f in ipairs{...} do
		local fn = parsefilter(f)
		local of = filter or function() return false end
		filter = function(e)
			return fn(e) or of(e)
		end
	end
	for lit,w in pairs(ctx.dict.words) do
		local e = {lit=lit, word=w}
		if filter == nil or filter(e) then
			table.insert(out, e)
		end
	end
	table.sort(out, function(a,b) return a.lit < b.lit end)
	local fo = ctx.sty[io.stdout]
	local function meanings(d,md,n)
		local start = md and 2 or 1
		local part = string.format('(%s)', d.part)
		local pad = md and string.rep(' ', #part) or ''
		local function note(n,insert)
			if not next(n.paras) then return end
			local pad = string.rep(' ',#(n.kind) + 9)
			insert('      ' .. fo.hl(' ' .. n.kind .. ' ') .. ' ' .. n.paras[1])
			for i=2,#n.paras do
				insert(pad..n.paras[2])
			end
		end
		local m = { (function()
			if d.means[1] then
				if md then return
					string.format("  %s 1. %s", fo.em(part), d.means[1].lit)
				end
			else return
				fo.em(string.format('  %s [empty definition #%u]', part,n))
			end
		end)() }
		for i=start,#d.means do local v = d.means[i]
			table.insert(m, string.format('  %s %u. %s', pad, i, v.lit))
			for j,n in ipairs(v.notes) do
				note(n, function(v) table.insert(m, v) end)
			end
		end
		return table.concat(m,'\n')
	end
	for i, w in ipairs(out) do
		local d = fo.ul(w.lit)
		if #w.word.defs == 1 then
			d=d .. ' ' .. fo.em('('..(w.word.defs[1].part)..')') ..'\n'
				.. meanings(w.word.defs[1],false,1)
		else
			for j, def in ipairs(w.word.defs) do
				d=d .. '\n' .. meanings(def,true,j)
			end
		end
		io.stdout:write(d..'\n')
	end
end

function cmds.export.exec(ctx)
	local function san(str)
		local d = 0
		local r = {}
		for i,cp in utf8.codes(str) do
			-- insert backslashes for characters that would
			-- disrupt strwords() parsing
			if cp == 0x5b then
				d = d + 1
			elseif cp == 0x5d then
				if d >= 1 then
					d = d - 1
				else
					table.insert(r, 0x5c)
				end
			end
			table.insert(r, cp)
		end
		return '[' .. utf8.char(table.unpack(r)) .. ']'
	end
	local function o(...) io.stdout:write(string.format(...)..'\n') end
	local d = ctx.dict
	o('pv0 %s %s', san(d.header.lang), san(d.header.meta))
	for lit, w in pairs(d.words) do
		o('w %s',san(lit))
		for i,def in ipairs(w.defs) do
			o('d %s',san(def.part))
			for _,s in ipairs(d.synonyms) do
				for _,sm in ipairs(s.members) do
					if sm.word == w and sm.def == i then
						o('ds %u',s.uid)
						break
					end
				end
			end
			for j,r in ipairs(def.branch) do
				o('dr %s',san(r))
			end
			for j,m in ipairs(def.means) do
				o('m %s', san(m.lit))
				for k,n in ipairs(m.notes) do
					o('n %s', san(n.kind))
					for a,p in ipairs(n.paras) do
						o('np %s', san(p))
					end
				end
			end
		end
	end
	for _,s in ipairs(d.synonyms) do o('s %u', s.uid) end
end

function cmds.mod.exec(ctx, orig, oper, dest, ...)
	if (not orig) or not oper then
		id10t '`mod` requires at least an origin and an operation'
	end
	local op, dp = parsePath(orig)
	local w,d,m,n = safeNavWord(ctx, op.w,op.dn,op.mn,op.nn)
	if oper == 'drop' then
		if not d then
			ctx.dict.words[op.w] = nil
		elseif not m then
			table.remove(w.defs, op.dn)
		elseif not n then
			table.remove(d.means, op.mn)
		else
			table.remove(m.notes, op.nn)
		end
	elseif oper == 'out' then
		if n or not m then
			id10t '`mod out` must target a meaning'
		end
		if not dest then id10t '`mod out` requires at least a part of speech' end
		local newdef = {
			part = dest;
			branch = {...};
			forms = {};
			means = {m};
		}
		table.insert(w.defs,op.dn+1, newdef)
		table.remove(d.means,op.mn)
	elseif oper == 'move' or oper == 'merge' or oper == 'clobber' then
		if dest
			then dp = parsePath(dest)
			else id10t('`mod %s` requires a target',oper)
		end
		if n then
			if not dp.mn then
				id10t '`mod` on a note requires a note or meaning destination'
			end
			local _,_,dm = safeNavWord(ctx, dp.w,dp.dn,dp.mn)
			if dp.nn then
				if oper == 'move' then
					table.insert(dm.notes, dp.nn, n)
				elseif oper == 'merge' then
					local top = #(dm.notes[dp.nn].paras)
					for i, v in ipairs(n.paras) do
						dm.notes[dp.nn].paras[i+top] = v
					end
				elseif oper == 'clobber' then
					dm.notes[dp.nn] = n
				end
			else
				if oper ~= 'move' then
					id10t('`mod note %s` requires a note target', oper)
				end
				table.insert(dm.notes, n)
			end
			if oper == 'move' and dp.nn and dm == m and op.nn > dp.nn then
				table.remove(m.notes,op.nn+1)
			else
				table.remove(m.notes,op.nn)
			end
		elseif m then
			if not dp.dn then
				local newdef = {
					part = d.part;
					branch = copy(d.branch);
					forms = copy(d.forms);
					means = {m};
				}
				if ctx.dict.words[dp.w] then
					table.insert(ctx.dict.words[dp.w].defs, newdef)
				else
					ctx.dict.words[dp.w] = {
						defs = {newdef};
					}
				end
				table.remove(d.means,dp.mn)
			else
				local dw, dd = safeNavWord(ctx, dp.w, dp.dn)
				if dp.mn then
					if dd.means[dp.mn] and (oper == 'merge' or oper=='clobber') then
						if oper == 'merge' then
							dd.means[dp.mn] = dd.means[dp.mn] .. '; ' .. m
						elseif oper == 'clobber' then
							dd.means[dp.mn] = m
						end
					else
						if oper == clobber then dd.means = {} end
						table.insert(dd.means, dp.mn, m)
					end
				else
					table.insert(dd.means, m)
				end
				if oper == 'move' and dp.mn and dd.means == d.means and op.mn > dp.mn then
					table.remove(d.means,op.mn+1)
				else
					table.remove(d.means,op.mn)
				end
			end
		elseif d then
			local ddefs = safeNavWord(ctx, dp.w).defs
			if dp.dn then
				if oper == 'merge' then
					local top = #(ddefs[dp.dn].means)
					for i,om in ipairs(d.means) do
						ddefs[dp.dn].means[top+i] = om
					end
					for k,p in pairs(d.forms) do
						ddefs[dp.dn].forms[k] = p -- clobbers!
					end
				else
					table.insert(ddefs, dp.dn, d)
				end
			else
				table.insert(ddefs, d)
			end
			if oper == 'move' and dp.mn and w.defs == ddefs and op.mn > dp.mn then
				table.remove(w.defs,op.dn+1)
			else
				table.remove(w.defs,op.dn)
			end
		else
			if ctx.dict.words[dp.w] then
				if oper ~= 'merge' then
					id10t('the word “%s” already exists; use `merge` if you want to merge the words together', dp.w)
				end
				for i,def in ipairs(w.defs) do
					if dp.dn then
						table.insert(ctx.dict.words[dp.w].defs, dp.dn+i-1, def)
					else
						table.insert(ctx.dict.words[dp.w].defs, def)
					end
				end
			else
				ctx.dict.words[dp.w] = w
			end
			ctx.dict.words[op.w] = nil
		end
	end
end

local function fileLegible(file)
	-- check if we can access the file
	local fd = io.open(file,"rb")
	local ret = false
	if fd then ret = true end
	fd:close()
	return ret
end

local function map(fn,lst)
	local new = {}
	for k,v in pairs(lst) do
		local nv, nk = fn(v,k)
		new[nk or k] = nv
	end
	return new
end
local function mapD(fn,lst) --destructive
	-- WARNING: this will not work if nk names an existing key!
	for k,v in pairs(lst) do
		local nv, nk = fn(v,k)
		if nk == nil or k == nk then
			lst[k] = nv
		else
			lst[k] = nil
			lst[nk] = nv
		end
	end
	return lst
end

local function
prompt(p,multiline)
	-- returns string if successful, nil if EOF, false if ^C
	io.stderr:write(p)
	local ok, res = pcall(function()
	                      return io.stdin:read(multiline and 'a' or 'l')
	                     end)
	if ok then return res end
	return false
end

function cmds.shell.exec(ctx)
	if not fileLegible(ctx.file) then
		-- avoid accidentally creating a file without the
		-- proper document structure and metadata
		id10t("file %s must already exist and be at least readable", ctx.file)
	end

	local fd, rw = io.open(ctx.file,"r+b"), true
	if not fd then -- not writable
		ctx.log('warn',string.format('file %s is not writable', ctx.file))
		fd, rw = io.open(ctx.file, "rb"), false
	end
	ctx.fd = fd
	ctx.dict = readDict(fd:read 'a')
	fd:close()

	local written = false
	local fo = ctx.sty[io.stdout]
	local fe = ctx.sty[io.stderr]
	repeat
		local cmd = prompt(fe.br(string.format('(parvan %s) ', ctx.file)))
		if cmd == false then
			io.stderr:write(fe.resetLine())
			if written then
				ctx.log('warn', 'abandoning changes!')
			end
			return 0
		end
		if cmd and cmd ~= '' then
			local words = strwords(cmd)
			if next(words) then
				if words[1] == 'bail'    or
				   words[1] == 'abandon' or
				   words[1] == 'q!'     then
					if written then
						ctx.log('warn', 'abandoning changes!')
					end
					return 0
				end
				local c = cmds[words[1]]
				if c then
					if c.raw then
						ctx.log('fatal', words[1] .. ' cannot be run from `shell`')
					elseif not implies(c.write, rw) then
						ctx.log('fatal', ctx.file .. ' is not writable')
					else
						local ok = ctx.try(c.exec, ctx, table.unpack(words,2))
						if ok then written = written or c.write end
					end
				elseif cmd == 'save' or cmd == 'wq' then
					if not written then
						ctx.log('info', 'no changes to save')
					end
					cmd = nil
				elseif cmd == 'quit' or cmd == 'q' then
					if not written then cmd = nil else
						ctx.log('fatal', 'dictionary has unsaved changes')
					end
				else
					ctx.log('fatal', words[1] .. ' is not a command')
				end
			end
		end
	until cmd == nil

	if written then
		ctx.log('info', 'saving file')
		local out = writeDict(ctx.dict)
		local fd = io.open(ctx.file,'w+b')
		fd:write(out)
		fd:close()
	end
end

local function
showHelp(ctx,k,v)
	if not v then
		id10t 'no such command'
	end

	if v.help then
		local fe = ctx.sty[io.stderr]
		local defs, synt = ''
		if type(v.syntax) == 'table' then
			synt = v.syntax[1]
			local pad = string.rep(' ', #k+5)
			for i=2,#v.syntax do
				defs = defs .. pad .. fe.color(v.syntax[i],5) .. '\n'
			end
		else synt = v.syntax end

		io.stderr:write(string.format(
			"  > %s %s\n" .. defs ..
			"    %s\n",
			fe.br(k), synt and fe.br(fe.color(synt,5)) or '',
			fe.em(fe.color(v.help,4,true))))
	end
end

function cmds.help.exec(ctx,cmd)
	if cmd then
		showHelp(ctx, cmd, cmds[cmd])
	else
		for cmd,c in pairs(cmds) do
			showHelp(ctx, cmd, c)
		end
	end
end

local function
usage(me,ctx)
	local ln = 0
	local ct = {}
	local fe = ctx.sty[io.stderr]
	io.stderr:write(string.format(fe.br"usage:".." %s <file> [<command> [args…]]\n",me))
	--[[
	for k,v in pairs(cmds) do
		local n = 1 + utf8.len(k) + utf8.len(v.syntax)
		ct[k] = n
		if n > ln then ln = n end
	end
	for k,v in pairs(cmds) do
		local pad = string.rep(" ", ln - ct[k] + 3)
		io.stderr:write(string.format("   "..fe.br'%s %s'.."%s%s\n",
			k, v.syntax, pad, v.help))
	end]]
	for k,v in pairs(cmds) do
		showHelp(ctx,k,v)
	end
	return 64
end

local function
dispatch(argv, ctx)
	local ferr = ctx.sty[io.stderr]
	local file, cmd = table.unpack(argv)
	if cmd and cmds[cmd] then
		local c,fd,dict = cmds[cmd]
		if (not c.raw) and not c.nofile then
			fd = safeopen(file, "rb")
			dict = readDict(fd:read 'a')
			fd:close()
			-- lua io has no truncate method, so we must
			-- rely on the clobbering behavior of the open()
			-- call instead :(
		end

		cmds[cmd].exec({
			sty = ctx.sty;
			try = ctx.try;
			log = ctx.log;

			file = file;
			fd = fd;
			dict = dict;
		}, table.unpack(argv,3))

		if (not c.raw) and c.write then
			local output = writeDict(dict)
			-- writeDict should always be given a chance to
			-- bail before the previous file is destroyed!!
			-- you don't want one bug to wipe out your entire
			-- dictionary in one fell swoop
			fd = safeopen(file,'w+b')
			fd:write(output)
			fd:close()
		end

		return 0
	else
		return usage(argv[0], ctx)
	end
end


local argv if arg
	then argv = arg
	else argv = {[0] = 'parvan', ...}
end

local sty = {
	[io.stdout] = ansi.formatter(io.stdout);
	[io.stderr] = ansi.formatter(io.stderr);
};

local function log(lvl, msg)
	local colors = {fatal=1,warn=3,info=4,debug=2}
	local ferr = sty[io.stderr]
	io.stderr:write(string.format(
		ferr.color(ferr.br("(%s)"),colors[lvl]).." %s\n", lvl, msg))
end
local function try(...)
	-- a wrapper around pcall that produces a standard error
	-- message format when an error occurs
	local res = { pcall(...) }
	if not res[1] then
		log('fatal', res[2])
	end
	return table.unpack(res)
end

local function stacktrace(err)
	return debug.traceback(err,3)
end
local ok, res = xpcall(dispatch, stacktrace, argv, {
	try = try, sty = sty, log = log
})

if not ok then
	log('fatal', res)
	os.exit(1)
end

os.exit(res)