util  Diff

Differences From Artifact [2e5da05ad6]:

To Artifact [30400bd2f4]:


     8      8   --      SSD  Social Sciences Directorate
     9      9   --       ELS External Linguistics Subdirectorate
    10     10   --    +WSO   Worlds Security Overdirectorate
    11     11   --      EID  External Influence Directorate ]
    12     12   
    13     13   local function implies(a,b) return a==b or not(a) end
    14     14   
           15  +local function map(lst,fn)
           16  +	local new = {}
           17  +	for k,v in pairs(lst) do
           18  +		local nv, nk = fn(v,k)
           19  +		new[nk or k] = nv
           20  +	end
           21  +	return new
           22  +end
           23  +local function mapD(lst,fn) --destructive
           24  +	-- WARNING: this will not work if nk names an existing key!
           25  +	for k,v in pairs(lst) do
           26  +		local nv, nk = fn(v,k)
           27  +		if nk == nil or k == nk then
           28  +			lst[k] = nv
           29  +		else
           30  +			lst[k] = nil
           31  +			lst[nk] = nv
           32  +		end
           33  +	end
           34  +	return lst
           35  +end
           36  +local function pushi(dest, idx, src, ...)
           37  +	if not src then return end
           38  +	dest[idx] = src
           39  +	pushi(dest, idx+1, ...)
           40  +end
           41  +local function push(dest, ...) pushi(dest,#dest+1,...) end
           42  +local function cons(car, cdr)
           43  +	local new = {car}
           44  +	for k,v in ipairs(cdr) do new[k+1] = v end
           45  +	return new
           46  +end
           47  +local function tcatD(dest, ...)
           48  +	local i = #dest
           49  +	local function iter(src, ...)
           50  +		if src == nil then return end
           51  +		local sc = #src
           52  +		for j=1,sc do dest[i+j] = src[j] end
           53  +		i = i + sc
           54  +		iter(...)
           55  +	end
           56  +	iter(...)
           57  +end
           58  +local function tcat(...)
           59  +	local new = {}
           60  +	tcatD(new, ...)
           61  +	return new
           62  +end
    15     63   local ansi = {
    16     64   	levels = {
    17     65   		plain = 0;
    18     66   		ansi = 1;
    19     67   		color = 2;
    20     68   		color8b = 3;
    21     69   		color24b = 4;
    22     70   	};
    23     71   }
    24     72   
    25     73   ansi.seqs = {
    26         -	br = {ansi.levels.ansi, "[1m", "[21m"};
           74  +	br = {ansi.levels.ansi, "[1m", "[22m"};
    27     75   	hl = {ansi.levels.ansi, "[7m", "[27m"};
    28     76   	ul = {ansi.levels.ansi, "[4m", "[24m"};
    29     77   	em = {ansi.levels.ansi, "[3m", "[23m"};
    30     78   };
    31     79   
    32     80   function ansi.termclass(fd) -- awkwardly emulate isatty
    33     81   	if fd:seek('cur',0) then
................................................................................
    83    131   	if cl == ansi.levels.color24b then
    84    132   		function f.rgb(str, r,g,b, bg)
    85    133   			return string.format("\27[%c8;2;%u;%u;%um", bg and 0x34 or 0x33,
    86    134   				ftoi(r,g,b)) .. str .. reset
    87    135   		end
    88    136   	elseif cl == ansi.levels.color8b then
    89    137   		function f.rgb(str, r,g,b, bg)
    90         -			local code = 16 + (r * 5)*36 + (g * 5)*6 + (b * 6)
          138  +			local code = 16 + math.floor(r * 5)*36 + math.floor(g * 5)*6 + math.floor(b * 6)
    91    139   			return string.format("\27[%c8;5;%um", bg and 0x34 or 0x33, code)
    92    140   				.. str .. reset
    93    141   		end
    94    142   	elseif cl == ansi.levels.color then
    95    143   		function f.rgb(str, r,g,b, bg)
    96    144   			local code = 0x30 + 1 -- TODO
    97    145   			return string.format("\27[%c%cm", bg and 0x34 or 0x33, code)
................................................................................
   105    153   	return f
   106    154   end
   107    155   
   108    156   
   109    157   local function dump(v,pfx,cyc,ismeta)
   110    158   	pfx = pfx or ''
   111    159   	cyc = cyc or {}
   112         -	local np = pfx .. '  '
          160  +	local np = pfx .. '   '
   113    161   
   114    162   	if type(v) == 'table' then
   115    163   		if cyc[v] then return '<...>' else cyc[v] = true end
   116    164   	end
   117    165   
   118    166   	if type(v) == 'string' then
   119    167   		return string.format('%q', v)
................................................................................
   126    174   		local meta = ''
   127    175   		if getmetatable(v) then
   128    176   			meta = dump(getmetatable(v),pfx,cyc,true) .. '::'
   129    177   		end
   130    178   		if ismeta then
   131    179   			return string.format('%s<|\n%s%s|>',meta,str,pfx)
   132    180   		else
   133         -			return meta..'{\n' .. str .. pfx .. '}\n'
          181  +			return meta..'{\n' .. str .. pfx .. '}'
   134    182   		end
   135    183   	else
   136    184   		return string.format('%s', v)
   137    185   	end
   138    186   end
   139    187   
   140    188   local struct = {
................................................................................
   186    234   fmt.string = qpack "s4"
   187    235   fmt.label = qpack "s2"
   188    236   fmt.tag = qpack "s1"
   189    237   fmt.u8 = qpack "I1"
   190    238   fmt.u16 = qpack "I2"
   191    239   fmt.u24 = qpack "I3"
   192    240   fmt.u32 = qpack "I4"
          241  +fmt.path = {
          242  +	-- encodes a FIXED path to an arbitrary type of object
          243  +	encode = function(a)
          244  +		local kind = 0
          245  +		local vals = {}
          246  +		if a.w then kind = 1
          247  +			table.insert(vals, marshal(fmt.label, a.w))
          248  +			if a.dn then kind = 2
          249  +				table.insert(vals, marshal(fmt.u8, a.dn))
          250  +				if a.mn then kind = 3
          251  +					table.insert(vals, marshal(fmt.u8, a.mn))
          252  +					if a.nn then kind = 4
          253  +						table.insert(vals, marshal(fmt.u8, a.nn))
          254  +					end
          255  +				end
          256  +			end
          257  +		end
          258  +		return marshal(fmt.u8,kind) .. table.concat(vals)
          259  +	end;
          260  +	decode = function(s)
          261  +		local kind <const> = parse(fmt.u8, s)
          262  +		local path = {}
          263  +		local components <const> = {
          264  +			{'w',fmt.label};
          265  +			{'dn',fmt.u8};
          266  +			{'mn',fmt.u8};
          267  +			{'nn',fmt.u8};
          268  +		}
          269  +		for i=1,kind do
          270  +			local label, ty = table.unpack(components[i])
          271  +			path[label] = parse(ty,s)
          272  +		end
          273  +		return path
          274  +	end;
          275  +}
   193    276   fmt.list = function(t,ty) ty = ty or fmt.u32
   194    277   	return {
   195    278   		encode = function(a)
   196    279   			local vals = {marshal(ty, #a)}
   197    280   			for i=1,#a do
   198    281   				table.insert(vals, marshal(t, a[i]))
   199    282   			end
................................................................................
   228    311   			local m = {}
   229    312   			for _,p in pairs(lst) do m[p.key] = p.val end
   230    313   			return m
   231    314   		end;
   232    315   	}
   233    316   end
   234    317   
   235         -fmt.form = {
   236         -	{'form', fmt.u16};
   237         -	{'text', fmt.label};
   238         -}
          318  +fmt.enum = function(...)
          319  +	local vals,rmap = {...},{}
          320  +	for k,v in pairs(vals) do rmap[v] = k-1 end
          321  +	local ty = fmt.u8
          322  +	if #vals > 0xffff then ty = fmt.u32 -- just in pathological case
          323  +	elseif #vals > 0xff then ty = fmt.u16 end
          324  +	return {
          325  +		encode = function(a)
          326  +			if not rmap[a] then error(string.format('"%s" is not part of enum "%s"', a, table.concat(vals,'","')),3) end
          327  +			return marshal(ty, rmap[a])
          328  +		end;
          329  +		decode = function(s)
          330  +			local n = parse(ty,s)
          331  +			if (n+1) > #vals then error(string.format('enum "%s" does not have %u members', table.concat(vals,'","'),n),3) end
          332  +			return vals[n+1]
          333  +		end;
          334  +	}
          335  +end
   239    336   
   240    337   fmt.note = {
   241    338   	{'kind', fmt.tag};
   242    339   	{'paras', fmt.list(fmt.string)};
   243    340   }
   244    341   
          342  +fmt.example = {
          343  +	{'quote',fmt.string};
          344  +	{'src',fmt.label};
          345  +}
   245    346   fmt.meaning = {
   246    347   	{'lit', fmt.string};
          348  +	{'examples', fmt.list(fmt.example,fmt.u8)};
   247    349   	{'notes', fmt.list(fmt.note,fmt.u8)};
   248    350   }
          351  +
          352  +fmt.phrase = {
          353  +	{'str',fmt.label};
          354  +	{'means',fmt.list(fmt.meaning,fmt.u8)};
          355  +	{'xref',fmt.list(fmt.path,fmt.u16)};
          356  +}
   249    357   
   250    358   fmt.def = {
   251    359   	{'part', fmt.u8};
   252    360   	{'branch', fmt.list(fmt.label,fmt.u8)};
   253    361   	{'means', fmt.list(fmt.meaning,fmt.u8)};
   254         -	{'forms', fmt.list(fmt.form,fmt.u16)};
          362  +	{'forms', fmt.map(fmt.u16,fmt.label,fmt.u16)};
          363  +	{'phrases', fmt.list(fmt.phrase,fmt.u16)};
   255    364   }
   256    365   
   257    366   fmt.word = {
   258    367   	{'defs', fmt.list(fmt.def,fmt.u8)};
   259    368   }
   260    369   
   261    370   fmt.dictHeader = {
   262    371   	{'lang', fmt.tag};
   263    372   	{'meta', fmt.string};
   264    373   	{'partsOfSpeech', fmt.list(fmt.tag,fmt.u16)};
          374  +	{'inflectionForms', fmt.list({
          375  +		{'name', fmt.tag};
          376  +		{'abbrev', fmt.tag};
          377  +		{'desc', fmt.string};
          378  +		{'parts', fmt.list(fmt.tag,fmt.u8)};
          379  +		-- which parts of speech does this form apply to?
          380  +		-- leave empty if not relevant
          381  +	},fmt.u16)};
   265    382   }
   266    383   
   267         -fmt.synonymSet = {
          384  +fmt.relSet = {
   268    385   	{'uid', fmt.u32};
   269    386   		-- IDs are persistent random values so they can be used
   270    387   		-- as reliable identifiers even when merging exports in
   271    388   		-- a parvan-unaware VCS
   272         -	{'members', fmt.list({
   273         -		{'word', fmt.label}, {'def', fmt.u8};
   274         -	},fmt.u16)};
          389  +	{'members', fmt.list(fmt.path,fmt.u16)};
          390  +	{'kind', fmt.enum('syn','ant','met')};
   275    391   }
   276    392   
   277    393   fmt.dict = {
   278    394   	{'header', fmt.dictHeader};
   279    395   	{'words', fmt.map(fmt.string,fmt.word)};
   280         -	{'synonyms', fmt.list(fmt.synonymSet)};
          396  +	{'relsets', fmt.list(fmt.relSet)};
   281    397   }
   282    398   
   283    399   function marshal(ty, val)
   284    400   	if ty.encode then
   285    401   		return ty.encode(val)
   286    402   	end
   287    403   	local ac = {}
   288    404   
   289    405   	for idx,fld in ipairs(ty) do
   290    406   		local name, fty = table.unpack(fld)
   291         -		table.insert(ac, marshal(fty, assert(val[name])))
          407  +		table.insert(ac, marshal(fty,
          408  +			assert(val[name],
          409  +				string.format('marshalling error: missing field %s', name)
          410  +			)
          411  +		))
   292    412   	end
   293    413   
   294    414   	return table.concat(ac)
   295    415   end
   296    416   
   297    417   function parse(ty, stream)
   298    418   	if ty.decode then
................................................................................
   315    435   		if map[v] then return map[v] else
   316    436   			map[v] = i
   317    437   			i=i+1
   318    438   			return i-1
   319    439   		end
   320    440   	end, map
   321    441   end
          442  +
          443  +local function rebuildRelationCache(d)
          444  +-- (re)build a dictionary's relation cache; needed
          445  +-- at load time and whenever any changes to relsets
          446  +-- are made (unless they're simple enough to update
          447  +-- the cache directly by hand, but that's very eeeh)
          448  +	local sc = {}
          449  +	for i,s in ipairs(d.relsets) do
          450  +		for j,m in ipairs(s.members) do
          451  +			sc[m.w] = sc[m.w] or {}
          452  +			table.insert(sc[m.w], s)
          453  +		end
          454  +	end
          455  +	d._relCache = sc
          456  +end
   322    457   
   323    458   local function
   324    459   writeDict(d)
   325    460   	local atomizePoS, posMap = atomizer()
   326    461   	for lit,w in pairs(d.words) do
   327    462   		for j,def in ipairs(w.defs) do
   328    463   			def.part = atomizePoS(def.part)
................................................................................
   345    480   	local d = parse(fmt.dict, s)
   346    481   	-- handle atoms
   347    482   	for lit,w in pairs(d.words) do
   348    483   		for j,def in ipairs(w.defs) do
   349    484   			def.part = d.header.partsOfSpeech[def.part]
   350    485   		end
   351    486   	end
          487  +
          488  +	-- create cachemaps for complex data structures to
          489  +	-- enable faster lookup that would otherwise require
          490  +	-- expensive scans
          491  +	rebuildRelationCache(d)
   352    492   	return d
   353    493   end
          494  +
   354    495   
   355    496   local function strwords(str) -- here be dragons
   356    497   	local wds = {}
   357    498   	local w = {}
   358    499   	local state, d, quo, dquo = 0,0
   359         -	local function flush(n)
   360         -		if next(w) then
          500  +	local function flush(n,final)
          501  +		if next(w) or state ~= 0 and state < 10 then
   361    502   			table.insert(wds, utf8.char(table.unpack(w)))
   362    503   			w = {}
          504  +		elseif final and state > 10 then
          505  +			table.insert(wds, '\\')
   363    506   		end
   364    507   		state = n
   365    508   		quo = nil
   366    509   		dquo = nil
   367    510   		d = 0
   368    511   	end
   369    512   	local function isws(c)
................................................................................
   418    561   				table.insert(w,0x0a)
   419    562   			else
   420    563   				table.insert(w,cp)
   421    564   			end
   422    565   			state = state - 10
   423    566   		end
   424    567   	end
   425         -	flush()
          568  +	flush(nil,true)
   426    569   	return wds
   427    570   end
   428    571   
   429    572   local predicates
   430    573   local function parsefilter(str)
   431    574   	local f = strwords(str)
   432    575   	if #f == 1 then return function(e) return predicates.lit.fn(e,f[1]) end end
................................................................................
   631    774   
   632    775   local function copy(tab)
   633    776   	local new = {}
   634    777   	for k,v in pairs(tab) do new[k] = v end
   635    778   	return new
   636    779   end
   637    780   
   638         -local function parsePath(p)
   639         -	local w,dn,mn,nn = p:match('^(.+)@([0-9]+)/([0-9]+):([0-9]+)$')
   640         -	if not w then w,dn,mn = p:match('^(.+)@([0-9]+)/([0-9]+)$') end
   641         -	if not w then w,dn = p:match('^(.+)@([0-9]+)$') end
          781  +local function pathParse(p)
          782  +-- this is cursed, rewrite without regex pls TODO
          783  +	if p == '.' then return {} end
          784  +	local function comp(pfx)
          785  +		return pfx .. '([0-9]+)'
          786  +	end
          787  +	local function mtry(...)
          788  +		local mstr = '^(.+)'
          789  +		for _, v in ipairs{...} do
          790  +			mstr = mstr .. comp(v)
          791  +		end
          792  +		return p:match(mstr .. '$')
          793  +	end
          794  +
          795  +	local xn
          796  +	local w,dn,pn,mn,nn = mtry('%.','/p','/m','/n')
          797  +	if not w then w,dn,pn,mn,xn = mtry('%.','/p','/m','/x') end
          798  +	if not w then w,dn,pn,mn = mtry('%.','/p','/m') end
          799  +	if not w then w,dn,pn= mtry('%.','/p') end
          800  +	if not w then
          801  +		local comps = {'%.','/m','/n'}
          802  +		for i=#comps, 1, -1 do
          803  +			local args = {table.unpack(comps,1,i)}
          804  +			w,dn,mn,nn = mtry(table.unpack(args))
          805  +			if not w and args[i] == '/n' then
          806  +				args[i] = '/x'
          807  +				w,dn,mn,xn = mtry(table.unpack(args))
          808  +			end
          809  +			if w then break end
          810  +		end
          811  +	end
   642    812   	if not w then w=p:match('^(.-)%.?$') end
   643         -	return {w = w, dn = tonumber(dn), mn = tonumber(mn), nn = tonumber(nn)}
          813  +	return {w = w, dn = tonumber(dn), mn = tonumber(mn), pn=tonumber(pn); nn = tonumber(nn), xn = tonumber(xn)}
          814  +end
          815  +local function pathString(p,styler)
          816  +	local function s(s, st, ...)
          817  +		if styler then
          818  +			return styler[st](tostring(s),...)
          819  +		else return s end
          820  +	end
          821  +
          822  +	local function comp(c,n,...)
          823  +		return s('/','color',5)
          824  +		    .. s(string.format("%s%u",c,n), 'color',...)
          825  +	end
          826  +	local t = {}
          827  +	if p.w then t[1] = s(p.w,'ul') else return '.' end
          828  +	if p.dn then t[2] = string.format(".%s", s(p.dn,'br')) end
          829  +	if p.pn then t[#t+1] = comp('p',p.pn,4,true) end
          830  +	if p.mn then t[#t+1] = comp('m',p.mn,5,true) end
          831  +	if p.xn then t[#t+1] = comp('x',p.xn,6,true)
          832  +	elseif p.nn then t[#t+1] = comp('n',p.nn,4) end
          833  +	if t[2] == nil then
          834  +		return p.w .. '.' --make sure paths are always valid
          835  +	end
          836  +	return s(table.concat(t),'em')
          837  +end
          838  +local function pathMatch(a,b)
          839  +	return a.w  == b.w
          840  +		and a.dn == b.dn
          841  +		and a.mn == b.mn
          842  +		and a.pn == b.pn
          843  +		and a.nn == b.nn
          844  +		and a.xn == b.xn
          845  +end
          846  +local function pathResolve(ctx, a)
          847  +	if not a.w then return end -- empty paths are valid!
          848  +	local function lookup(seg, tbl,val)
          849  +		if not tbl then error('bad table',2) end
          850  +		local v = tbl[val]
          851  +		if v then return v end
          852  +		id10t('bad %s in path: %s', seg, val)
          853  +	end
          854  +
          855  +	local res = {}
          856  +	res.word = lookup('word', ctx.dict.words, a.w)
          857  +	if not a.dn then return res end
          858  +
          859  +	res.def = lookup('definition', w.defs, a.dn)
          860  +	if (not a.pn) and (not a.mn) then return res end
          861  +
          862  +	local m if a.pn then
          863  +		res.phrase = lookup('phrase', d.phrases, a.pn)
          864  +		res.meaning = lookup('meaning', p.means, a.mn)
          865  +	else
          866  +		res.meaning = lookup('meaning', d.means, a.mn)
          867  +	end
          868  +
          869  +	if a.xn then
          870  +		res.ex = lookup('example',m.examples,a.xn)
          871  +	elseif a.nn then
          872  +		res.note = lookup('note',m.notes,a.nn)
          873  +	end
          874  +
          875  +	return res
          876  +end
          877  +
          878  +local function pathNav(...)
          879  +	local t = pathResolve(...)
          880  +	return t.word,t.def,t.phrase,t.meaning,t.ex or t.note
          881  +end
          882  +
          883  +local function pathRef(ctx, a)
          884  +	local w,d,p,m,n = pathNav(ctx,a)
          885  +	return n or m or p or d or w
          886  +end
          887  +
          888  +local function pathSub(super,sub)
          889  +	if super.w == nil then return true end
          890  +	if sub.w ~= super.w then return false end
          891  +
          892  +	if super.pn == nil then goto checkMN end
          893  +	if sub.pn ~= super.pn then return false end
          894  +
          895  +	::checkMN::
          896  +	if super.mn == nil then return true end
          897  +	if sub.mn ~= super.mn then return false end
          898  +
          899  +	if super.xn then
          900  +		if sub.nn then return false end
          901  +		if sub.xn ~= super.xn then return false end
          902  +	elseif super.nn then
          903  +		if sub.xn then return false end
          904  +		if sub.nn ~= super.nn then return false end
          905  +	end
          906  +
          907  +	return true
   644    908   end
   645    909   
   646    910   local cmds = {
   647    911   	create = {
   648    912   		help = "initialize a new dictionary file";
   649    913   		syntax = "<lang>";
   650    914   		raw = true;
................................................................................
   654    918   			end
   655    919   			local fd = safeopen(ctx.file,"wb")
   656    920   			local new = {
   657    921   				header = {
   658    922   					lang = lang;
   659    923   					meta = "";
   660    924   					partsOfSpeech = {};
   661         -					branch = {};
          925  +					inflectionForms = {};
   662    926   				};
   663    927   				words = {};
   664         -				synonyms = {};
          928  +				relsets = {};
   665    929   			}
   666    930   			local o = writeDict(new);
   667    931   			fd:write(o)
   668    932   			fd:close()
   669    933   		end;
   670    934   	};
   671    935   	coin = {
................................................................................
   691    955   			if not ctx.dict.words[word] then
   692    956   				ctx.dict.words[word] = {defs={}}
   693    957   			end
   694    958   			local n = #(ctx.dict.words[word].defs)+1
   695    959   			ctx.dict.words[word].defs[n] = {
   696    960   				part = part;
   697    961   				branch = etym;
   698         -				means = {means and {lit=means,notes={}} or nil};
          962  +				means = {means and {
          963  +					lit=means;
          964  +					examples={};
          965  +					notes={};
          966  +				} or nil};
   699    967   				forms = {};
   700    968   			}
   701    969   			ctx.log('info', string.format('added definition #%u to “%s”', n, word))
   702    970   		end;
   703    971   	};
   704    972   	mean = {
   705    973   		help = "add a meaning to a definition";
   706    974   		syntax = "<word> <def#> <meaning>";
   707    975   		write = true;
   708    976   		exec = function(ctx,word,dn,m)
   709         -			local _,d = safeNavWord(ctx,word,dn)
   710         -			table.insert(d.means, {lit=m,notes={}})
          977  +			local t = pathResolve(ctx,{w=word,dn=dn})
          978  +			table.insert(t.d.means, {lit=m,notes={}})
   711    979   		end;
   712    980   	};
   713         -	syn = {
   714         -		help = "manage synonym groups";
          981  +	rel = {
          982  +		help = "manage groups of related words";
   715    983   		syntax = {
   716         -			"(show|purge) <path>";
          984  +			"(show|purge) <path> [<kind>]";
   717    985   			"(link|drop) <word> <group#> <path>…";
   718         -			"new <path> <path>…";
   719         -			"clear <word> [<group#>]";
          986  +			"new <rel> <path> <path>…";
          987  +			"destroy <word> [<group#>]";
          988  +			"rel ::= (syn|ant|co)"
   720    989   		};
   721    990   		write = true;
   722         -		exec = function(ctx, op, tgtw, ...)
          991  +		exec = function(ctx, op, ...)
          992  +			local fo = ctx.sty[io.stdout]
          993  +			if op == nil  then id10t "not enough arguments" end
   723    994   			local groups = {}
   724         -			local wp = parsePath(tgtw)
   725         -			local w,d = safeNavWord(ctx, wp.w, wp.dn)
   726         -			if not (op=='new' or op=='link' or op=='drop' or op=='clear' or op=='show' or op=='purge') then
   727         -				id10t('invalid operation “%s” for `syn`', op)
          995  +			if not (op=='new' or op=='link' or op=='drop' or op=='destroy' or op=='show' or op=='purge') then
          996  +				id10t('invalid operation “%s” for `rel`', op)
   728    997   			end
   729    998   			if op == 'new' then
   730         -				local links = {{word = wp.w, def = wp.dn or 1}}
   731         -				for i,l in ipairs{...} do
   732         -					local parsed = parsePath(l)
   733         -					links[i+1] = {word = parsed.w, def = parsed.dn or 1}
          999  +				local rel = ...
         1000  +				if rel ~= 'syn' and rel ~= 'ant' and rel ~= 'met' then
         1001  +					id10t 'relationships must be synonymous, antonymous, or metonymous'
   734   1002   				end
   735         -				table.insert(ctx.dict.synonyms, {
   736         -					uid=math.random(0,0xffffFFFF);
         1003  +				local links={}
         1004  +				for i,l in ipairs{select(2,...)} do
         1005  +					links[i] = pathParse(l)
         1006  +				end
         1007  +				local newstruct = {
         1008  +					uid=math.random(1,0xffffFFFF);
   737   1009   					members=links;
   738         -				})
         1010  +					kind = rel;
         1011  +				}
         1012  +				table.insert(ctx.dict.relsets, newstruct)
         1013  +
         1014  +				local rc = ctx.dict._relCache
         1015  +				for i,l in pairs(links) do
         1016  +					rc[l.w] = rc[l.w] or {}
         1017  +					table.insert(rc[l.w], newstruct)
         1018  +				end
         1019  +				rebuildRelationCache(ctx.dict)
   739   1020   			else -- assemble a list of groups
   740         -				for i,ss in ipairs(ctx.dict.synonyms) do
         1021  +				local tgtw = ...
         1022  +				local wp = pathParse(tgtw)
         1023  +				local w,d,m = pathNav(ctx, wp)
         1024  +				for i,ss in ipairs(ctx.dict.relsets) do
   741   1025   					for j,s in ipairs(ss.members) do
   742         -						if s.word == wp.w and (wp.dn == nil or s.def == wp.dn) then
   743         -							table.insert(groups, {set = ss, mem = s})
         1026  +						if pathSub(s, wp) then
         1027  +-- 						if s.word == wp.w and (wp.dn == nil or s.def == wp.dn) then
         1028  +							table.insert(groups, {set = ss, mem = s, id = i})
   744   1029   							break
   745   1030   						end
   746   1031   					end
   747   1032   				end
   748   1033   
   749   1034   				if op == 'show' then
   750   1035   					for i, g in ipairs(groups) do
   751         -						local w,d = safeNavWord(ctx, g.mem.word, g.mem.def)
   752         -						local function label(wd,defn)
   753         -							local fulldef = {}
   754         -							for i,v in ipairs(defn.means) do
   755         -								fulldef[i] = v.lit
         1036  +						local w = pathResolve(ctx, {w=g.mem.w}).w
         1037  +						local function label(path,w)
         1038  +							local repr = path.w
         1039  +							if path.dn then
         1040  +								repr = repr .. string.format("(%s)", w.defs[path.dn].part)
         1041  +								if path.mn then
         1042  +									repr = repr .. string.format(": %u. %s", path.dn, w.defs[path.dn].means[path.mn].lit)
         1043  +								else
         1044  +									local fulldef = {}
         1045  +									for i,v in ipairs(w.defs) do
         1046  +										fulldef[i] = v.lit
         1047  +									end
         1048  +									repr = repr..table.concat(fulldef, '; ')
         1049  +								end
   756   1050   							end
   757         -							fulldef = table.concat(fulldef, '; ')
   758         -							return string.format("%s(%s): %s",wd,defn.part,fulldef)
         1051  +
         1052  +							return repr
   759   1053   						end
         1054  +
   760   1055   						local others = {}
   761   1056   						for j, o in ipairs(g.set.members) do
   762         -							if not (o.word == g.mem.word and o.def == (wp.dn or 1)) then
   763         -								local ow, od = safeNavWord(ctx, o.word,o.def)
   764         -								table.insert(others, '      '..label(o.word,od))
         1057  +							local ow = pathResolve(ctx, {w=o.w}).w
         1058  +							if (g.set.kind == 'ant' or not pathMatch(o, g.mem)) and
         1059  +								--exclude antonym headwords
         1060  +								not (g.set.kind == 'ant' and j==1) then
         1061  +								table.insert(others, '      '..label(o,ow))
         1062  +							end
         1063  +						end
         1064  +						local llab do
         1065  +							local cdw = ctx.dict.words
         1066  +							if g.set.kind == 'ant' then
         1067  +								local ap = g.set.members[1]
         1068  +								llab = fo.br(label(ap,cdw[ap.w]) or '')
         1069  +							else
         1070  +								llab = fo.br(label(g.mem,cdw[g.mem.w]) or '')
   765   1071   							end
   766   1072   						end
   767         -						io.stdout:write(string.format("% 4u) %s\n%s", i, label(g.mem.word,d),table.concat(others,'\n')))
         1073  +						local kls = {
         1074  +							syn = fo.color('synonyms',2,true)..' of';
         1075  +							ant = fo.color('antonyms',1,true)..' of';
         1076  +							met = fo.color('metonyms',4,true)..' of';
         1077  +						}
         1078  +						io.stdout:write(string.format("% 4u) %s\n%s", i, fo.ul(kls[g.set.kind] .. ' ' .. llab), table.concat(others,'\n')) .. '\n')
   768   1079   					end
         1080  +					return false -- no changes made
   769   1081   				elseif op == 'link' or op == 'drop' then
   770         -					local tgtn, paths = (...), { select(2, ...) }
         1082  +					local tgtn, paths = (select(2,...)), { select(3, ...) }
         1083  +					rebuildRelationCache(ctx.dict)
         1084  +				elseif op == 'destroy' then
         1085  +					local tgtw, tgtn = ...
         1086  +					if not tgtn then id10t 'missing group number' end
         1087  +					local delendum = groups[tonumber(tgtn)]
         1088  +					if not delendum then id10t 'bad group number' end
         1089  +					local rs = ctx.dict.relsets
         1090  +					local last = #rs
         1091  +					if delendum.id == last then
         1092  +						rs[delendum.id] = nil
         1093  +					else -- since order doesn't matter, we can use a
         1094  +						-- silly swapping trick to reduce the deletion
         1095  +						-- worst case from O(n) to O(2)
         1096  +						rs[delendum.id] = rs[last]
         1097  +						rs[last] = nil
         1098  +					end
         1099  +					rebuildRelationCache(ctx.dict)
         1100  +				else
         1101  +					id10t 'invalid operation'
   771   1102   				end
   772   1103   			end
   773   1104   		end;
   774   1105   	};
   775   1106   	mod = {
   776   1107   		help = "move, merge, split, or delete words or definitions";
   777   1108   		syntax = {
................................................................................
   783   1114   	note = {
   784   1115   		help = "add a note to a definition or a paragraph to a note";
   785   1116   		syntax = {"(<m-path> (add|for) <kind> | <m-path>:<note#>) <para>…";
   786   1117   			"m-path ::= <word>@<def#>/<meaning#>"};
   787   1118   		write = true;
   788   1119   		exec = function(ctx,path,...)
   789   1120   			local paras, mng
   790         -			local dest = parsePath(path)
         1121  +			local dest = pathParse(path)
   791   1122   			local _,_,m = safeNavWord(ctx,dest.w,dest.dn,dest.mn)
   792   1123   			if dest.nn then
   793   1124   				paras = {...}
   794   1125   			else
   795   1126   				local op, kind = ...
   796   1127   				paras = { select(3, ...) }
   797   1128   				if op == 'add' then
................................................................................
   828   1159   	predicates = {
   829   1160   		help = "show available filter predicates";
   830   1161   		nofile = true;
   831   1162   		syntax = "[<predicate>]";
   832   1163   	};
   833   1164   	export = {
   834   1165   		help = "create a text file dump compatible with source control";
         1166  +		syntax = "[<target-file>]";
         1167  +	};
         1168  +	import = {
         1169  +		help = "generate a usable dictionary from a text export file";
         1170  +		syntax = "[<input-file>]";
         1171  +		raw = true;
         1172  +		write = true;
   835   1173   	};
   836   1174   	dump = {
   837   1175   		exec = function(ctx) print(dump(ctx.dict)) end
   838   1176   	};
   839   1177   	ls = {
   840   1178   		help = "list all words that meet any given <filter>";
   841   1179   		syntax = {"[<filter>…]";
................................................................................
   848   1186   	local list = predicates
   849   1187   	if pred then list = {predicates[pred]} end
   850   1188   	local f = ctx.sty[io.stderr]
   851   1189   	for k,p in pairs(predicates) do
   852   1190   		if p.help then
   853   1191   			io.stderr:write(
   854   1192   				f.br('  - ' ..
   855         -					f.rgb('[',.8,.3,1) ..
         1193  +					f.rgb('[',1,0,.5) ..
   856   1194   					k .. ' ' ..
   857   1195   					(f.color(p.syntax,5) or '…') ..
   858         -					f.rgb(']',.8,.3,1)) .. ': ' ..
         1196  +					f.rgb(']',1,0,.5)) .. ': ' ..
   859   1197   				f.color(p.help,4,true) .. '\n')
   860   1198   		end
   861   1199   	end
   862   1200   end
   863   1201   
   864   1202   function cmds.ls.exec(ctx,...)
   865   1203   	local filter = nil
................................................................................
   875   1213   		local e = {lit=lit, word=w}
   876   1214   		if filter == nil or filter(e) then
   877   1215   			table.insert(out, e)
   878   1216   		end
   879   1217   	end
   880   1218   	table.sort(out, function(a,b) return a.lit < b.lit end)
   881   1219   	local fo = ctx.sty[io.stdout]
   882         -	local function meanings(d,md,n)
         1220  +
         1221  +	local function gatherRelSets(path)
         1222  +		local antonymSets, synonymSets, metonymSets = {},{},{}
         1223  +		if ctx.dict._relCache[path.w] then
         1224  +			for i, rel in ipairs(ctx.dict._relCache[path.w]) do
         1225  +				local specuset,tgt,anto = {}
         1226  +				for j, mbr in ipairs(rel.members) do
         1227  +					if pathMatch(mbr, path) then
         1228  +						if     rel.kind == 'syn' then tgt = synonymSets
         1229  +						elseif rel.kind == 'met' then tgt = metonymSets
         1230  +						elseif rel.kind == 'ant' then
         1231  +							if j == 1 -- is this the headword?
         1232  +								then tgt = antonymSets
         1233  +								else tgt = synonymSets
         1234  +							end
         1235  +						end
         1236  +					elseif j == 1 and rel.kind == 'ant' then
         1237  +						anto = mbr
         1238  +					else
         1239  +						table.insert(specuset, mbr)
         1240  +					end
         1241  +				end
         1242  +				if tgt then
         1243  +					table.insert(tgt, specuset)
         1244  +					if anto then
         1245  +						table.insert(antonymSets, {anto})
         1246  +					end
         1247  +				end
         1248  +			end
         1249  +		end
         1250  +		local function flatten(lst)
         1251  +			local new = {}
         1252  +			for i, l in ipairs(lst) do tcatD(new, l) end
         1253  +			return new
         1254  +		end
         1255  +		return {
         1256  +			syn = flatten(synonymSets);
         1257  +			ant = flatten(antonymSets);
         1258  +			met = flatten(metonymSets);
         1259  +		}
         1260  +	end
         1261  +
         1262  +	local function formatRels(rls, padlen)
         1263  +		-- optimize for the common case
         1264  +		if next(rls.syn) == nil and
         1265  +		   next(rls.ant) == nil and
         1266  +			next(rls.met) == nil then return {} end
         1267  +		local pad = string.rep(' ',padlen)
         1268  +		local function format(label, set)
         1269  +			local each = map(set, function(e)
         1270  +				local ew,ed = pathNav(ctx, e)
         1271  +            local str = fo.ul(e.w)
         1272  +				if ed then str = string.format('%s(%s)',str,ed.part) end
         1273  +				if e.mn then str = string.format('%s§%u',str,e.mn) end
         1274  +				return str
         1275  +			end)
         1276  +			return fo.em(string.format("%s%s %s",pad,label,table.concat(each,', ')))
         1277  +		end
         1278  +		local lines = {}
         1279  +		local function add(l,c,lst)
         1280  +			table.insert(lines, format(fo.color(l,c,true),lst))
         1281  +		end
         1282  +		if next(rls.syn) then add('synonyms:',2,rls.syn) end
         1283  +		if next(rls.ant) then add('antonyms:',1,rls.ant) end
         1284  +		if next(rls.met) then add('metonyms:',4,rls.met) end
         1285  +		return lines
         1286  +	end
         1287  +
         1288  +	local function meanings(w,d,md,n)
   883   1289   		local start = md and 2 or 1
   884   1290   		local part = string.format('(%s)', d.part)
   885   1291   		local pad = md and string.rep(' ', #part) or ''
   886   1292   		local function note(n,insert)
   887   1293   			if not next(n.paras) then return end
   888   1294   			local pad = string.rep(' ',#(n.kind) + 9)
   889   1295   			insert('      ' .. fo.hl(' ' .. n.kind .. ' ') .. ' ' .. n.paras[1])
   890   1296   			for i=2,#n.paras do
   891   1297   				insert(pad..n.paras[2])
   892   1298   			end
   893   1299   		end
   894   1300   		local m = { (function()
   895   1301   			if d.means[1] then
   896         -				if md then return
   897         -					string.format("  %s 1. %s", fo.em(part), d.means[1].lit)
         1302  +				if md then
         1303  +					local id = ''
         1304  +					if ctx.flags.ident then
         1305  +						id=' ['..pathString({w=w.lit,dn=n,mn=1}, fo)..']'
         1306  +					end
         1307  +					return string.format(" %s %s 1. %s", id, fo.em(part), d.means[1].lit)
   898   1308   				end
   899   1309   			else return
   900   1310   				fo.em(string.format('  %s [empty definition #%u]', part,n))
   901   1311   			end
   902   1312   		end)() }
         1313  +		tcatD(m, formatRels(gatherRelSets{w=w.lit,dn=n,mn=1}, 6))
   903   1314   		for i=start,#d.means do local v = d.means[i]
   904         -			table.insert(m, string.format('  %s %u. %s', pad, i, v.lit))
         1315  +			local id = ''
         1316  +			if ctx.flags.ident then id='['..pathString({w=w.lit,dn=n,mn=n}, fo)..']' end
         1317  +			table.insert(m, string.format('  %s%s %u. %s', pad, id, i, v.lit))
         1318  +			tcatD(m, formatRels(gatherRelSets{w=w.lit,dn=n,mn=i}, 6))
   905   1319   			for j,n in ipairs(v.notes) do
   906   1320   				note(n, function(v) table.insert(m, v) end)
   907   1321   			end
   908   1322   		end
   909   1323   		return table.concat(m,'\n')
   910   1324   	end
         1325  +	local function autobreak(str)
         1326  +		if str ~= '' then return str..'\n' else return str end
         1327  +	end
   911   1328   	for i, w in ipairs(out) do
   912         -		local d = fo.ul(w.lit)
         1329  +		local d = fo.ul(fo.br(w.lit))
         1330  +		local wordrels = autobreak(table.concat(
         1331  +				formatRels(gatherRelSets{w=w.lit}, 2),
         1332  +				'\n'
         1333  +			))
         1334  +		local wc = ctx.dict._relCache[w.lit]
   913   1335   		if #w.word.defs == 1 then
   914         -			d=d .. ' ' .. fo.em('('..(w.word.defs[1].part)..')') ..'\n'
   915         -				.. meanings(w.word.defs[1],false,1)
         1336  +			d=d .. ' '
         1337  +			    .. fo.rgb(fo.em('('..(w.word.defs[1].part)..')'),.8,.5,1) .. '\n'
         1338  +			    .. meanings(w,w.word.defs[1],false,1) .. '\n'
         1339  +			    .. autobreak(table.concat(formatRels(gatherRelSets{w=w.lit,dn=1}, 4), '\n'))
         1340  +				 .. wordrels .. '\n'
   916   1341   		else
   917   1342   			for j, def in ipairs(w.word.defs) do
   918         -				d=d .. '\n' .. meanings(def,true,j)
   919         -			end
   920         -		end
   921         -		io.stdout:write(d..'\n')
   922         -	end
   923         -end
   924         -
   925         -function cmds.export.exec(ctx)
         1343  +				local syn if wsc and wsc[j] then syn = wsc[j] end
         1344  +				d=d .. '\n'
         1345  +				    .. meanings(w,syn,def,true,j) .. '\n'
         1346  +				    .. autobreak(table.concat(
         1347  +						 formatRels(gatherRelSets{w=w.lit,dn=j}, 4),
         1348  +						 '\n'
         1349  +					 ))
         1350  +			end
         1351  +			d=d .. wordrels .. '\n'
         1352  +		end
         1353  +		io.stdout:write(d)
         1354  +	end
         1355  +end
         1356  +
         1357  +function cmds.import.exec(ctx,file)
         1358  +	local ifd = io.stdin
         1359  +	if file then
         1360  +		ifd = safeopen(file,'r')
         1361  +	end
         1362  +
         1363  +	local new = {
         1364  +		header = {
         1365  +			lang = lang;
         1366  +			meta = "";
         1367  +			partsOfSpeech = {};
         1368  +			inflectionForms = {};
         1369  +		};
         1370  +		words = {};
         1371  +		relsets = {};
         1372  +	}
         1373  +
         1374  +	local state = 0
         1375  +	local relsets = {}
         1376  +	local path = {}
         1377  +	local inflmap, lastinfl = {}
         1378  +	for l in ifd:lines() do
         1379  +		local words = strwords(l)
         1380  +		local c = words[1]
         1381  +		local function syn(mn,mx)
         1382  +			local nw = #words - 1
         1383  +			if nw < mn or (mx ~= nil and nw > mx) then
         1384  +				if mx ~= nil then
         1385  +					id10t('command %s needs between %u~%u words',c,mn,mx)
         1386  +				else
         1387  +					id10t('command %s needs at least %u words',c,mn)
         1388  +				end
         1389  +			end
         1390  +		end
         1391  +		if c ~= '*' and c~='meta' then -- comments
         1392  +			if state == 0 then
         1393  +				if c ~= 'pv0' then
         1394  +					id10t "not a parvan export"
         1395  +				end
         1396  +				new.header.lang = words[2]
         1397  +				new.header.meta = words[3]
         1398  +				state = 1
         1399  +			else
         1400  +				print(pathString(path, ctx.sty[io.stderr]))
         1401  +				local W,D,M,N = pathNav({dict=new}, path)
         1402  +				if c == 'w' then syn(1) state = 2
         1403  +					path = {w=words[2]}
         1404  +					new.words[words[2]] = {defs={}}
         1405  +				elseif c == 'f' then syn(1)
         1406  +					local nf = {
         1407  +						name = words[2];
         1408  +						abbrev = words[3] or "";
         1409  +						desc = words[4] or "";
         1410  +						parts = {};
         1411  +					}
         1412  +					table.insert(new.header.inflectionForms, nf)
         1413  +					inflmap[words[2]] = #(new.header.inflectionForms)
         1414  +					lastinfl = nf
         1415  +				elseif c == 'fp' then syn(1)
         1416  +					if not lastinfl then
         1417  +						id10t 'fp can only be used after f' end
         1418  +					table.insert(lastinfl.parts,words[2])
         1419  +				elseif c == 's' then syn(2)
         1420  +					relsets[words[3]] = relsets[words[3]] or {}
         1421  +					relsets[words[3]].kind = words[2]
         1422  +					relsets[words[3]].uid = tonumber(words[3])
         1423  +					relsets[words[3]].members = relsets[words[3]].members or {}
         1424  +				elseif state >= 2 and c == 'r' then syn(1)
         1425  +					relsets[words[2]] = relsets[words[2]] or {
         1426  +						uid = tonumber(words[2]);
         1427  +						members={};
         1428  +					}
         1429  +					table.insert(relsets[words[2]].members, path)
         1430  +				elseif state >= 2 and c == 'd' then syn(1) state = 3
         1431  +					table.insert(W.defs, {
         1432  +						part = words[2];
         1433  +						branch = {};
         1434  +						means = {};
         1435  +						forms = {};
         1436  +						phrases = {};
         1437  +					})
         1438  +					path = {w = path.w, dn = #(W.defs)}
         1439  +				elseif state >= 3 and c == 'dr' then syn(1)
         1440  +					table.insert(D.branch, words[2])
         1441  +				elseif state >= 3 and c == 'df' then syn(2)
         1442  +					if not inflmap[words[2]] then
         1443  +						id10t('no inflection form %s defined', words[2])
         1444  +					end
         1445  +					D.forms[inflmap[words[2]]] = words[3]
         1446  +				elseif state >= 3 and c == 'm' then syn(1) state = 4
         1447  +					table.insert(D.means, {
         1448  +						lit = words[2];
         1449  +						notes = {};
         1450  +						examples = {};
         1451  +					});
         1452  +					path = {w = path.w, dn = path.dn, mn = #(D.means)}
         1453  +				elseif state >= 4 and c == 'n' then syn(1) state = 5
         1454  +					table.insert(M.notes, {kind=words[2], paras={}})
         1455  +					path = {w = path.w, dn = path.dn, mn = path.mn, nn = #(M.notes)};
         1456  +				elseif state >= 5 and c == 'np' then syn(1)
         1457  +					table.insert(N.paras, words[2])
         1458  +				end
         1459  +				-- we ignore invalid ctls, for sake of forward-compat
         1460  +			end
         1461  +		end
         1462  +	end
         1463  +
         1464  +	for k,v in pairs(relsets) do
         1465  +		if not v.uid then
         1466  +			--handle non-numeric export ids
         1467  +			v.uid = math.random(0,0xffffFFFF)
         1468  +		end
         1469  +		table.insert(new.relsets, v)
         1470  +	end
         1471  +
         1472  +	local ofd = safeopen(ctx.file,"w+b")
         1473  +	local o = writeDict(new);
         1474  +	ofd:write(o)
         1475  +	ofd:close()
         1476  +end
         1477  +
         1478  +function cmds.export.exec(ctx,file)
         1479  +	local ofd = io.stdout
         1480  +	if file then ofd = safeopen(file, 'w+') end
   926   1481   	local function san(str)
   927   1482   		local d = 0
   928   1483   		local r = {}
   929   1484   		for i,cp in utf8.codes(str) do
   930   1485   			-- insert backslashes for characters that would
   931   1486   			-- disrupt strwords() parsing
   932         -			if cp == 0x5b then
   933         -				d = d + 1
   934         -			elseif cp == 0x5d then
   935         -				if d >= 1 then
   936         -					d = d - 1
   937         -				else
   938         -					table.insert(r, 0x5c)
         1487  +			if cp == 0x0a then
         1488  +				table.insert(r, 0x5c)
         1489  +				table.insert(r, 0x6e)
         1490  +			else
         1491  +				if cp == 0x5b then
         1492  +					d = d + 1
         1493  +				elseif cp == 0x5d then
         1494  +					if d >= 1 then
         1495  +						d = d - 1
         1496  +					else
         1497  +						table.insert(r, 0x5c)
         1498  +					end
   939   1499   				end
         1500  +				table.insert(r, cp)
   940   1501   			end
   941         -			table.insert(r, cp)
   942   1502   		end
   943   1503   		return '[' .. utf8.char(table.unpack(r)) .. ']'
   944   1504   	end
   945         -	local function o(...) io.stdout:write(string.format(...)..'\n') end
         1505  +	local function o(lvl,...)
         1506  +		local pfx = ''
         1507  +		if ctx.flags.human and lvl > 0 then
         1508  +			pfx = string.rep('\t', lvl)
         1509  +		end
         1510  +		ofd:write(pfx..string.format(...)..'\n')
         1511  +	end
   946   1512   	local d = ctx.dict
   947         -	o('pv0 %s %s', san(d.header.lang), san(d.header.meta))
         1513  +	o(0,'pv0 %s %s', san(d.header.lang), san(d.header.meta))
         1514  +	local function checksyn(obj)
         1515  +-- 		for _,s in ipairs(d.synonyms) do
         1516  +		local lvl = 0
         1517  +		if     obj.nn then lvl = 4
         1518  +		elseif obj.mn then lvl = 3
         1519  +		elseif obj.dn then lvl = 2
         1520  +		elseif obj.w  then lvl = 1 end
         1521  +		if not d._relCache[obj.w] then return end
         1522  +		for _,s in ipairs(d._relCache[obj.w]) do
         1523  +			for _,sm in ipairs(s.members) do
         1524  +				if pathMatch(obj, sm) then
         1525  +					o(lvl,'r %u',s.uid)
         1526  +					break
         1527  +				end
         1528  +			end
         1529  +		end
         1530  +	end
         1531  +	for i,f in pairs(d.header.inflectionForms) do
         1532  +		o(0,'f %s %s %s', san(f.name), san(f.abbrev), san(f.desc))
         1533  +		for j,p in pairs(f.parts) do
         1534  +			o(1,'fp %s', san(p))
         1535  +		end
         1536  +	end
   948   1537   	for lit, w in pairs(d.words) do
   949         -		o('w %s',san(lit))
         1538  +		o(0,'w %s',san(lit))
         1539  +		checksyn{w=lit}
   950   1540   		for i,def in ipairs(w.defs) do
   951         -			o('d %s',san(def.part))
   952         -			for _,s in ipairs(d.synonyms) do
   953         -				for _,sm in ipairs(s.members) do
   954         -					if sm.word == w and sm.def == i then
   955         -						o('ds %u',s.uid)
   956         -						break
   957         -					end
   958         -				end
   959         -			end
         1541  +			o(1,'d %s',san(def.part))
         1542  +			checksyn{w=lit,dn=i}
   960   1543   			for j,r in ipairs(def.branch) do
   961         -				o('dr %s',san(r))
         1544  +				o(2,'dr %s',san(r))
   962   1545   			end
   963   1546   			for j,m in ipairs(def.means) do
   964         -				o('m %s', san(m.lit))
         1547  +				o(2,'m %s', san(m.lit))
         1548  +				checksyn{w=lit,dn=i,mn=j}
   965   1549   				for k,n in ipairs(m.notes) do
   966         -					o('n %s', san(n.kind))
         1550  +					o(3,'n %s', san(n.kind))
   967   1551   					for a,p in ipairs(n.paras) do
   968         -						o('np %s', san(p))
         1552  +						o(4,'np %s', san(p))
   969   1553   					end
   970   1554   				end
   971   1555   			end
   972   1556   		end
   973   1557   	end
   974         -	for _,s in ipairs(d.synonyms) do o('s %u', s.uid) end
         1558  +	for _,s in ipairs(d.relsets) do o(0,'s %s %u', s.kind, s.uid) end
         1559  +end
         1560  +
         1561  +local function filterD(lst, fn)
         1562  +-- cheap algorithm to destructively filter a list
         1563  +-- DOES NOT preserve order!!
         1564  +	local top = #lst
         1565  +	for i=top,1,-1 do local m = lst[i]
         1566  +		if not fn(m,i) then
         1567  +			lst[i] = lst[top]
         1568  +			lst[top] = nil
         1569  +			top = top - 1
         1570  +		end
         1571  +	end
         1572  +	return lst
   975   1573   end
   976   1574   
   977   1575   function cmds.mod.exec(ctx, orig, oper, dest, ...)
   978   1576   	if (not orig) or not oper then
   979   1577   		id10t '`mod` requires at least an origin and an operation'
   980   1578   	end
   981         -	local op, dp = parsePath(orig)
         1579  +	local op, dp = pathParse(orig)
   982   1580   	local w,d,m,n = safeNavWord(ctx, op.w,op.dn,op.mn,op.nn)
         1581  +	-- unfortunately, "pointers" exist elsewhere in the
         1582  +	-- structure, currently just from relsets, that must
         1583  +	-- be updated whenever an object moves or changes.
         1584  +	-- this is miserable and takes a lot of work, using
         1585  +	-- algorithms provided by the following functions.
         1586  +	-- note that we don't bother trying to update the
         1587  +	-- relcache as we go, it's simply not worth the work;
         1588  +	-- instead we simply rebuild the whole cache when
         1589  +	-- this command returns
         1590  +	local function cleanupRels(path, fn)
         1591  +		local rc = ctx.dict._relCache[path.w]
         1592  +		if rc then
         1593  +			for k,s in pairs(rc) do fn(s,k) end
         1594  +		end
         1595  +	end
         1596  +	local function cleanupRelsEach(path, fn)
         1597  +		cleanupRels(path, function(s,k)
         1598  +			local top = #s.members
         1599  +			for i=1,top do local m=s.members[i]
         1600  +				if pathSub(path, m) then
         1601  +					local val = fn(m,s,i)
         1602  +					if val ~= nil then
         1603  +						s.members[i] = val
         1604  +					end
         1605  +				end
         1606  +			end
         1607  +		end)
         1608  +	end
         1609  +	local function deleteRefsTo(path)
         1610  +		cleanupRels(path, function(s)
         1611  +		-- antonyms: delete the headword and transform the group
         1612  +		-- to a list of synonyms
         1613  +			if s.kind == 'ant' and pathSub(path,s.members[1]) then
         1614  +				s.kind = 'syn'
         1615  +			end
         1616  +			filterD(s.members, function(m)
         1617  +				return not pathSub(path, m)
         1618  +			end)
         1619  +		end)
         1620  +		if not path.dn then
         1621  +			ctx.dict._relCache[path.w] = nil
         1622  +		end
         1623  +	end
         1624  +	local function moveRelTree(op,dp)
         1625  +		cleanupRelsEach(op, function(old,set,idx)
         1626  +			local new = {}
         1627  +			for _,elt in pairs{'w','dn','mn','nn'} do
         1628  +				if dp[elt] ~= nil then
         1629  +					new[elt] = dp[elt]
         1630  +				else
         1631  +					new[elt] = op[elt] or old[elt]
         1632  +				end
         1633  +			end
         1634  +			return new
         1635  +		end)
         1636  +	end
         1637  +	local function shiftRelTree(dp, fld, nid, amt)
         1638  +		local cleanupTargetMask = ({
         1639  +			dn = {w=dp.w};
         1640  +			mn = {w=dp.w,dn=dp.dn};
         1641  +			nn = {w=dp.w,dn=dp.dn,mn=dp.mn};
         1642  +		})[fld] -- >____<
         1643  +		cleanupRelsEach(cleanupTargetMask, function(old,set,i)
         1644  +			if old[fld] >= nid then
         1645  +				old[fld] = old[fld] + amt
         1646  +			end
         1647  +		end)
         1648  +	end
         1649  +	local function insertAndMoveRelTree(tbl,n,op,dp,fld)
         1650  +		local nid = #tbl
         1651  +		local path = copy(dp)
         1652  +		path[fld] = nid
         1653  +		tbl[nid] = n
         1654  +		shiftRelTree(dp,fld,1)
         1655  +		moveRelTree(op, path)
         1656  +	end
   983   1657   	if oper == 'drop' then
         1658  +		-- clean out the cache and delete relationships
         1659  +		deleteRefsTo(op)
   984   1660   		if not d then
   985   1661   			ctx.dict.words[op.w] = nil
   986   1662   		elseif not m then
   987   1663   			table.remove(w.defs, op.dn)
   988   1664   		elseif not n then
   989   1665   			table.remove(d.means, op.mn)
   990   1666   		else
   991   1667   			table.remove(m.notes, op.nn)
   992   1668   		end
   993   1669   	elseif oper == 'out' then
   994         -		if n or not m then
   995         -			id10t '`mod out` must target a meaning'
   996         -		end
         1670  +		if n or not m then id10t '`mod out` must target a meaning' end
   997   1671   		if not dest then id10t '`mod out` requires at least a part of speech' end
   998   1672   		local newdef = {
   999   1673   			part = dest;
  1000   1674   			branch = {...};
  1001   1675   			forms = {};
  1002   1676   			means = {m};
  1003   1677   		}
         1678  +		shiftRelTree(op, 'dn', op.dn, 1)
  1004   1679   		table.insert(w.defs,op.dn+1, newdef)
         1680  +		moveRelTree(op,{w=op.w, dn=op.dn+1, mn=1})
  1005   1681   		table.remove(d.means,op.mn)
  1006   1682   	elseif oper == 'move' or oper == 'merge' or oper == 'clobber' then
  1007   1683   		if dest
  1008         -			then dp = parsePath(dest)
         1684  +			then dp = pathParse(dest)
  1009   1685   			else id10t('`mod %s` requires a target',oper)
  1010   1686   		end
  1011   1687   		if n then
  1012   1688   			if not dp.mn then
  1013   1689   				id10t '`mod` on a note requires a note or meaning destination'
  1014   1690   			end
  1015   1691   			local _,_,dm = safeNavWord(ctx, dp.w,dp.dn,dp.mn)
  1016   1692   			if dp.nn then
  1017   1693   				if oper == 'move' then
         1694  +					shiftRelTree(dp, 'nn', dp.nn, 1)
  1018   1695   					table.insert(dm.notes, dp.nn, n)
  1019   1696   				elseif oper == 'merge' then
  1020   1697   					local top = #(dm.notes[dp.nn].paras)
  1021   1698   					for i, v in ipairs(n.paras) do
  1022   1699   						dm.notes[dp.nn].paras[i+top] = v
  1023   1700   					end
  1024   1701   				elseif oper == 'clobber' then
         1702  +					deleteRefsTo(dp)
  1025   1703   					dm.notes[dp.nn] = n
  1026   1704   				end
         1705  +				moveRelTree(op,dp)
  1027   1706   			else
  1028   1707   				if oper ~= 'move' then
  1029   1708   					id10t('`mod note %s` requires a note target', oper)
  1030   1709   				end
  1031         -				table.insert(dm.notes, n)
         1710  +				insertAndMoveRelTree(dm.notes,n,op,dp,'nn')
  1032   1711   			end
  1033   1712   			if oper == 'move' and dp.nn and dm == m and op.nn > dp.nn then
  1034   1713   				table.remove(m.notes,op.nn+1)
  1035   1714   			else
  1036   1715   				table.remove(m.notes,op.nn)
  1037   1716   			end
  1038   1717   		elseif m then
................................................................................
  1039   1718   			if not dp.dn then
  1040   1719   				local newdef = {
  1041   1720   					part = d.part;
  1042   1721   					branch = copy(d.branch);
  1043   1722   					forms = copy(d.forms);
  1044   1723   					means = {m};
  1045   1724   				}
         1725  +				local didx
  1046   1726   				if ctx.dict.words[dp.w] then
  1047         -					table.insert(ctx.dict.words[dp.w].defs, newdef)
         1727  +					local defst = ctx.dict.words[dp.w].defs
         1728  +					didx = #defst
         1729  +					defst[didx] = newdef
  1048   1730   				else
  1049   1731   					ctx.dict.words[dp.w] = {
  1050   1732   						defs = {newdef};
  1051   1733   					}
         1734  +					didx = 1
  1052   1735   				end
         1736  +				cleanupRelsEach(op, function(oldpath,set,mi)
         1737  +					return {w=dp.w,dn=didx,mn=1,nn=oldpath.nn}
         1738  +				end)
  1053   1739   				table.remove(d.means,dp.mn)
  1054   1740   			else
  1055   1741   				local dw, dd = safeNavWord(ctx, dp.w, dp.dn)
  1056   1742   				if dp.mn then
  1057   1743   					if dd.means[dp.mn] and (oper == 'merge' or oper=='clobber') then
  1058   1744   						if oper == 'merge' then
  1059   1745   							dd.means[dp.mn] = dd.means[dp.mn] .. '; ' .. m
  1060   1746   						elseif oper == 'clobber' then
         1747  +							deleteRefsTo(dp)
  1061   1748   							dd.means[dp.mn] = m
  1062   1749   						end
  1063   1750   					else
  1064         -						if oper == clobber then dd.means = {} end
         1751  +						cleanupRelsEach({w=dp.w,dn=dp.dn}, function(old,set,i)
         1752  +							if old.mn >= dp.mn then
         1753  +								old.mn = old.mn + 1
         1754  +							end
         1755  +						end)
  1065   1756   						table.insert(dd.means, dp.mn, m)
  1066   1757   					end
         1758  +					moveRelTree(op,dp)
  1067   1759   				else
  1068         -					table.insert(dd.means, m)
         1760  +					insertAndMoveRelTree(dd.means,m, op,dp,'mn')
         1761  +-- 					table.insert(dd.means, m)
  1069   1762   				end
  1070   1763   				if oper == 'move' and dp.mn and dd.means == d.means and op.mn > dp.mn then
  1071   1764   					table.remove(d.means,op.mn+1)
  1072   1765   				else
  1073   1766   					table.remove(d.means,op.mn)
  1074   1767   				end
  1075   1768   			end
................................................................................
  1078   1771   			if dp.dn then
  1079   1772   				if oper == 'merge' then
  1080   1773   					local top = #(ddefs[dp.dn].means)
  1081   1774   					for i,om in ipairs(d.means) do
  1082   1775   						ddefs[dp.dn].means[top+i] = om
  1083   1776   					end
  1084   1777   					for k,p in pairs(d.forms) do
         1778  +						deleteRefsTo(dp)
  1085   1779   						ddefs[dp.dn].forms[k] = p -- clobbers!
  1086   1780   					end
  1087   1781   				else
         1782  +					shiftRelTree(dp,'dn',dp.dn,1)
  1088   1783   					table.insert(ddefs, dp.dn, d)
  1089   1784   				end
         1785  +				moveRelTree(op,dp)
  1090   1786   			else
  1091         -				table.insert(ddefs, d)
         1787  +				insertAndMoveRelTree(ddefs,d, op,dp,'dn')
         1788  +-- 				table.insert(ddefs, d)
  1092   1789   			end
  1093   1790   			if oper == 'move' and dp.mn and w.defs == ddefs and op.mn > dp.mn then
  1094   1791   				table.remove(w.defs,op.dn+1)
  1095   1792   			else
  1096   1793   				table.remove(w.defs,op.dn)
  1097   1794   			end
  1098   1795   		else
  1099   1796   			if ctx.dict.words[dp.w] then
  1100   1797   				if oper ~= 'merge' then
  1101   1798   					id10t('the word “%s” already exists; use `merge` if you want to merge the words together', dp.w)
  1102   1799   				end
  1103   1800   				for i,def in ipairs(w.defs) do
         1801  +					local odp = copy(op) odp.dn = i
         1802  +					local ddp = {w=dp.w, dn=dp.dn+i-1}
  1104   1803   					if dp.dn then
         1804  +						shiftRelTree(dp, 'dn', dp.dn+i-1, 1)
  1105   1805   						table.insert(ctx.dict.words[dp.w].defs, dp.dn+i-1, def)
         1806  +						moveRelTree(odp,ddp)
  1106   1807   					else
  1107         -						table.insert(ctx.dict.words[dp.w].defs, def)
         1808  +-- 						table.insert(ctx.dict.words[dp.w].defs, def)
         1809  +						insertAndMoveRelTree(ctx.dict.words[dp.w].defs, def,
         1810  +													odp,dp,'dn')
  1108   1811   					end
  1109   1812   				end
  1110   1813   			else
  1111   1814   				ctx.dict.words[dp.w] = w
         1815  +				moveRelTree(op,dp)
         1816  +-- 				ctx.dict._relCache[dp.w] = ctx.dict._relCache[op.w]
         1817  +-- 				ctx.dict._relCache[op.w] = nil
  1112   1818   			end
  1113   1819   			ctx.dict.words[op.w] = nil
  1114   1820   		end
  1115   1821   	end
         1822  +	rebuildRelationCache(ctx.dict)
  1116   1823   end
  1117   1824   
  1118   1825   local function fileLegible(file)
  1119   1826   	-- check if we can access the file
  1120   1827   	local fd = io.open(file,"rb")
  1121   1828   	local ret = false
  1122   1829   	if fd then ret = true end
  1123   1830   	fd:close()
  1124   1831   	return ret
  1125   1832   end
  1126   1833   
  1127         -local function map(fn,lst)
  1128         -	local new = {}
  1129         -	for k,v in pairs(lst) do
  1130         -		local nv, nk = fn(v,k)
  1131         -		new[nk or k] = nv
  1132         -	end
  1133         -	return new
  1134         -end
  1135         -local function mapD(fn,lst) --destructive
  1136         -	-- WARNING: this will not work if nk names an existing key!
  1137         -	for k,v in pairs(lst) do
  1138         -		local nv, nk = fn(v,k)
  1139         -		if nk == nil or k == nk then
  1140         -			lst[k] = nv
  1141         -		else
  1142         -			lst[k] = nil
  1143         -			lst[nk] = nv
  1144         -		end
  1145         -	end
  1146         -	return lst
  1147         -end
  1148   1834   
  1149   1835   local function
  1150   1836   prompt(p,multiline)
  1151   1837   	-- returns string if successful, nil if EOF, false if ^C
  1152   1838   	io.stderr:write(p)
  1153   1839   	local ok, res = pcall(function()
  1154   1840   	                      return io.stdin:read(multiline and 'a' or 'l')
................................................................................
  1199   1885   				local c = cmds[words[1]]
  1200   1886   				if c then
  1201   1887   					if c.raw then
  1202   1888   						ctx.log('fatal', words[1] .. ' cannot be run from `shell`')
  1203   1889   					elseif not implies(c.write, rw) then
  1204   1890   						ctx.log('fatal', ctx.file .. ' is not writable')
  1205   1891   					else
  1206         -						local ok = ctx.try(c.exec, ctx, table.unpack(words,2))
  1207         -						if ok then written = written or c.write end
         1892  +						local ok, outcome = ctx.try(c.exec, ctx, table.unpack(words,2))
         1893  +						if ok and outcome ~= false then written = written or c.write end
  1208   1894   					end
  1209   1895   				elseif cmd == 'save' or cmd == 'wq' then
  1210   1896   					if not written then
  1211   1897   						ctx.log('info', 'no changes to save')
  1212   1898   					end
  1213   1899   					cmd = nil
  1214   1900   				elseif cmd == 'quit' or cmd == 'q' then
................................................................................
  1261   1947   		showHelp(ctx, cmd, cmds[cmd])
  1262   1948   	else
  1263   1949   		for cmd,c in pairs(cmds) do
  1264   1950   			showHelp(ctx, cmd, c)
  1265   1951   		end
  1266   1952   	end
  1267   1953   end
         1954  +
         1955  +local globalFlags <const> = {
         1956  +	human = {'h','human','enable human-readable exports'};
         1957  +	ident = {'i','ident','show identifier paths for all items'}
         1958  +}
  1268   1959   
  1269   1960   local function
  1270   1961   usage(me,ctx)
  1271   1962   	local ln = 0
  1272   1963   	local ct = {}
  1273   1964   	local fe = ctx.sty[io.stderr]
  1274         -	io.stderr:write(string.format(fe.br"usage:".." %s <file> [<command> [args…]]\n",me))
         1965  +	local fstr = ""
         1966  +	local flagHelp = {}
         1967  +	for k,v in pairs(globalFlags) do
         1968  +		fstr = fstr .. v[1]
         1969  +		table.insert(flagHelp, string.format("  -%s --%s: %s\n",table.unpack(v)))
         1970  +	end
         1971  +	io.stderr:write(string.format(fe.br"usage:".." %s [-%s] <file> [<command> [args…]]\n",me,fstr) .. table.concat(flagHelp))
  1275   1972   	--[[
  1276   1973   	for k,v in pairs(cmds) do
  1277   1974   		local n = 1 + utf8.len(k) + utf8.len(v.syntax)
  1278   1975   		ct[k] = n
  1279   1976   		if n > ln then ln = n end
  1280   1977   	end
  1281   1978   	for k,v in pairs(cmds) do
................................................................................
  1288   1985   	end
  1289   1986   	return 64
  1290   1987   end
  1291   1988   
  1292   1989   local function
  1293   1990   dispatch(argv, ctx)
  1294   1991   	local ferr = ctx.sty[io.stderr]
  1295         -	local file, cmd = table.unpack(argv)
         1992  +	local args = {}
         1993  +	local flags = {}
         1994  +	local i = 1 while i <= #argv do
         1995  +		local a = argv[i]
         1996  +		if a == '--' then i=i+1 break
         1997  +		elseif a:sub(1,2) == '--' then
         1998  +			local fs <const> = a:sub(3)
         1999  +			for k,v in pairs(globalFlags) do
         2000  +				if v[2] == fs then flags[k] = true end
         2001  +			end
         2002  +		elseif a:sub(1,1) == '-' then
         2003  +			for p,cp in utf8.codes(''), a, #'-' do
         2004  +				local c <const> = utf8.char(cp)
         2005  +				for k,v in pairs(globalFlags) do
         2006  +					if v[1] == c then flags[k] = true break end
         2007  +				end
         2008  +			end
         2009  +		else table.insert(args, a) end
         2010  +	i = i + 1 end
         2011  +	for j=i,#argv do table.insert(args,argv[j]) end
         2012  +
         2013  +	local file, cmd = table.unpack(args)
  1296   2014   	if cmd and cmds[cmd] then
  1297   2015   		local c,fd,dict = cmds[cmd]
  1298   2016   		if (not c.raw) and not c.nofile then
  1299   2017   			fd = safeopen(file, "rb")
  1300   2018   			dict = readDict(fd:read 'a')
  1301   2019   			fd:close()
  1302   2020   			-- lua io has no truncate method, so we must
................................................................................
  1305   2023   		end
  1306   2024   
  1307   2025   		cmds[cmd].exec({
  1308   2026   			sty = ctx.sty;
  1309   2027   			try = ctx.try;
  1310   2028   			log = ctx.log;
  1311   2029   
         2030  +			flags = flags;
  1312   2031   			file = file;
  1313   2032   			fd = fd;
  1314   2033   			dict = dict;
  1315         -		}, table.unpack(argv,3))
         2034  +		}, table.unpack(args,3))
  1316   2035   
  1317   2036   		if (not c.raw) and c.write then
  1318   2037   			local output = writeDict(dict)
  1319   2038   			-- writeDict should always be given a chance to
  1320   2039   			-- bail before the previous file is destroyed!!
  1321   2040   			-- you don't want one bug to wipe out your entire
  1322   2041   			-- dictionary in one fell swoop
................................................................................
  1344   2063   
  1345   2064   local function log(lvl, msg)
  1346   2065   	local colors = {fatal=1,warn=3,info=4,debug=2}
  1347   2066   	local ferr = sty[io.stderr]
  1348   2067   	io.stderr:write(string.format(
  1349   2068   		ferr.color(ferr.br("(%s)"),colors[lvl]).." %s\n", lvl, msg))
  1350   2069   end
  1351         -local function try(...)
         2070  +
         2071  +local function stacktrace(err)
         2072  +	return debug.traceback(err,3)
         2073  +end
         2074  +
         2075  +local function try(fn,...)
  1352   2076   	-- a wrapper around pcall that produces a standard error
  1353   2077   	-- message format when an error occurs
  1354         -	local res = { pcall(...) }
         2078  +	local res = { xpcall(fn,stacktrace,...) }
  1355   2079   	if not res[1] then
  1356   2080   		log('fatal', res[2])
  1357   2081   	end
  1358   2082   	return table.unpack(res)
  1359   2083   end
  1360   2084   
  1361         -local function stacktrace(err)
  1362         -	return debug.traceback(err,3)
  1363         -end
  1364   2085   local ok, res = xpcall(dispatch, stacktrace, argv, {
  1365   2086   	try = try, sty = sty, log = log
  1366   2087   })
  1367   2088   
  1368   2089   if not ok then
  1369   2090   	log('fatal', res)
  1370   2091   	os.exit(1)
  1371   2092   end
  1372   2093   
  1373   2094   os.exit(res)