293 lines
7.9 KiB
Python
293 lines
7.9 KiB
Python
|
#!/usr/bin/python2
|
||
|
|
# Copyright 2003-2008, Nick Mathewson. See LICENSE for licensing info.
|
||
|
|
|
||
|
|
"""Code to determine which entries are new and which are old.
|
||
|
|
|
||
|
|
To scan a new file, run "python reconcile.py anonbib.cfg new-file.bib". This
|
||
|
|
will generate a new bibtex file called 'tmp.bib', with all the new entries
|
||
|
|
cleaned up a little, and all the duplicate entries commented out.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import sys
|
||
|
|
import re
|
||
|
|
|
||
|
|
assert sys.version_info[:3] >= (2,2,0)
|
||
|
|
|
||
|
|
import BibTeX
|
||
|
|
import config
|
||
|
|
import metaphone
|
||
|
|
|
||
|
|
_MPCACHE = {}
|
||
|
|
def soundsLike(s1, s2):
|
||
|
|
c = _MPCACHE
|
||
|
|
s1 = clean(s1)
|
||
|
|
s2 = clean(s2)
|
||
|
|
try:
|
||
|
|
m1 = c[s1]
|
||
|
|
except KeyError:
|
||
|
|
m1 = c[s1] = metaphone.metaphone(s1)
|
||
|
|
try:
|
||
|
|
m2 = c[s2]
|
||
|
|
except KeyError:
|
||
|
|
m2 = c[s2] = metaphone.metaphone(s2)
|
||
|
|
|
||
|
|
return m1 == m2
|
||
|
|
|
||
|
|
def mphone(s):
|
||
|
|
c = _MPCACHE
|
||
|
|
s = clean(s)
|
||
|
|
try:
|
||
|
|
return c[s]
|
||
|
|
except:
|
||
|
|
m = c[s] = metaphone.metaphone(s)
|
||
|
|
return m
|
||
|
|
|
||
|
|
def clean(s):
|
||
|
|
s = re.sub(r'\s+', ' ', s)
|
||
|
|
s = s.strip()
|
||
|
|
return s
|
||
|
|
|
||
|
|
class MasterBibTeX(BibTeX.BibTeX):
|
||
|
|
def __init__(self):
|
||
|
|
BibTeX.BibTeX.__init__(self)
|
||
|
|
|
||
|
|
def buildIndex(self):
|
||
|
|
self.byTitle = {}
|
||
|
|
for ent in self.entries:
|
||
|
|
for t in self._titleForms(ent['title']):
|
||
|
|
self.byTitle.setdefault(t, []).append(ent)
|
||
|
|
|
||
|
|
def _titleForms(self, title):
|
||
|
|
title = title.lower()
|
||
|
|
title = re.sub(r'\b(an|a|the|of)\b', "", title)
|
||
|
|
title = clean(title)
|
||
|
|
res = [ mphone(title) ]
|
||
|
|
if ':' in title:
|
||
|
|
for t in title.split(":"):
|
||
|
|
res.append(mphone(t.strip()))
|
||
|
|
#print "%r\n => %s" % (title,res)
|
||
|
|
return res
|
||
|
|
|
||
|
|
def _titlesAlike(self, t1, t2):
|
||
|
|
t1 = clean(t1)
|
||
|
|
t2 = clean(t2)
|
||
|
|
if t1 == t2:
|
||
|
|
return 2
|
||
|
|
tf1 = self._titleForms(t1)
|
||
|
|
tf2 = self._titleForms(t2)
|
||
|
|
for t in tf1:
|
||
|
|
if t in tf2: return 1
|
||
|
|
return 0
|
||
|
|
|
||
|
|
def _authorsAlike(self, a1, a2):
|
||
|
|
if not soundsLike(" ".join(a1.last)," ".join(a2.last)):
|
||
|
|
return 0
|
||
|
|
|
||
|
|
if (a1.first == a2.first and a1.von == a2.von
|
||
|
|
and a1.jr == a2.jr):
|
||
|
|
return 2
|
||
|
|
|
||
|
|
|
||
|
|
if soundsLike(" ".join(a1.first), " ".join(a2.first)):
|
||
|
|
return 1
|
||
|
|
|
||
|
|
if not a1.first or not a2.first:
|
||
|
|
return 1
|
||
|
|
|
||
|
|
if self._initialize(a1.first) == self._initialize(a2.first):
|
||
|
|
return 1
|
||
|
|
|
||
|
|
return 0
|
||
|
|
|
||
|
|
def _initialize(self, name):
|
||
|
|
name = " ".join(name).lower()
|
||
|
|
name = re.sub(r'([a-z])[a-z\.]*', r'\1', name)
|
||
|
|
name = clean(name)
|
||
|
|
return name
|
||
|
|
|
||
|
|
def _authorListsAlike(self, a1, a2):
|
||
|
|
if len(a1) != len(a2):
|
||
|
|
return 0
|
||
|
|
a1 = [ (a.last, a) for a in a1 ]
|
||
|
|
a2 = [ (a.last, a) for a in a2 ]
|
||
|
|
a1.sort()
|
||
|
|
a2.sort()
|
||
|
|
if len(a1) != len(a2):
|
||
|
|
return 0
|
||
|
|
r = 2
|
||
|
|
for (_, a1), (_, a2) in zip(a1,a2):
|
||
|
|
x = self._authorsAlike(a1,a2)
|
||
|
|
if not x:
|
||
|
|
return 0
|
||
|
|
elif x == 1:
|
||
|
|
r = 1
|
||
|
|
return r
|
||
|
|
|
||
|
|
def _entryDatesAlike(self, e1, e2):
|
||
|
|
try:
|
||
|
|
if clean(e1['year']) == clean(e2['year']):
|
||
|
|
return 2
|
||
|
|
else:
|
||
|
|
return 0
|
||
|
|
except KeyError:
|
||
|
|
return 1
|
||
|
|
|
||
|
|
def includes(self, ent, all=0):
|
||
|
|
title = ent['title']
|
||
|
|
candidates = []
|
||
|
|
for form in self._titleForms(title):
|
||
|
|
try:
|
||
|
|
candidates.extend(self.byTitle[form])
|
||
|
|
except KeyError:
|
||
|
|
pass
|
||
|
|
goodness = []
|
||
|
|
for knownEnt in candidates:
|
||
|
|
match = (self._entryDatesAlike(ent, knownEnt) *
|
||
|
|
self._titlesAlike(ent['title'], knownEnt['title']) *
|
||
|
|
self._authorListsAlike(ent.parsedAuthor,
|
||
|
|
knownEnt.parsedAuthor) )
|
||
|
|
if match:
|
||
|
|
goodness.append((match, knownEnt))
|
||
|
|
goodness.sort()
|
||
|
|
if all:
|
||
|
|
return goodness
|
||
|
|
if goodness:
|
||
|
|
return goodness[-1]
|
||
|
|
else:
|
||
|
|
return None, None
|
||
|
|
|
||
|
|
def demo(self):
|
||
|
|
for e in self.entries:
|
||
|
|
matches = self.includes(e, 1)
|
||
|
|
m2 = []
|
||
|
|
mids = []
|
||
|
|
for g,m in matches:
|
||
|
|
if id(m) not in mids:
|
||
|
|
mids.append(id(m))
|
||
|
|
m2.append((g,m))
|
||
|
|
matches = m2
|
||
|
|
|
||
|
|
if not matches:
|
||
|
|
print "No match for %s"%e.key
|
||
|
|
if matches[-1][1] is e:
|
||
|
|
print "%s matches for %s: OK."%(len(matches), e.key)
|
||
|
|
else:
|
||
|
|
print "%s matches for %s: %s is best!" %(len(matches), e.key,
|
||
|
|
matches[-1][1].key)
|
||
|
|
if len(matches) > 1:
|
||
|
|
for g, m in matches:
|
||
|
|
print "%%%% goodness", g
|
||
|
|
print m
|
||
|
|
|
||
|
|
|
||
|
|
def noteToURL(note):
|
||
|
|
" returns tp, url "
|
||
|
|
note = note.replace("\n", " ")
|
||
|
|
m = re.match(r'\s*(?:\\newline\s*)*\s*\\url{(.*)}\s*(?:\\newline\s*)*',
|
||
|
|
note)
|
||
|
|
if not m:
|
||
|
|
return None
|
||
|
|
url = m.group(1)
|
||
|
|
for suffix, tp in ((".html", "html"),
|
||
|
|
(".ps", "ps"),
|
||
|
|
(".ps.gz", "ps_gz"),
|
||
|
|
(".pdf", "pdf"),
|
||
|
|
(".txt", "txt")):
|
||
|
|
if url.endswith(suffix):
|
||
|
|
return tp,url
|
||
|
|
return "???", url
|
||
|
|
|
||
|
|
all_ok = 1
|
||
|
|
def emit(f,ent):
|
||
|
|
global all_ok
|
||
|
|
|
||
|
|
errs = ent._check()
|
||
|
|
if master.byKey.has_key(ent.key.strip().lower()):
|
||
|
|
errs.append("ERROR: Key collision with master file")
|
||
|
|
|
||
|
|
if errs:
|
||
|
|
all_ok = 0
|
||
|
|
|
||
|
|
note = ent.get("note")
|
||
|
|
if ent.getURL() and not note:
|
||
|
|
ent['note'] = "\url{%s}"%ent.getURL()
|
||
|
|
elif note:
|
||
|
|
m = re.match(r'\\url{(.*)}', note)
|
||
|
|
if m:
|
||
|
|
url = m.group(0)
|
||
|
|
tp = None
|
||
|
|
if url.endswith(".txt"):
|
||
|
|
tp = "txt"
|
||
|
|
elif url.endswith(".ps.gz"):
|
||
|
|
tp = "ps_gz"
|
||
|
|
elif url.endswith(".ps"):
|
||
|
|
tp = "ps_gz"
|
||
|
|
elif url.endswith(".pdf"):
|
||
|
|
tp = "pdf"
|
||
|
|
elif url.endswith(".html"):
|
||
|
|
tp = "html"
|
||
|
|
if tp:
|
||
|
|
ent['www_%s_url'%tp] = url
|
||
|
|
|
||
|
|
if errs:
|
||
|
|
all_ok = 0
|
||
|
|
for e in errs:
|
||
|
|
print >>f, "%%%%", e
|
||
|
|
|
||
|
|
print >>f, ent.format(77, 4, v=1, invStrings=invStrings)
|
||
|
|
|
||
|
|
def emitKnown(f, ent, matches):
|
||
|
|
print >>f, "%% Candidates are:", ", ".join([e.key for g,e in matches])
|
||
|
|
print >>f, "%%"
|
||
|
|
print >>f, "%"+(ent.format(77,4,1,invStrings).replace("\n", "\n%"))
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
if len(sys.argv) != 3:
|
||
|
|
print "reconcile.py expects 2 arguments"
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
config.load(sys.argv[1])
|
||
|
|
|
||
|
|
print "========= Scanning master =========="
|
||
|
|
master = MasterBibTeX()
|
||
|
|
master = BibTeX.parseFile(config.MASTER_BIB, result=master)
|
||
|
|
master.buildIndex()
|
||
|
|
|
||
|
|
print "========= Scanning new file ========"
|
||
|
|
try:
|
||
|
|
fn = sys.argv[2]
|
||
|
|
input = BibTeX.parseFile(fn)
|
||
|
|
except BibTeX.ParseError, e:
|
||
|
|
print "Error parsing %s: %s"%(fn,e)
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
f = open('tmp.bib', 'w')
|
||
|
|
keys = input.newStrings.keys()
|
||
|
|
keys.sort()
|
||
|
|
for k in keys:
|
||
|
|
v = input.newStrings[k]
|
||
|
|
print >>f, "@string{%s = {%s}}"%(k,v)
|
||
|
|
|
||
|
|
invStrings = input.invStrings
|
||
|
|
|
||
|
|
for e in input.entries:
|
||
|
|
if not (e.get('title') and e.get('author')):
|
||
|
|
print >>f, "%%\n%%%% Not enough information to search for a match: need title and author.\n%%"
|
||
|
|
emit(f, e)
|
||
|
|
continue
|
||
|
|
|
||
|
|
matches = master.includes(e, all=1)
|
||
|
|
if not matches:
|
||
|
|
print >>f, "%%\n%%%% This entry is probably new: No match found.\n%%"
|
||
|
|
emit(f, e)
|
||
|
|
else:
|
||
|
|
print >>f, "%%"
|
||
|
|
print >>f, "%%%% Possible match found for this entry; max goodness",\
|
||
|
|
matches[-1][0], "\n%%"
|
||
|
|
emitKnown(f, e, matches)
|
||
|
|
|
||
|
|
if not all_ok:
|
||
|
|
print >>f, "\n\n\nErrors remain; not finished.\n"
|
||
|
|
|
||
|
|
f.close()
|