Merge pull request #152 from seasidesparrow/master

major speed improvement with jats parsing; minor fixes to jats/jats_contrib

pyingest/config/config.py

@@ -200,12 +200,16 @@ def get_uat(data,data_dict):
                 HEX_ENTITY = carr[2].lower()
                 DEC_ENTITY = carr[3].lower()
                 for c in NAME_ENTITY.strip().split():
-                    try:
+                    # preserve greek letters, convert all other high-bit chars
+                    eValue = int(DEC_ENTITY.lstrip('&#').rstrip(';'))
+                    if (eValue >= 913 and eValue <= 969) or (eValue >= 192 and eValue <= 382):
+                        ENTITY_DICTIONARY[UNI_ENTITY.strip()] = c.strip()
+                        ENTITY_DICTIONARY[HEX_ENTITY.strip()] = c.strip()
+                        ENTITY_DICTIONARY[DEC_ENTITY.strip()] = c.strip()
+                    else:
+                        ENTITY_DICTIONARY[UNI_ENTITY.strip()] = DEC_ENTITY.strip()
+                        ENTITY_DICTIONARY[HEX_ENTITY.strip()] = DEC_ENTITY.strip()
                         ENTITY_DICTIONARY[c.strip()] = DEC_ENTITY.strip()
-                    except Exception as e:
-                        print("Error splitting NAME_ENTITY: '%s'" % NAME_ENTITY)
-                ENTITY_DICTIONARY[UNI_ENTITY.strip()] = DEC_ENTITY.strip()
-                ENTITY_DICTIONARY[HEX_ENTITY.strip()] = DEC_ENTITY.strip()
             else:
                 print("broken HTML entity:", l.rstrip())
                 NAME_ENTITY = "xxxxx"
@@ -216,20 +220,33 @@ def get_uat(data,data_dict):
 # ADS-specific translations
 # have been added to html5.txt
 ENTITY_DICTIONARY['&sim;'] = "~"
+ENTITY_DICTIONARY['&#8764;'] = "~"
 ENTITY_DICTIONARY['&Tilde;'] = "~"
 ENTITY_DICTIONARY['&rsquo;'] = "'"
+ENTITY_DICTIONARY['&#8217;'] = "'"
 ENTITY_DICTIONARY['&lsquo;'] = "'"
+ENTITY_DICTIONARY['&#8216;'] = "'"
 ENTITY_DICTIONARY['&nbsp;'] = " "
 ENTITY_DICTIONARY['&mdash;'] = "-"
+ENTITY_DICTIONARY['&#8212;'] = "-"
 ENTITY_DICTIONARY['&ndash;'] = "-"
+ENTITY_DICTIONARY['&#8211;'] = "-"
 ENTITY_DICTIONARY['&rdquo;'] = '"'
+ENTITY_DICTIONARY['&#8221;'] = '"'
 ENTITY_DICTIONARY['&ldquo;'] = '"'
+ENTITY_DICTIONARY['&#8220;'] = '"'
 ENTITY_DICTIONARY['&minus;'] = "-"
+ENTITY_DICTIONARY['&#8722;'] = "-"
 ENTITY_DICTIONARY['&plus;'] = "+"
+ENTITY_DICTIONARY['&#43;'] = "+"
 ENTITY_DICTIONARY['&thinsp;'] = " "
+ENTITY_DICTIONARY['&#8201;'] = " "
 ENTITY_DICTIONARY['&hairsp;'] = " "
+ENTITY_DICTIONARY['&#8202;'] = " "
 ENTITY_DICTIONARY['&ensp;'] = " "
+ENTITY_DICTIONARY['&#8194;'] = " "
 ENTITY_DICTIONARY['&emsp;'] = " "
+ENTITY_DICTIONARY['&#8195;'] = " "
 
 
 # ProQuest harvester

pyingest/config/html5.dat

@@ -1392,144 +1392,3 @@
 𝕩	𝕩	𝕩	𝕩	MATHEMATICAL DOUBLE-STRUCK SMALL X
 𝕪	𝕪	𝕪	𝕪	MATHEMATICAL DOUBLE-STRUCK SMALL Y
 𝕫	𝕫	𝕫	𝕫	MATHEMATICAL DOUBLE-STRUCK SMALL Z
-Α	&Agr;	Α	Α	GREEK CAPITAL LETTER ALPHA
-Β	&Bgr;	Β	Β	GREEK CAPITAL LETTER BETA
-Γ	&Ggr;	Γ	Γ	GREEK CAPITAL LETTER GAMMA
-Δ	&Dgr;	Δ	Δ	GREEK CAPITAL LETTER DELTA
-Ε	&Egr;	Ε	Ε	GREEK CAPITAL LETTER EPSILON
-Ζ	&Zgr;	Ζ	Ζ	GREEK CAPITAL LETTER ZETA
-Η	&EEgr;	Η	Η	GREEK CAPITAL LETTER ETA
-Θ	&THgr;	Θ	Θ	GREEK CAPITAL LETTER THETA
-Ι	&Igr;	Ι	Ι	GREEK CAPITAL LETTER IOTA
-Κ	&Kgr;	Κ	Κ	GREEK CAPITAL LETTER KAPPA
-Λ	&Lgr;	Λ	Λ	GREEK CAPITAL LETTER LAMDA
-Μ	&Mgr;	Μ	Μ	GREEK CAPITAL LETTER MU
-Ν	&Ngr;	Ν	Ν	GREEK CAPITAL LETTER NU
-Ξ	&Xgr;	Ξ	Ξ	GREEK CAPITAL LETTER XI
-Ο	&Ogr;	Ο	Ο	GREEK CAPITAL LETTER OMICRON
-Π	&Pgr;	Π	Π	GREEK CAPITAL LETTER PI
-Ρ	&Rgr;	Ρ	Ρ	GREEK CAPITAL LETTER RHO
-Σ	&Sgr;	Σ	Σ	GREEK CAPITAL LETTER SIGMA
-Τ	&Tgr;	Τ	Τ	GREEK CAPITAL LETTER TAU
-Υ	&Ugr;	Υ	Υ	GREEK CAPITAL LETTER UPSILON
-Φ	&PHgr;	Φ	Φ	GREEK CAPITAL LETTER PHI
-Χ	&KHgr;	Χ	Χ	GREEK CAPITAL LETTER CHI
-Ψ	&PSgr;	Ψ	Ψ	GREEK CAPITAL LETTER PSI
-Ω	&OHgr;	Ω	Ω	GREEK CAPITAL LETTER OMEGA
-α	&agr;	α	α	GREEK SMALL LETTER ALPHA
-β	&bgr;	β	β	GREEK SMALL LETTER BETA
-γ	&ggr;	γ	γ	GREEK SMALL LETTER GAMMA
-δ	&dgr;	δ	δ	GREEK SMALL LETTER DELTA
-ε	&egr;	ε	ε	GREEK SMALL LETTER EPSILON
-ζ	&zgr;	ζ	ζ	GREEK SMALL LETTER ZETA
-η	&eegr;	η	η	GREEK SMALL LETTER ETA
-θ	&thgr;	θ	θ	GREEK SMALL LETTER THETA
-ι	&igr;	ι	ι	GREEK SMALL LETTER IOTA
-κ	&kgr;	κ	κ	GREEK SMALL LETTER KAPPA
-λ	&lgr;	λ	λ	GREEK SMALL LETTER LAMDA
-μ	&mgr;	μ	μ	GREEK SMALL LETTER MU
-ν	&ngr;	ν	ν	GREEK SMALL LETTER NU
-ξ	&xgr;	ξ	ξ	GREEK SMALL LETTER XI
-ο	&ogr;	ο	ο	GREEK SMALL LETTER OMICRON
-π	&pgr;	π	π	GREEK SMALL LETTER PI
-ρ	&rgr;	ρ	ρ	GREEK SMALL LETTER RHO
-ς	&sfgr;	ς	ς	GREEK SMALL LETTER FINAL SIGMA
-σ	&sgr;	σ	σ	GREEK SMALL LETTER SIGMA
-τ	&tgr;	τ	τ	GREEK SMALL LETTER TAU
-υ	&ugr;	υ	υ	GREEK SMALL LETTER UPSILON
-φ	&phgr;	φ	φ	GREEK SMALL LETTER PHI
-χ	&khgr;	χ	χ	GREEK SMALL LETTER CHI
-ψ	&psgr;	ψ	ψ	GREEK SMALL LETTER PSI
-ω	&ohgr;	ω	ω	GREEK SMALL LETTER OMEGA
-Ё	Ё	Ё	Ё	capital IO, Russian
-Ђ	Ђ	Ђ	Ђ	capital DJE, Serbian
-Ѓ	Ѓ	Ѓ	Ѓ	capital GJE Macedonian
-Є	Є	Є	Є	capital JE, Ukrainian
-Ѕ	Ѕ	Ѕ	Ѕ	capital DSE, Macedonian
-І	І	І	І	capital I, Ukrainian
-Ї	Ї	Ї	Ї	capital YI, Ukrainian
-Ј	Ј	Ј	Ј	capital JE, Serbian
-Љ	Љ	Љ	Љ	capital LJE, Serbian
-Њ	Њ	Њ	Њ	capital NJE, Serbian
-Ћ	Ћ	Ћ	Ћ	capital TSHE, Serbian
-Ќ	Ќ	Ќ	Ќ	capital KJE, Macedonian
-Ў	Ў	Ў	Ў	capital U, Byelorussian
-Џ	Џ	Џ	Џ	capital dze, Serbian
-А	А	А	А	capital A, Cyrillic
-Б	Б	Б	Б	capital BE, Cyrillic
-В	В	В	В	capital VE, Cyrillic
-Г	Г	Г	Г	capital GHE, Cyrillic
-Д	Д	Д	Д	capital DE, Cyrillic
-Е	Е	Е	Е	capital IE, Cyrillic
-Ж	Ж	Ж	Ж	capital ZHE, Cyrillic
-З	З	З	З	capital ZE, Cyrillic
-И	И	И	И	capital I, Cyrillic
-Й	Й	Й	Й	capital short I, Cyrillic
-К	К	К	К	capital KA, Cyrillic
-Л	Л	Л	Л	capital EL, Cyrillic
-М	М	М	М	capital EM, Cyrillic
-Н	Н	Н	Н	capital EN, Cyrillic
-О	О	О	О	capital O, Cyrillic
-П	П	П	П	capital PE, Cyrillic
-Р	Р	Р	Р	capital ER, Cyrillic
-С	С	С	С	capital ES, Cyrillic
-Т	Т	Т	Т	capital TE, Cyrillic
-У	У	У	У	capital U, Cyrillic
-Ф	Ф	Ф	Ф	capital EF, Cyrillic
-Х	Х	Х	Х	capital HA, Cyrillic
-Ц	Ц	Ц	Ц	capital TSE, Cyrillic
-Ч	Ч	Ч	Ч	capital CHE, Cyrillic
-Ш	Ш	Ш	Ш	capital SHA, Cyrillic
-Щ	Щ	Щ	Щ	capital SHCHA, Cyrillic
-Ъ	Ъ	Ъ	Ъ	capital HARD sign, Cyrillic
-Ы	Ы	Ы	Ы	capital YERU, Cyrillic
-Ь	Ь	Ь	Ь	capital SOFT sign, Cyrillic
-Э	Э	Э	Э	capital E, Cyrillic
-Ю	Ю	Ю	Ю	capital YU, Cyrillic
-Я	Я	Я	Я	capital YA, Cyrillic
-а	а	а	а	small a, Cyrillic
-б	б	б	б	small be, Cyrillic
-в	в	в	в	small ve, Cyrillic
-г	г	г	г	small ghe, Cyrillic
-д	д	д	д	small de, Cyrillic
-е	е	е	е	small ie, Cyrillic
-ж	ж	ж	ж	small zhe, Cyrillic
-з	з	з	з	small ze, Cyrillic
-и	и	и	и	small i, Cyrillic
-й	й	й	й	small short i, Cyrillic
-к	к	к	к	small ka, Cyrillic
-л	л	л	л	small el, Cyrillic
-м	м	м	м	small em, Cyrillic
-н	н	н	н	small en, Cyrillic
-о	о	о	о	small o, Cyrillic
-п	п	п	п	small pe, Cyrillic
-р	р	р	р	small er, Cyrillic
-с	с	с	с	small es, Cyrillic
-т	т	т	т	small te, Cyrillic
-у	у	у	у	small u, Cyrillic
-ф	ф	ф	ф	small ef, Cyrillic
-х	х	х	х	small ha, Cyrillic
-ц	ц	ц	ц	small tse, Cyrillic
-ч	ч	ч	ч	small che, Cyrillic
-ш	ш	ш	ш	small sha, Cyrillic
-щ	щ	щ	щ	small shcha, Cyrillic
-ъ	ъ	ъ	ъ	small hard sign, Cyrillic
-ы	ы	ы	ы	small yeru, Cyrillic
-ь	ь	ь	ь	small soft sign, Cyrillic
-э	э	э	э	small e, Cyrillic
-ю	ю	ю	ю	small yu, Cyrillic
-я	я	я	я	small ya, Cyrillic
-ё	ё	ё	ё	small io, Russian
-ђ	ђ	ђ	ђ	small dje, Serbian
-ѓ	ѓ	ѓ	ѓ	small gje, Macedonian
-є	є	є	є	small je, Ukrainian
-ѕ	ѕ	ѕ	ѕ	small dse, Macedonian
-і	і	і	і	small i, Ukrainian
-ї	ї	ї	ї	small yi, Ukrainian
-ј	ј	ј	ј	small je, Serbian
-љ	љ	љ	љ	small lje, Serbian
-њ	њ	њ	њ	small nje, Serbian
-ћ	ћ	ћ	ћ	small tshe, Serbian
-ќ	ќ	ќ	ќ	small kje Macedonian
-ў	ў	ў	ў	small u, Byelorussian
-џ	џ	џ	џ	small dze, Serbian

pyingest/parsers/entity_convert.py

@@ -1,7 +1,9 @@
 import re
-import namedentities
+from namedentities import named_entities
 from pyingest.config import config
 
+re_ents = re.compile(r'&[a-z0-9]+;|&#[0-9]{1,6};|&#x[0-9a-fA-F]{1,6};')
+
 
 class EntityConverter(object):
 
@@ -11,9 +13,14 @@ def __init__(self):
         self.ent_dict = config.ENTITY_DICTIONARY
 
     def convert(self):
-        o = namedentities.named_entities(self.input_text)
-        # ox = o
-        for k, v in self.ent_dict.items():
-            # ox = re.sub(k, v, ox)
-            o = re.sub(k, v, o)
+        o = named_entities(self.input_text)
+        oents = list(dict.fromkeys(re.findall(re_ents,o)))
+
+        for e in oents:
+            try:
+                enew = self.ent_dict[e]
+            except:
+                pass
+            else:
+                o = re.sub(e, enew, o)
         self.output_text = o