{}
'.format(s) for s in splits) def bibtype(key, entries, ulog): type_mapping = { 'journal-article': 'article', 'unpublished': 'unpublished', # TODO: More type mappings. Is there any dx.doi.org documentation for these? } s = entries[key].get('type', '') if s in type_mapping: return type_mapping[s] if str(s).strip() == "": ulog.warn("No type specified for {}. Using `article`".format(key)) return 'article' ulog.warn("Unknown type `{}` specified for {}".format(s, key)) return str(s) def latex_escape(s): # http://stackoverflow.com/questions/16259923/ # http://stackoverflow.com/a/4580132 conv = {'&': r'\&', '%': r'\%', '$': r'\$', '#': r'\#', '_': r'\_', '{': r'\{', '}': r'\}', '~': r'\textasciitilde{}', '^': r'\^{}', '\\': r'\textbackslash{}', '<': r'\textless', '>': r'\textgreater', # no breaking space '\u00A0': '~', } accents = dict([ # Grave accents (u"à", "\\`a"), (u"è", "\\`e"), (u"ì", "\\`\\i"), (u"ò", "\\`o"), (u"ù", "\\`u"), (u"ỳ", "\\`y"), (u"À", "\\`A"), (u"È", "\\`E"), (u"Ì", "\\`\\I"), (u"Ò", "\\`O"), (u"Ù", "\\`U"), (u"Ỳ", "\\`Y"), (u"á", "\\'a"), # Acute accent (u"é", "\\'e"), (u"í", "\\'\\i"), (u"ó", "\\'o"), (u"ú", "\\'u"), (u"ý", "\\'y"), (u"Á", "\\'A"), (u"É", "\\'E"), (u"Í", "\\'\\I"), (u"Ó", "\\'O"), (u"Ú", "\\'U"), (u"Ý", "\\'Y"), (u"â", "\\^a"), # Circumflex (u"ê", "\\^e"), (u"î", "\\^\\i"), (u"ô", "\\^o"), (u"û", "\\^u"), (u"ŷ", "\\^y"), (u"Â", "\\^A"), (u"Ê", "\\^E"), (u"Î", "\\^\\I"), (u"Ô", "\\^O"), (u"Û", "\\^U"), (u"Ŷ", "\\^Y"), (u"ä", "\\\"a"), # Umlaut or dieresis (u"ë", "\\\"e"), (u"ï", "\\\"\\i"), (u"ö", "\\\"o"), (u"ü", "\\\"u"), (u"ÿ", "\\\"y"), (u"Ä", "\\\"A"), (u"Ë", "\\\"E"), (u"Ï", "\\\"\\I"), (u"Ö", "\\\"O"), (u"Ü", "\\\"U"), (u"Ÿ", "\\\"Y"), (u"ç", "\\c{c}"), # Cedilla (u"Ç", "\\c{C}"), (u"œ", "{\\oe}"), # Ligatures (u"Œ", "{\\OE}"), (u"æ", "{\\ae}"), (u"Æ", "{\\AE}"), (u"å", "{\\aa}"), (u"Å", "{\\AA}"), (u"–", "--"), # Dashes (u"—", "---"), (u"ø", "{\\o}"), # Misc latin-1 letters (u"Ø", "{\\O}"), (u"ß", "{\\ss}"), (u"¡", "{!`}"), (u"¿", "{?`}"), (u"\\", "\\\\"), # Characters that should be quoted (u"~", "\\~"), (u"&", "\\&"), (u"$", "\\$"), (u"{", "\\{"), (u"}", "\\}"), (u"%", "\\%"), (u"#", "\\#"), (u"_", "\\_"), (u"≥", "$\\ge$"), # Math operators (u"≤", "$\\le$"), (u"≠", "$\\neq$"), (u"©", "\copyright"), # Misc (u"ı", "{\\i}"), (u"µ", "$\\mu$"), (u"°", "$\\deg$"), (u"‘", "`"), # Quotes (u"’", "'"), (u"“", "``"), (u"”", "''"), (u"‚", ","), (u"„", ",,"), ]) for k in accents: # This extra bracketing is necessary for bibtex accents[k] = "{%s}" % accents[k] conv.update(accents) regex = re.compile('|'.join(re.escape(key) for key in sorted(conv.keys(), key=lambda item: - len(item)))) return regex.sub(lambda match: conv[match.group()], s) # Rendering is straightforward application of jinja2. Note that we have # to pass a sorted list of ident's (keys to the entries dictionary) def sort_entry_date(entries, k): entry = entries[k] if 'published-online' in entry: return entry['published-online'] if 'published-print' in entry: return entry['published-print'] log.warn("Missing date for {}".format(k)) return datetime.date(1970, 1, 1) def sort_entry_title(entries, k): entry = entries[k] if 'title' in entry: return entry['title'] log.warn("Missing title for {} (for sorting)".format(k)) return "zzzz" def sort_entry_key(entries, k): return sort_entry_date(entries, k), sort_entry_title(entries, k) def is_stubbable(ident): return ident.startswith("doi:") or ident.startswith("arxiv:") def stub(ident, *, session, ulog): my_meta = {} if ident.startswith('doi:'): my_meta['doi'] = ident[len('doi:'):] elif ident.startswith('arxiv:'): my_meta['arxiv'] = ident[len('arxiv:'):] else: raise ValueError("Not stubbable") ulog.info("Creating a stub for {}".format(ident)) return ident, _internal_representation(ident, my_meta, session=session, ulog=ulog) def extract_citations_from_description(text, *, ulog): cites = [] references = [] # [ident] or [ident=111] NOT [ident](... for ma in re.finditer(IN_TEXT_CITATION_RE, text): i, _doi_ident, _arxiv_ident, _normal_ident, _equals_sign, n = ma.groups() if n is not None: # If no number is specified, we don't want it to show up in the references table cites += [{'id': i, 'num': n}] ulog.debug('Extracted citation for {} numbered {}'.format(i, n)) else: references += [{'id': i}] ulog.debug("Extracted a reference to {}".format(i)) return cites, references def resolve_short_description_crossrefs(text, ident, entry, *, ulog): def _replace1(ma): num = int(ma.groups()[0]) if not 'cites' in entry: ulog.warn("{} uses short description references for ref {} " "but doesn't have citations listed".format(ident, num)) return "(ref. {n})".format(n=num) for cite in entry['cites']: if 'num' in cite and 'id' in cite and cite['num'] == int(num): return "[{i}={n}]".format(i=cite['id'], n=num) ulog.warn("{} uses short description references for ref {} " "but this citation wasn't found in the citation list".format(ident, num)) return "(ref. {n})".format(n=num) # [ident] or [ident=111] NOT [ident](... text = re.sub(SHORT_IN_TEXT_CITATION_RE, _replace1, text) return text def resolve_crossrefs(entries, *, session, ulog): # TODO: Maybe do (a subset of the markdownification) here and # TODO: also add those things to cites / do error checking / whatever stubs = [] for ident, entry in entries.items(): if 'description' in entry: ulog.debug("Trying to extract references from {}'s description".format(ident)) cites, references = extract_citations_from_description(entry['description'], ulog=ulog) if len(cites) > 0: if 'cites' in entry: entry['cites'] += cites else: entry['cites'] = cites if len(references) > 0: if 'references' in entry: entry['references'] += references else: entry['references'] = references if 'cites' in entry: ulog.debug("Processing citations for {}".format(ident)) for cite in entry['cites']: if 'id' in cite: if cite['id'] in entries: cite['resolved'] = True else: if is_stubbable(cite['id']): stubs += [stub(cite['id'], session=session, ulog=ulog)] cite['resolved'] = True else: cite['resolved'] = False else: ulog.warn("{}'s citation doesn't contain `id`: {}".format(ident, cite)) if 'references' in entry: ulog.debug("Processing crossreferences for {}".format(ident)) for ref in entry['references']: if ref['id'] in entries: ref['resolved'] = True else: ulog.warn("{}'s reference to {} is unresolved".format(ident, ref['id'])) ref['resolved'] = False if 'description' in entry: entry['description'] = resolve_short_description_crossrefs(entry['description'], ident, entry, ulog=ulog) entries.update(dict(stubs)) return entries def render_all(entries, should_be_true, *, ulog): if not should_be_true: ulog.error("When using `all` output, set it to `True`") return {} return sorted(entries.keys()) def render_categories(entries, want_tags, *, ulog): matching_idents = [] for ident, entry in entries.items(): its_tags = entry.get('tags', []) for it in its_tags: if it in want_tags: matching_idents += [ident] return matching_idents def _render_tree(node_ident, entries, matching_entries, ulog): # Deprecated node = entries[node_ident] matching_entries[node_ident] = node if 'cites' in node: for cite in node['cites']: if 'resolved' in cite and cite['resolved']: _render_tree(cite['id'], entries, matching_entries, ulog) else: if 'id' in cite: ulog.warn("{}'s unresolved citation {} won't be included in the `tree` output." .format(node_ident, cite['id'])) return matching_entries def render_tree(entries, root, *, ulog): # Deprecated matching_entries = dict() matching_entries = _render_tree(root, entries, matching_entries, ulog) return matching_entries def _descendants(ident, entries, out_idents, *, ulog): node = entries[ident] if 'cites' in node: for cite in node['cites']: if 'resolved' in cite and cite['resolved']: out_idents.add(cite['id']) _descendants(cite['id'], entries, out_idents, ulog=ulog) else: if 'id' in cite: ulog.warn("{}'s unresolved citation {} won't be included as a descendant." .format(ident, cite['id'])) return out_idents def descendants(idents, entries, *, ulog): out_idents = set() for ident in idents: out_idents.update(_descendants(ident, entries, out_idents, ulog=ulog)) return sorted(out_idents) def render_by_input_filename(entries, input_fn, *, ulog): idents = [] for k, v in entries.items(): if v.get('input_fn', None) == input_fn: idents += [k] return idents class Renderfunc: def __init__(self, fn, fext, list_of_idents, entries, ulog): env = Environment(loader=PackageLoader('gitbib')) env.filters['latex_escape'] = latex_escape env.filters['bibtype'] = lambda k: bibtype(k, entries, ulog) env.filters['pretty_author_list'] = pretty_author_list env.filters['bibtex_author_list'] = bibtex_author_list env.filters['bibtex_capitalize'] = bibtex_capitalize env.filters['to_isodate'] = to_isodate env.filters['to_prettydate'] = to_prettydate env.filters['respace'] = respace env.filters['safe_css'] = safe_css env.filters['list_of_pdbs'] = list_of_pdbs env.filters['markdownify'] = lambda s: markdownify(s, entries) sorted_tags = sorted(set( itertools.chain.from_iterable(entries[k].get('tags', []) for k in itertools.chain.from_iterable(list_of_idents)))) idents_by_tag = {tag: [k for k in itertools.chain.from_iterable(list_of_idents) if tag in entries[k].get('tags', [])] for tag in sorted_tags} list_of_sorted_ids = [] for idents in list_of_idents: sorted_idents = sorted(idents, reverse=True, key=lambda k: sort_entry_key(entries, k)) list_of_sorted_ids += [sorted_idents] self.fn = fn self.fext = fext self.env = env self.entries = entries self.list_of_sorted_ids = list_of_sorted_ids self.all_tags = sorted_tags self.idents_by_tag = idents_by_tag def __call__(self, out_f, user_info): template = self.env.get_template('template.{}'.format(self.fext)) out_f.write(template.render( fn=self.fn, entries=self.entries, list_of_idents=self.list_of_sorted_ids, idents_by_tag=self.idents_by_tag, all_tags=self.all_tags, user_info=user_info, ).encode()) class IndexRenderfunc: def __init__(self, out_config, out_fmts): env = Environment(loader=PackageLoader('gitbib')) self.env = env self.out_config = out_config self.out_fmts = out_fmts def __call__(self, out_f, user_info): template = self.env.get_template('index.html') out_f.write(template.render( out_fmts=self.out_fmts, out_config=self.out_config, ).encode()) class Gitbib: def __init__(self, *, session, user_logger, repo_dir=".", gitbib_yaml_fn='gitbib.yaml'): ulog = user_logger ulog.debug("Connected to logging.") unknown_err_str = "An unknown error occured ({}): {} ({}). Please alert the developers" try: config, my_meta = read_yamls(repo_dir, gitbib_yaml_fn, ulog=ulog) except GitbibFileNotFoundError as e: ulog.error("Gitbib file not found. Are you sure this is a gitbib repository?") raise e except DuplicateKeyError as e: ulog.error("Found a duplicate key: `{}`. Keys must be unique.".format(e.duplicate_key)) raise e except Exception as e: ulog.error(unknown_err_str.format(1, e, type(e))) raise e try: entries = internal_representation(my_meta, session=session, ulog=ulog) except Exception as e: ulog.error(unknown_err_str.format(2, e, type(e))) raise e try: entries = resolve_crossrefs(entries, session=session, ulog=ulog) except Exception as e: ulog.error(unknown_err_str.format(3, e, type(e))) raise e self.entries = entries self.config = config out_render_types = { 'all': render_all, 'categories': render_categories, 'input-fn': render_by_input_filename, } out_render_formats = { 'html': 'text/html', 'bib': 'application/x-bibtex', 'tex': 'application/x-latex', 'md': 'text/markdown', } def renderers(self, out_formats, user_logger): ulog = user_logger fns = set() out_formats = set(out_formats) for out_spec in self.config['outputs']: if 'fn' not in out_spec: ulog.error("No output filename given: {}".format(out_spec)) continue fn = out_spec['fn'] if not re.match(r'[\w\-\.]+$', fn): ulog.error("Please use a filename that is only alphanumeric characters. Not {}".format(fn)) continue if fn in fns: ulog.error("Duplicate output filename! {}".format(fn)) continue fns.add(fn) n_out_types = 0 out_type = None for ot in self.out_render_types: if ot in out_spec: n_out_types += 1 out_type = ot if n_out_types > 1: ulog.error("Too many output specifications given for {}".format(fn)) continue if n_out_types < 1: ulog.error("No output specification given for {}".format(fn)) continue idents = self.out_render_types[out_type](self.entries, out_spec[out_type], ulog=ulog) list_of_idents = [idents] if out_type in ['all']: if 'include_descendants' in out_spec: ulog.warn("`include_descendants` option is ignored for output type `{}`".format(out_type)) else: include_descendants = out_spec.get('include_descendants', False) if include_descendants: list_of_idents += [descendants(idents, self.entries, ulog=ulog)] if len(idents) > 0: for ofmt in out_formats: yield "{}.{}".format(fn, ofmt), self.out_render_formats[ofmt], Renderfunc(fn, ofmt, list_of_idents, self.entries, ulog=ulog) else: ulog.warn("No entries matched the specification for {}".format(fn)) yield "index.html", "text/html", IndexRenderfunc(self.config, out_formats) PK *]J gitbib/query.pyimport requests import time headers = {'Accept': 'application/json; charset=utf-8'} url = "https://api.crossref.org" def query(entry): e_query = [] if 'title' in entry: e_query += [('query.title', entry['title'])] if 'author' in entry: e_query += [('query.author', '+'.join('+'.join(x.split(', ')) for x in entry['author']))] if 'journal' in entry: e_query += [('query.container-title', entry['journal'])] e_query += [('sort', 'score')] e_query = ['{}={}'.format(x1, x2) for x1, x2 in e_query] q_string = "{}/works?{}".format(url, '&'.join(e_query)) r = requests.get(q_string, headers=headers) items = r.json()['message']['items'] return { 'doi': items[0]['DOI'], 'score': items[0]['score'], } PK iJ9 9 gitbib/templates/index.html{# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. #}{{ out_config['description'] }}
{% endif %} {% for out_spec in out_config['outputs'] %}{{ out_spec['description'] }}
{% endif %} {% endfor %}{{entry['author'] | pretty_author_list }}
{% if entry['published-online'] %} {{entry['published-online'] | to_isodate }} (online) {% endif %} {% if entry['published-online'] and entry['published-print'] %} – {% endif %} {% if entry['published-print'] %} {{entry['published-print'] | to_isodate }} (print) {% endif %}
{% if entry['container-title'] %}{{entry['container-title']['full']}} ({{entry['container-title']['short']}}).{% endif %} {% if entry['volume'] %}{{entry['volume']}}, {% endif %} {% if entry['issue'] %}{{entry['issue']}}, {% endif %} {% if entry['page'] %}{{entry['page']}}. {% endif %} {% if entry['doi'] %}doi:{{entry['doi']}}{% endif %} {% if entry['arxiv']%}arxiv:{{entry['arxiv']}}{% endif %}
{% if entry['description'] %}{{ entry['pdb'] | list_of_pdbs }}
{% endif %} {% if entry['cites'] %} Show References| Num | Entry | Why | |
|---|---|---|---|
| {{ cite['num'] }} | {% if cite['resolved'] %}{{ cite['id'] }} | {% else %}{{ cite['id'] }} | {% endif %}{{ cite['why'] }} |
| {{ cite['nums'] | join(', ') }} | {{ cite['why'] }} |
{% for tag in entry['tags'] %} {{tag}} {% endfor %}
{% endif %}