from bs4 import BeautifulSoup import requests import urllib.parse JISHO_BASE_URL = "https://jisho.org/search" def _meaning_tag_to_abbrev(tag: str) -> str: if "Godan" in tag: return "G1" if "Ichidan" in tag: return "G2" if "Suru" in tag: return "G3" if "Noun" in tag: return "n." if "I-adjective" in tag: return "い-adj." if "Na-adjective" in tag: return "な-adj." if "No-adjective" in tag: return "の-adj." if "Adverb" in tag: return "adv." if "Expression" in tag: return "expr." return tag def inject_kanji_meanings(entries: list) -> list: entries_with_meanings = [] for index, entry in enumerate(entries): kanji = entry.kanji if len(entry.meanings) > 0: print(f"Meaning data exists for #{index}: {kanji}. Skipping...") entries_with_meanings.append(entry) continue print(f"Fetching meaning data from Jisho for #{index}: {kanji}") resource_part = urllib.parse.quote_plus(f"{kanji}#kanji") url = f"{JISHO_BASE_URL}/{resource_part}" resp = requests.get(url) resp.raise_for_status() soup = BeautifulSoup(resp.text, "html.parser") container = soup.select_one("div#page_container") meanings = container.select(".kanji-details__main-meanings") kanji_meaning_texts = map(lambda e: e.text.strip(), meanings) kanji_characters = container.select(".character") kanji_character_texts = map(lambda e: e.text.strip(), kanji_characters) result_tuples = zip(kanji_character_texts, kanji_meaning_texts) result_strs = map(lambda t: " ".join(t), result_tuples) result = "
".join(result_strs) entry.meanings = result entries_with_meanings.append(entry) return entries_with_meanings def inject_english_translation_and_note(entries: list) -> list: entries_with_translations = [] for index, entry in enumerate(entries): kanji = entry.kanji if len(entry.english) > 0: print(f"English translation exists for #{index}: {kanji}. Skipping...") entries_with_translations.append(entry) continue print(f"Fetching translation from Jisho for #{index}: {kanji}") resource_part = urllib.parse.quote_plus(f"{kanji}") url = f"{JISHO_BASE_URL}/{resource_part}" resp = requests.get(url) resp.raise_for_status() soup = BeautifulSoup(resp.text, "html.parser") container = soup.select_one("div#page_container") meaning_wrapper = container.select(".meanings-wrapper")[0] meaning_tags = meaning_wrapper.select(".meaning-tags") meaning_tags_texts = map(lambda e: e.text.strip(), meaning_tags) meaning_tag_abbrevs = map(_meaning_tag_to_abbrev, meaning_tags_texts) meaning_translations = meaning_wrapper.select(".meaning-wrapper .meaning-meaning") meaning_translations_texts = map(lambda e: e.text.strip(), meaning_translations) meaning_sentences = meaning_wrapper.select(".sentence") meaning_sentence = meaning_sentences[0] if len(meaning_sentences) > 0 else None if meaning_sentence is not None: sentence_japanese_parts = meaning_sentence.select(".japanese .unlinked") sentence_japanese_parts_texts = map(lambda e: e.text.strip(), sentence_japanese_parts) sentence_japanese_text = "".join(sentence_japanese_parts_texts) sentence_english_text = meaning_sentence.select(".english")[0].text.strip() note_sentence = f"{sentence_japanese_text}
{sentence_english_text}" else: note_sentence = "" meanings_zipped = zip(meaning_tag_abbrevs, meaning_translations_texts) first_translation_tuple = list(meanings_zipped)[0] result = f"{first_translation_tuple[0]} {first_translation_tuple[1]}" entry.english = result entry.note = note_sentence entries_with_translations.append(entry) return entries_with_translations