1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
|
from bs4 import BeautifulSoup
import requests
import urllib.parse
JISHO_BASE_URL = "https://jisho.org/search"
def _meaning_tag_to_abbrev(tag: str) -> str:
if "Godan" in tag: return "G1"
if "Ichidan" in tag: return "G2"
if "Suru" in tag: return "G3"
if "Noun" in tag: return "n."
if "I-adjective" in tag: return "い-adj."
if "Na-adjective" in tag: return "な-adj."
if "No-adjective" in tag: return "の-adj."
if "Adverb" in tag: return "adv."
return tag
def inject_kanji_meanings(entries: list) -> list:
entries_with_meanings = []
for index, entry in enumerate(entries):
kanji = entry.kanji
if len(entry.meanings) > 0:
print(f"Meaning data exists for #{index}: {kanji}. Skipping...")
entries_with_meanings.append(entry)
continue
print(f"Fetching meaning data from Jisho for #{index}: {kanji}")
resource_part = urllib.parse.quote_plus(f"{kanji}#kanji")
url = f"{JISHO_BASE_URL}/{resource_part}"
resp = requests.get(url)
resp.raise_for_status()
soup = BeautifulSoup(resp.text, "html.parser")
container = soup.select_one("div#page_container")
meanings = container.select(".kanji-details__main-meanings")
kanji_meaning_texts = map(lambda e: e.text.strip(), meanings)
kanji_characters = container.select(".character")
kanji_character_texts = map(lambda e: e.text.strip(), kanji_characters)
result_tuples = zip(kanji_character_texts, kanji_meaning_texts)
result_strs = map(lambda t: " ".join(t), result_tuples)
result = "<br>".join(result_strs)
entry.meanings = result
entries_with_meanings.append(entry)
return entries_with_meanings
def inject_english_translation_and_note(entries: list) -> list:
entries_with_translations = []
for index, entry in enumerate(entries):
kanji = entry.kanji
if len(entry.english) > 0:
print(f"English translation exists for #{index}: {kanji}. Skipping...")
entries_with_translations.append(entry)
continue
print(f"Fetching translation from Jisho for #{index}: {kanji}")
resource_part = urllib.parse.quote_plus(f"{kanji}")
url = f"{JISHO_BASE_URL}/{resource_part}"
resp = requests.get(url)
resp.raise_for_status()
soup = BeautifulSoup(resp.text, "html.parser")
container = soup.select_one("div#page_container")
meaning_wrapper = container.select(".meanings-wrapper")[0]
meaning_tags = meaning_wrapper.select(".meaning-tags")
meaning_tags_texts = map(lambda e: e.text.strip(), meaning_tags)
meaning_tag_abbrevs = map(_meaning_tag_to_abbrev, meaning_tags_texts)
meaning_translations = meaning_wrapper.select(".meaning-wrapper .meaning-meaning")
meaning_translations_texts = map(lambda e: e.text.strip(), meaning_translations)
meaning_sentences = meaning_wrapper.select(".sentence")
meaning_sentence = meaning_sentences[0] if len(meaning_sentences) > 0 else None
if meaning_sentence is not None:
sentence_japanese_parts = meaning_sentence.select(".japanese .unlinked")
sentence_japanese_parts_texts = map(lambda e: e.text.strip(), sentence_japanese_parts)
sentence_japanese_text = "".join(sentence_japanese_parts_texts)
sentence_english_text = meaning_sentence.select(".english")[0].text.strip()
note_sentence = f"{sentence_japanese_text}<br>{sentence_english_text}"
else:
note_sentence = ""
meanings_zipped = zip(meaning_tag_abbrevs, meaning_translations_texts)
first_translation_tuple = list(meanings_zipped)[0]
result = f"{first_translation_tuple[0]} {first_translation_tuple[1]}"
entry.english = result
entry.note = note_sentence
entries_with_translations.append(entry)
return entries_with_translations
|
