From ae6c39c8e9758172c92136128c811f2387531228 Mon Sep 17 00:00:00 2001 From: Jan Tuomi Date: Tue, 2 Feb 2021 21:28:12 +0200 Subject: Rename to kanji_info_fetcher --- attain_vocab_to_anki_py/__init__.py | 42 --------------- attain_vocab_to_anki_py/io.py | 50 ----------------- attain_vocab_to_anki_py/jisho_client.py | 92 -------------------------------- attain_vocab_to_anki_py/wadoku_client.py | 42 --------------- kanji_info_fetcher/__init__.py | 42 +++++++++++++++ kanji_info_fetcher/io.py | 50 +++++++++++++++++ kanji_info_fetcher/jisho_client.py | 92 ++++++++++++++++++++++++++++++++ kanji_info_fetcher/wadoku_client.py | 42 +++++++++++++++ pyproject.toml | 4 +- tests/kanji_info_fetcher.py | 5 ++ tests/test_attain_vocab_to_anki_py.py | 5 -- 11 files changed, 233 insertions(+), 233 deletions(-) delete mode 100644 attain_vocab_to_anki_py/__init__.py delete mode 100644 attain_vocab_to_anki_py/io.py delete mode 100644 attain_vocab_to_anki_py/jisho_client.py delete mode 100644 attain_vocab_to_anki_py/wadoku_client.py create mode 100644 kanji_info_fetcher/__init__.py create mode 100644 kanji_info_fetcher/io.py create mode 100644 kanji_info_fetcher/jisho_client.py create mode 100644 kanji_info_fetcher/wadoku_client.py create mode 100644 tests/kanji_info_fetcher.py delete mode 100644 tests/test_attain_vocab_to_anki_py.py diff --git a/attain_vocab_to_anki_py/__init__.py b/attain_vocab_to_anki_py/__init__.py deleted file mode 100644 index 59b2753..0000000 --- a/attain_vocab_to_anki_py/__init__.py +++ /dev/null @@ -1,42 +0,0 @@ -import argparse -import attain_vocab_to_anki_py.io -import attain_vocab_to_anki_py.jisho_client -import attain_vocab_to_anki_py.wadoku_client - -__version__ = '0.1.0' - -DEFAULT_FORMAT = 'kanji,english' - -def main(): - parser = argparse.ArgumentParser(description='Convert course vocab into Anki deck') - parser.add_argument('--format', metavar='format', type=str, default=DEFAULT_FORMAT, - help='comma-separated row format, e.g. kanji,hiragana,english,readings') - parser.add_argument('file', metavar='file', type=str, - help='vocabulary file path') - - args = parser.parse_args() - file_path = args.file - row_format = args.format - - print("row_format:", row_format) - - # read entries from file - print(f"[STATUS] Reading file {file_path}") - entries = io.read_from_file(file_path, row_format) - - # fetch translations from Jisho - print(f"[STATUS] Fetching English translations from Jisho") - entries = jisho_client.inject_english_translation(entries) - - # fetch kanji meanings from Jisho - print(f"[STATUS] Fetching kanji meaning data from Jisho") - entries = jisho_client.inject_kanji_meanings(entries) - - # fetch hiragana readings from Wadoku w/ pitch accents - print(f"[STATUS] Fetching reading data from Wadoku") - entries = wadoku_client.inject_hiragana_with_accents(entries) - - # write entries to file - out_file_name = file_path.replace(".csv", ".out.csv") if ".csv" in file_path else f"{file_path}.out" - print(f"[STATUS] Writing result to file {out_file_name}") - io.write_to_file(out_file_name, entries) diff --git a/attain_vocab_to_anki_py/io.py b/attain_vocab_to_anki_py/io.py deleted file mode 100644 index 90c9507..0000000 --- a/attain_vocab_to_anki_py/io.py +++ /dev/null @@ -1,50 +0,0 @@ -from os import path -from dataclasses import dataclass - -@dataclass -class Entry: - kanji: str - hiragana: str - english: str - meanings: str - -def read_from_file(path_str: str, row_format: str) -> list: - p = path.join(path_str) - lines = [] - - format_segments = row_format.split(",") - - with open(p, "r") as f: - lines = f.readlines() - - entries = [] - for line in lines: - if len(line.strip()) == 0: - continue - - value_dict = {} - - row_values = line.split(",", len(format_segments) - 1) - for i, format_segment in enumerate(format_segments): - value_dict[format_segment] = row_values[i] - - entries.append(Entry( - kanji=value_dict.get("kanji", "").strip(), - hiragana=value_dict.get("hiragana", "").strip(), - english=value_dict.get("english", "").strip(), - meanings=value_dict.get("meanings", "").strip() - )) - - return entries - -def _entry_to_row(entry: Entry, delimiter: str = "§") -> str: - s = delimiter.join([entry.kanji, entry.hiragana, entry.english, entry.meanings]) - return f"{s}\n" - -def write_to_file(path_str: str, entries: list): - p = path.join(path_str) - - lines = map(_entry_to_row, entries) - - with open(p, "w") as f: - f.writelines(lines) diff --git a/attain_vocab_to_anki_py/jisho_client.py b/attain_vocab_to_anki_py/jisho_client.py deleted file mode 100644 index d2c0e4f..0000000 --- a/attain_vocab_to_anki_py/jisho_client.py +++ /dev/null @@ -1,92 +0,0 @@ -from bs4 import BeautifulSoup -import requests -import urllib.parse - -JISHO_BASE_URL = "https://jisho.org/search" - -def _meaning_tag_to_abbrev(tag: str) -> str: - if "Godan" in tag: return "G1" - if "Ichidan" in tag: return "G2" - if "Suru" in tag: return "G3" - if "Noun" in tag: return "N." - if "I-adjective" in tag: return "い-adj." - if "Na-adjective" in tag: return "な-adj." - if "No-adjective" in tag: return "の-adj." - if "Adverb" in tag: return "adv." - return tag - -def inject_kanji_meanings(entries: list) -> list: - entries_with_meanings = [] - - for index, entry in enumerate(entries): - kanji = entry.kanji - - if len(entry.meanings) > 0: - print(f"Meaning data exists for #{index}: {kanji}. Skipping...") - entries_with_meanings.append(entry) - continue - - print(f"Fetching meaning data from Jisho for #{index}: {kanji}") - - resource_part = urllib.parse.quote_plus(f"{kanji}#kanji") - url = f"{JISHO_BASE_URL}/{resource_part}" - resp = requests.get(url) - resp.raise_for_status() - - soup = BeautifulSoup(resp.text, "html.parser") - container = soup.select_one("div#page_container") - - meanings = container.select(".kanji-details__main-meanings") - kanji_meaning_texts = map(lambda e: e.text.strip(), meanings) - - kanji_characters = container.select(".character") - kanji_character_texts = map(lambda e: e.text.strip(), kanji_characters) - - result_tuples = zip(kanji_character_texts, kanji_meaning_texts) - result_strs = map(lambda t: " ".join(t), result_tuples) - result = "
".join(result_strs) - - entry.meanings = result - entries_with_meanings.append(entry) - - return entries_with_meanings - -def inject_english_translation(entries: list) -> list: - entries_with_translations = [] - - for index, entry in enumerate(entries): - kanji = entry.kanji - - if len(entry.english) > 0: - print(f"English translation exists for #{index}: {kanji}. Skipping...") - entries_with_translations.append(entry) - continue - - print(f"Fetching translation from Jisho for #{index}: {kanji}") - - resource_part = urllib.parse.quote_plus(f"{kanji}") - url = f"{JISHO_BASE_URL}/{resource_part}" - resp = requests.get(url) - resp.raise_for_status() - - soup = BeautifulSoup(resp.text, "html.parser") - container = soup.select_one("div#page_container") - - meaning_wrapper = container.select(".meanings-wrapper")[0] - - meaning_tags = meaning_wrapper.select(".meaning-tags") - meaning_tags_texts = map(lambda e: e.text.strip(), meaning_tags) - meaning_tag_abbrevs = map(_meaning_tag_to_abbrev, meaning_tags_texts) - - meaning_translations = meaning_wrapper.select(".meaning-wrapper .meaning-meaning") - meaning_translations_texts = map(lambda e: e.text.strip(), meaning_translations) - - meanings_zipped = zip(meaning_tag_abbrevs, meaning_translations_texts) - - first_translation_pair = list(meanings_zipped)[0] - result = f"{first_translation_pair[0]} {first_translation_pair[1]}" - - entry.english = result - entries_with_translations.append(entry) - - return entries_with_translations diff --git a/attain_vocab_to_anki_py/wadoku_client.py b/attain_vocab_to_anki_py/wadoku_client.py deleted file mode 100644 index 550f801..0000000 --- a/attain_vocab_to_anki_py/wadoku_client.py +++ /dev/null @@ -1,42 +0,0 @@ -from bs4 import BeautifulSoup -import requests -import urllib.parse - -WADOKU_BASE_URL = "https://www.wadoku.de/search" - -def inject_hiragana_with_accents(entries: list) -> list: - entries_with_readings = [] - - for index, entry in enumerate(entries): - kanji = entry.kanji.replace("~", "") - - if len(entry.hiragana) > 0: - print(f"Hiragana exists for #{index}: {kanji}. Skipping...") - entries_with_readings.append(entry) - continue - - # Remove nasty characters from kanji since Wadoku - # fails to find anything otherwise - print(f"Fetching hiragana data for #{index}: {kanji}") - - resource_part = urllib.parse.quote_plus(kanji) - url = f"{WADOKU_BASE_URL}/{resource_part}" - resp = requests.get(url) - resp.raise_for_status() - - soup = BeautifulSoup(resp.text, "html.parser") - container = soup.select_one("section#content") - - readings = container.select("span.reading") - first_reading = str(readings[0]) - - # Remove extra characters added by Wadoku - # https://www.wadoku.de/wiki/display/WAD/Hinweise+zur+Notation - result = first_reading \ - .replace("~", "") \ - .replace("|", "") \ - .replace("・", "") - entry.hiragana = result - entries_with_readings.append(entry) - - return entries_with_readings diff --git a/kanji_info_fetcher/__init__.py b/kanji_info_fetcher/__init__.py new file mode 100644 index 0000000..c4b95bb --- /dev/null +++ b/kanji_info_fetcher/__init__.py @@ -0,0 +1,42 @@ +import argparse +import kanji_info_fetcher.io +import kanji_info_fetcher.jisho_client +import kanji_info_fetcher.wadoku_client + +__version__ = '0.1.0' + +DEFAULT_FORMAT = 'kanji,english' + +def main(): + parser = argparse.ArgumentParser(description='Convert course vocab into Anki deck') + parser.add_argument('--format', metavar='format', type=str, default=DEFAULT_FORMAT, + help='comma-separated row format, e.g. kanji,hiragana,english,readings') + parser.add_argument('file', metavar='file', type=str, + help='vocabulary file path') + + args = parser.parse_args() + file_path = args.file + row_format = args.format + + print("row_format:", row_format) + + # read entries from file + print(f"[STATUS] Reading file {file_path}") + entries = io.read_from_file(file_path, row_format) + + # fetch translations from Jisho + print(f"[STATUS] Fetching English translations from Jisho") + entries = jisho_client.inject_english_translation(entries) + + # fetch kanji meanings from Jisho + print(f"[STATUS] Fetching kanji meaning data from Jisho") + entries = jisho_client.inject_kanji_meanings(entries) + + # fetch hiragana readings from Wadoku w/ pitch accents + print(f"[STATUS] Fetching reading data from Wadoku") + entries = wadoku_client.inject_hiragana_with_accents(entries) + + # write entries to file + out_file_name = file_path.replace(".csv", ".out.csv") if ".csv" in file_path else f"{file_path}.out" + print(f"[STATUS] Writing result to file {out_file_name}") + io.write_to_file(out_file_name, entries) diff --git a/kanji_info_fetcher/io.py b/kanji_info_fetcher/io.py new file mode 100644 index 0000000..90c9507 --- /dev/null +++ b/kanji_info_fetcher/io.py @@ -0,0 +1,50 @@ +from os import path +from dataclasses import dataclass + +@dataclass +class Entry: + kanji: str + hiragana: str + english: str + meanings: str + +def read_from_file(path_str: str, row_format: str) -> list: + p = path.join(path_str) + lines = [] + + format_segments = row_format.split(",") + + with open(p, "r") as f: + lines = f.readlines() + + entries = [] + for line in lines: + if len(line.strip()) == 0: + continue + + value_dict = {} + + row_values = line.split(",", len(format_segments) - 1) + for i, format_segment in enumerate(format_segments): + value_dict[format_segment] = row_values[i] + + entries.append(Entry( + kanji=value_dict.get("kanji", "").strip(), + hiragana=value_dict.get("hiragana", "").strip(), + english=value_dict.get("english", "").strip(), + meanings=value_dict.get("meanings", "").strip() + )) + + return entries + +def _entry_to_row(entry: Entry, delimiter: str = "§") -> str: + s = delimiter.join([entry.kanji, entry.hiragana, entry.english, entry.meanings]) + return f"{s}\n" + +def write_to_file(path_str: str, entries: list): + p = path.join(path_str) + + lines = map(_entry_to_row, entries) + + with open(p, "w") as f: + f.writelines(lines) diff --git a/kanji_info_fetcher/jisho_client.py b/kanji_info_fetcher/jisho_client.py new file mode 100644 index 0000000..e74412e --- /dev/null +++ b/kanji_info_fetcher/jisho_client.py @@ -0,0 +1,92 @@ +from bs4 import BeautifulSoup +import requests +import urllib.parse + +JISHO_BASE_URL = "https://jisho.org/search" + +def _meaning_tag_to_abbrev(tag: str) -> str: + if "Godan" in tag: return "G1" + if "Ichidan" in tag: return "G2" + if "Suru" in tag: return "G3" + if "Noun" in tag: return "n." + if "I-adjective" in tag: return "い-adj." + if "Na-adjective" in tag: return "な-adj." + if "No-adjective" in tag: return "の-adj." + if "Adverb" in tag: return "adv." + return tag + +def inject_kanji_meanings(entries: list) -> list: + entries_with_meanings = [] + + for index, entry in enumerate(entries): + kanji = entry.kanji + + if len(entry.meanings) > 0: + print(f"Meaning data exists for #{index}: {kanji}. Skipping...") + entries_with_meanings.append(entry) + continue + + print(f"Fetching meaning data from Jisho for #{index}: {kanji}") + + resource_part = urllib.parse.quote_plus(f"{kanji}#kanji") + url = f"{JISHO_BASE_URL}/{resource_part}" + resp = requests.get(url) + resp.raise_for_status() + + soup = BeautifulSoup(resp.text, "html.parser") + container = soup.select_one("div#page_container") + + meanings = container.select(".kanji-details__main-meanings") + kanji_meaning_texts = map(lambda e: e.text.strip(), meanings) + + kanji_characters = container.select(".character") + kanji_character_texts = map(lambda e: e.text.strip(), kanji_characters) + + result_tuples = zip(kanji_character_texts, kanji_meaning_texts) + result_strs = map(lambda t: " ".join(t), result_tuples) + result = "
".join(result_strs) + + entry.meanings = result + entries_with_meanings.append(entry) + + return entries_with_meanings + +def inject_english_translation(entries: list) -> list: + entries_with_translations = [] + + for index, entry in enumerate(entries): + kanji = entry.kanji + + if len(entry.english) > 0: + print(f"English translation exists for #{index}: {kanji}. Skipping...") + entries_with_translations.append(entry) + continue + + print(f"Fetching translation from Jisho for #{index}: {kanji}") + + resource_part = urllib.parse.quote_plus(f"{kanji}") + url = f"{JISHO_BASE_URL}/{resource_part}" + resp = requests.get(url) + resp.raise_for_status() + + soup = BeautifulSoup(resp.text, "html.parser") + container = soup.select_one("div#page_container") + + meaning_wrapper = container.select(".meanings-wrapper")[0] + + meaning_tags = meaning_wrapper.select(".meaning-tags") + meaning_tags_texts = map(lambda e: e.text.strip(), meaning_tags) + meaning_tag_abbrevs = map(_meaning_tag_to_abbrev, meaning_tags_texts) + + meaning_translations = meaning_wrapper.select(".meaning-wrapper .meaning-meaning") + meaning_translations_texts = map(lambda e: e.text.strip(), meaning_translations) + + meanings_zipped = zip(meaning_tag_abbrevs, meaning_translations_texts) + + first_translation_pair = list(meanings_zipped)[0] + result = f"{first_translation_pair[0]} {first_translation_pair[1]}" + + entry.english = result + entries_with_translations.append(entry) + + return entries_with_translations diff --git a/kanji_info_fetcher/wadoku_client.py b/kanji_info_fetcher/wadoku_client.py new file mode 100644 index 0000000..550f801 --- /dev/null +++ b/kanji_info_fetcher/wadoku_client.py @@ -0,0 +1,42 @@ +from bs4 import BeautifulSoup +import requests +import urllib.parse + +WADOKU_BASE_URL = "https://www.wadoku.de/search" + +def inject_hiragana_with_accents(entries: list) -> list: + entries_with_readings = [] + + for index, entry in enumerate(entries): + kanji = entry.kanji.replace("~", "") + + if len(entry.hiragana) > 0: + print(f"Hiragana exists for #{index}: {kanji}. Skipping...") + entries_with_readings.append(entry) + continue + + # Remove nasty characters from kanji since Wadoku + # fails to find anything otherwise + print(f"Fetching hiragana data for #{index}: {kanji}") + + resource_part = urllib.parse.quote_plus(kanji) + url = f"{WADOKU_BASE_URL}/{resource_part}" + resp = requests.get(url) + resp.raise_for_status() + + soup = BeautifulSoup(resp.text, "html.parser") + container = soup.select_one("section#content") + + readings = container.select("span.reading") + first_reading = str(readings[0]) + + # Remove extra characters added by Wadoku + # https://www.wadoku.de/wiki/display/WAD/Hinweise+zur+Notation + result = first_reading \ + .replace("~", "") \ + .replace("|", "") \ + .replace("・", "") + entry.hiragana = result + entries_with_readings.append(entry) + + return entries_with_readings diff --git a/pyproject.toml b/pyproject.toml index 74e45ed..508a7d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.poetry] -name = "attain-vocab-to-anki-py" +name = "kanji-info-fetcher" version = "0.1.0" description = "" authors = ["Jan Tuomi "] @@ -17,4 +17,4 @@ requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" [tool.poetry.scripts] -main = "attain_vocab_to_anki_py:main" +main = "kanji_info_fetcher:main" diff --git a/tests/kanji_info_fetcher.py b/tests/kanji_info_fetcher.py new file mode 100644 index 0000000..d50e289 --- /dev/null +++ b/tests/kanji_info_fetcher.py @@ -0,0 +1,5 @@ +from kanji_info_fetcher import __version__ + + +def test_version(): + assert __version__ == '0.1.0' diff --git a/tests/test_attain_vocab_to_anki_py.py b/tests/test_attain_vocab_to_anki_py.py deleted file mode 100644 index 3512c18..0000000 --- a/tests/test_attain_vocab_to_anki_py.py +++ /dev/null @@ -1,5 +0,0 @@ -from attain_vocab_to_anki_py import __version__ - - -def test_version(): - assert __version__ == '0.1.0' -- cgit v1.3