From 74ee6b327de8db0cf3ea8bd5a56040eb3fb02f08 Mon Sep 17 00:00:00 2001 From: Jan Tuomi Date: Sun, 10 Sep 2023 20:40:49 +0300 Subject: Small fixes, add Docker --- plugins/FacebookSourcePlugin.py | 6 +++--- plugins/ScraperSourcePlugin.py | 14 ++++++++------ 2 files changed, 11 insertions(+), 9 deletions(-) (limited to 'plugins') diff --git a/plugins/FacebookSourcePlugin.py b/plugins/FacebookSourcePlugin.py index e7ce1ab..24784e3 100644 --- a/plugins/FacebookSourcePlugin.py +++ b/plugins/FacebookSourcePlugin.py @@ -83,7 +83,7 @@ def fetch_page_posts(email: str, password: str, page_id: str, limit: int) -> lis if cookie_page_resp.status_code >= 400: raise Exception(cookie_page_resp.text) - cookie_page = BeautifulSoup(cookie_page_resp.text, features="xml") + cookie_page = BeautifulSoup(cookie_page_resp.text, features=["xml", "lxml"]) lsd: str = cookie_page.find("input", {"name": "lsd"})["value"] # type: ignore jazoest: str = cookie_page.find("input", {"name": "jazoest"})["value"] # type: ignore @@ -106,7 +106,7 @@ def fetch_page_posts(email: str, password: str, page_id: str, limit: int) -> lis if login_page_resp.status_code >= 400: raise Exception(login_page_resp.text) - login_page = BeautifulSoup(login_page_resp.text, features="lxml") + login_page = BeautifulSoup(login_page_resp.text, features=["xml", "lxml"]) lsd: str = login_page.find("input", {"name": "lsd"})["value"] # type: ignore jazoest: str = login_page.find("input", {"name": "jazoest"})["value"] # type: ignore @@ -158,7 +158,7 @@ def fetch_page_posts(email: str, password: str, page_id: str, limit: int) -> lis if timeline_resp.status_code >= 400: raise Exception(timeline_resp.text) - timeline = BeautifulSoup(timeline_resp.text, features="lxml") + timeline = BeautifulSoup(timeline_resp.text, features=["xml", "lxml"]) posts = timeline.select("section > article") for post in posts: diff --git a/plugins/ScraperSourcePlugin.py b/plugins/ScraperSourcePlugin.py index caa4f75..ea28f97 100644 --- a/plugins/ScraperSourcePlugin.py +++ b/plugins/ScraperSourcePlugin.py @@ -46,7 +46,7 @@ class Plugin(PluginInterface): result_items: list[Item] = [] with requests.session() as session: page_resp = session.get(self.url, allow_redirects=True) - page_elem = BeautifulSoup(page_resp.text) + page_elem = BeautifulSoup(page_resp.text, features="lxml") post_elems = eval(self.selector_post, {"page": page_elem}) for post_elem in post_elems: @@ -64,7 +64,9 @@ class Plugin(PluginInterface): detail_page_resp = session.get( detail_page_url, allow_redirects=True ) - detail_page_elem = BeautifulSoup(detail_page_resp.text) + detail_page_elem = BeautifulSoup( + detail_page_resp.text, features="lxml" + ) guid = detail_page_url else: detail_page_elem = None @@ -79,7 +81,7 @@ class Plugin(PluginInterface): "detail_page": detail_page_elem, }, )[0] - title = title_elem.get_text() + title = title_elem.get_text().strip() else: title = None @@ -92,7 +94,7 @@ class Plugin(PluginInterface): "detail_page": detail_page_elem, }, )[0] - description = str(description_elem) + description = str(description_elem).strip() else: description = None @@ -105,7 +107,7 @@ class Plugin(PluginInterface): "detail_page": detail_page_elem, }, )[0] - date = date_elem.get_text() + date = date_elem.get_text().strip() else: date = None @@ -118,7 +120,7 @@ class Plugin(PluginInterface): "detail_page": detail_page_elem, }, )[0] - author = author_elem.get_text() + author = author_elem.get_text().strip() else: author = None -- cgit v1.3