diff options
Diffstat (limited to 'plugins')
| -rw-r--r-- | plugins/FacebookSourcePlugin.py | 12 | ||||
| -rw-r--r-- | plugins/ScraperSourcePlugin.py | 6 |
2 files changed, 5 insertions, 13 deletions
diff --git a/plugins/FacebookSourcePlugin.py b/plugins/FacebookSourcePlugin.py index a43aee7..55abbc1 100644 --- a/plugins/FacebookSourcePlugin.py +++ b/plugins/FacebookSourcePlugin.py @@ -83,9 +83,7 @@ def fetch_page_posts(email: str, password: str, page_id: str, limit: int) -> lis if cookie_page_resp.status_code >= 400: raise Exception(cookie_page_resp.text) - cookie_page = BeautifulSoup( - cookie_page_resp.text, features=["xml", "lxml", "lxml-xml"] - ) + cookie_page = BeautifulSoup(cookie_page_resp.text, "html.parser") lsd: str = cookie_page.find("input", {"name": "lsd"})["value"] # type: ignore jazoest: str = cookie_page.find("input", {"name": "jazoest"})["value"] # type: ignore @@ -108,9 +106,7 @@ def fetch_page_posts(email: str, password: str, page_id: str, limit: int) -> lis if login_page_resp.status_code >= 400: raise Exception(login_page_resp.text) - login_page = BeautifulSoup( - login_page_resp.text, features=["xml", "lxml", "lxml-xml"] - ) + login_page = BeautifulSoup(login_page_resp.text, "html.parser") lsd: str = login_page.find("input", {"name": "lsd"})["value"] # type: ignore jazoest: str = login_page.find("input", {"name": "jazoest"})["value"] # type: ignore @@ -162,9 +158,7 @@ def fetch_page_posts(email: str, password: str, page_id: str, limit: int) -> lis if timeline_resp.status_code >= 400: raise Exception(timeline_resp.text) - timeline = BeautifulSoup( - timeline_resp.text, features=["xml", "lxml", "lxml-xml"] - ) + timeline = BeautifulSoup(timeline_resp.text, "html.parser") posts = timeline.select("section > article") for post in posts: diff --git a/plugins/ScraperSourcePlugin.py b/plugins/ScraperSourcePlugin.py index 75aa376..b51943b 100644 --- a/plugins/ScraperSourcePlugin.py +++ b/plugins/ScraperSourcePlugin.py @@ -47,9 +47,7 @@ class Plugin(PluginInterface): result_items: list[Item] = [] with requests.session() as session: page_resp = session.get(self.url, allow_redirects=True) - page_elem = BeautifulSoup( - page_resp.text, features=["xml", "lxml", "lxml-xml"] - ) + page_elem = BeautifulSoup(page_resp.text, "html.parser") post_elems = eval(self.selector_post, {"page": page_elem}) for post_elem in post_elems: @@ -67,7 +65,7 @@ class Plugin(PluginInterface): detail_page_url, allow_redirects=True ) detail_page_elem = BeautifulSoup( - detail_page_resp.text, features=["xml", "lxml", "lxml-xml"] + detail_page_resp.text, "html.parser" ) guid = ItemGUID(detail_page_url, is_perma_link=True) else: |