From a8aed014dbbfed67ef56fdda6219e7f17922f383 Mon Sep 17 00:00:00 2001 From: Jan Tuomi Date: Mon, 11 Sep 2023 09:51:05 +0300 Subject: Improvements all around --- Aggrofile | 3 +- app/Item.py | 10 +++++- plugins/DigestPlugin.py | 4 +-- plugins/FacebookSourcePlugin.py | 30 ++++++++++++---- plugins/FeedSinkPlugin.py | 8 +++-- plugins/FeedSourcePlugin.py | 7 ++-- plugins/ScraperSourcePlugin.py | 79 +++++++++++++++++++++++++++++++---------- 7 files changed, 107 insertions(+), 34 deletions(-) diff --git a/Aggrofile b/Aggrofile index 7d8eec7..623b8ef 100644 --- a/Aggrofile +++ b/Aggrofile @@ -20,7 +20,8 @@ "selector_title": "post.select('h1')", "selector_date": "post.select('.date')", "selector_link": "post.select('a.event-link')", - "selector_description": "detail_page.select('.event-page')" + "selector_description": "detail_page.select('.event-page')", + "selector_image": "post.select('.event-link > .image')" }, "aaniwalli_feed": { "plugin": "FeedSinkPlugin", diff --git a/app/Item.py b/app/Item.py index 7376301..a7af6ce 100644 --- a/app/Item.py +++ b/app/Item.py @@ -11,6 +11,14 @@ class ItemEnclosure: url: str +@dataclass +class ItemGUID: + """RSS Item GUID object""" + + value: str + is_perma_link: bool = False + + @dataclass class Item: """RSS Item""" @@ -23,4 +31,4 @@ class Item: category: str | None comments: str | None enclosures: list[ItemEnclosure] - guid: str + guid: ItemGUID diff --git a/plugins/DigestPlugin.py b/plugins/DigestPlugin.py index b56af0d..f1debeb 100644 --- a/plugins/DigestPlugin.py +++ b/plugins/DigestPlugin.py @@ -4,7 +4,7 @@ from datetime import datetime, timedelta import re from tinydb import Query -from app.Item import Item +from app.Item import Item, ItemGUID from app.PluginInterface import Params, PluginInterface from app.DatabaseManager import database_manager from app.utils import dict_to_item, get_config_or_default, item_to_dict, get_config @@ -183,7 +183,7 @@ class Plugin(PluginInterface): category=None, comments=None, enclosures=[], - guid=f"aggro__{self.id}__{digest_pub_date.isoformat()}", + guid=ItemGUID(f"aggro__{self.id}__{digest_pub_date.isoformat()}"), ) digest_items.append(digest_item) diff --git a/plugins/FacebookSourcePlugin.py b/plugins/FacebookSourcePlugin.py index 24784e3..a43aee7 100644 --- a/plugins/FacebookSourcePlugin.py +++ b/plugins/FacebookSourcePlugin.py @@ -5,7 +5,7 @@ import re import urllib.parse from datetime import datetime, timedelta from bs4 import BeautifulSoup, Tag -from app.Item import Item +from app.Item import Item, ItemGUID from app.PluginInterface import Params, PluginInterface from app.utils import get_config, get_config_or_default @@ -83,7 +83,9 @@ def fetch_page_posts(email: str, password: str, page_id: str, limit: int) -> lis if cookie_page_resp.status_code >= 400: raise Exception(cookie_page_resp.text) - cookie_page = BeautifulSoup(cookie_page_resp.text, features=["xml", "lxml"]) + cookie_page = BeautifulSoup( + cookie_page_resp.text, features=["xml", "lxml", "lxml-xml"] + ) lsd: str = cookie_page.find("input", {"name": "lsd"})["value"] # type: ignore jazoest: str = cookie_page.find("input", {"name": "jazoest"})["value"] # type: ignore @@ -106,7 +108,9 @@ def fetch_page_posts(email: str, password: str, page_id: str, limit: int) -> lis if login_page_resp.status_code >= 400: raise Exception(login_page_resp.text) - login_page = BeautifulSoup(login_page_resp.text, features=["xml", "lxml"]) + login_page = BeautifulSoup( + login_page_resp.text, features=["xml", "lxml", "lxml-xml"] + ) lsd: str = login_page.find("input", {"name": "lsd"})["value"] # type: ignore jazoest: str = login_page.find("input", {"name": "jazoest"})["value"] # type: ignore @@ -158,13 +162,22 @@ def fetch_page_posts(email: str, password: str, page_id: str, limit: int) -> lis if timeline_resp.status_code >= 400: raise Exception(timeline_resp.text) - timeline = BeautifulSoup(timeline_resp.text, features=["xml", "lxml"]) + timeline = BeautifulSoup( + timeline_resp.text, features=["xml", "lxml", "lxml-xml"] + ) posts = timeline.select("section > article") for post in posts: - link_tag: Tag | None = post.find("a", string="Full Story") # type: ignore time_tag: Tag = post.find("abbr") # type: ignore - link = f"{base_url}{link_tag['href']}" if link_tag is not None else None + + link_tag: Tag | None = post.find("a", string="Full Story") # type: ignore + if link_tag is not None: + link = f"{base_url}{link_tag['href']}" + # drop tracking parameters that change at a whim + link = link.split("&eav")[0] + else: + link = None + pub_date_str: str = time_tag.get_text() pub_date = parse_custom_date(pub_date_str) story_body_container = str(post.find("div")) @@ -175,7 +188,10 @@ def fetch_page_posts(email: str, password: str, page_id: str, limit: int) -> lis title=title, description=story_body_container, link=link, - guid=link if link is not None else f"aggro__facebook__{title}", + guid=ItemGUID( + link if link is not None else f"aggro__facebook__{title}", + is_perma_link=link is not None, + ), author=page_id, category=None, comments=None, diff --git a/plugins/FeedSinkPlugin.py b/plugins/FeedSinkPlugin.py index 623bb49..f126b6d 100644 --- a/plugins/FeedSinkPlugin.py +++ b/plugins/FeedSinkPlugin.py @@ -56,8 +56,12 @@ class Plugin(PluginInterface): item_author = ET.SubElement(item_elem, "author") item_author.text = item.author - item_guid = ET.SubElement(item_elem, "guid") - item_guid.text = item.guid + item_guid = ET.SubElement( + item_elem, + "guid", + {"isPermaLink": "true" if item.guid.is_perma_link else "false"}, + ) + item_guid.text = item.guid.value for enc in item.enclosures: ET.SubElement( diff --git a/plugins/FeedSourcePlugin.py b/plugins/FeedSourcePlugin.py index fd27081..9eba538 100644 --- a/plugins/FeedSourcePlugin.py +++ b/plugins/FeedSourcePlugin.py @@ -2,7 +2,7 @@ import feedparser # type: ignore import time from datetime import datetime, timezone from typing import Any -from app.Item import Item, ItemEnclosure +from app.Item import Item, ItemEnclosure, ItemGUID from app.PluginInterface import Params, PluginInterface from app.utils import ItemDict, get_config @@ -56,17 +56,18 @@ class Plugin(PluginInterface): else datetime_1970 ) + link: str = d["link"] # type: ignore result_items.append( Item( title=d.get("title", None), # type: ignore - link=d["link"], # type: ignore + link=link, description=d.get("description", None), # type: ignore author=d.get("author", None), # type: ignore pub_date=published_datetime, category=d.get("category", None), # type: ignore comments=d.get("comments"), # type: ignore enclosures=item_enclosures, - guid=d["link"], # type: ignore + guid=ItemGUID(link, is_perma_link=True), ) ) diff --git a/plugins/ScraperSourcePlugin.py b/plugins/ScraperSourcePlugin.py index ea28f97..75aa376 100644 --- a/plugins/ScraperSourcePlugin.py +++ b/plugins/ScraperSourcePlugin.py @@ -1,26 +1,18 @@ from bs4 import BeautifulSoup -import feedparser # type: ignore import time from datetime import datetime, timezone from typing import Any - +import re import requests -from app.Item import Item, ItemEnclosure +from app.Item import Item, ItemEnclosure, ItemGUID from app.PluginInterface import Params, PluginInterface from app.utils import ItemDict, get_config, get_config_or_default -def struct_time_to_utc_datetime(struct_time: time.struct_time) -> datetime: - timestamp = time.mktime(struct_time) - naive_datetime = datetime.fromtimestamp(timestamp) - aware_datetime = naive_datetime.replace(tzinfo=timezone.utc) - return aware_datetime - - class Plugin(PluginInterface): def __init__(self, id: str, params: Params) -> None: super().__init__(id, params) - self.url: str = get_config(params, "url") + self.url: str = get_config(params, "url").strip("/") self.selector_post: str = get_config(params, "selector_post") self.selector_title: str | None = get_config_or_default( params, "selector_title", None @@ -37,16 +29,27 @@ class Plugin(PluginInterface): self.selector_author: str | None = get_config_or_default( params, "selector_author", None ) + self.selector_image: str | None = get_config_or_default( + params, "selector_image", None + ) print(f"[ScraperSourcePlugin#{self.id}] initialized") + def absolute_link(self, link: str) -> str: + if link.startswith("/"): + link = self.url + link + + return link + def process(self, source_id: str | None, items: list[Item]) -> list[Item]: print(f"[ScraperSourcePlugin#{self.id}] process called") result_items: list[Item] = [] with requests.session() as session: page_resp = session.get(self.url, allow_redirects=True) - page_elem = BeautifulSoup(page_resp.text, features="lxml") + page_elem = BeautifulSoup( + page_resp.text, features=["xml", "lxml", "lxml-xml"] + ) post_elems = eval(self.selector_post, {"page": page_elem}) for post_elem in post_elems: @@ -54,9 +57,8 @@ class Plugin(PluginInterface): link_elem = eval( self.selector_link, {"page": page_elem, "post": post_elem} )[0] - detail_page_url = link_elem["href"] - if detail_page_url.startswith("/"): - detail_page_url = self.url.strip("/") + detail_page_url + detail_page_url = self.absolute_link(link_elem["href"]) + else: detail_page_url = None @@ -65,9 +67,9 @@ class Plugin(PluginInterface): detail_page_url, allow_redirects=True ) detail_page_elem = BeautifulSoup( - detail_page_resp.text, features="lxml" + detail_page_resp.text, features=["xml", "lxml", "lxml-xml"] ) - guid = detail_page_url + guid = ItemGUID(detail_page_url, is_perma_link=True) else: detail_page_elem = None guid = None @@ -124,8 +126,49 @@ class Plugin(PluginInterface): else: author = None + if self.selector_image: + image_elem = eval( + self.selector_image, + { + "page": page_elem, + "post": post_elem, + "detail_page": detail_page_elem, + }, + )[0] + if image_elem.has_attr("src"): + image_src = image_elem["src"] + elif ( + image_elem.has_attr("style") + and "background-image:" in image_elem["style"] + ): + match = re.match( + r"background-image:\s*url\((?P.*?)\)", + image_elem["style"], + ) + if match is None: + raise Exception( + f"[ScraperSourcePlugin#{self.id}] weird regex result when looking for background-image" + ) + image_src = match.group("url") + else: + image_src = None + else: + image_src = None + if guid is None: - guid = f"aggro__{self.id}__{title}" + guid = ItemGUID(f"aggro__{self.id}__{title}") + + if image_src is not None: + image_src = self.absolute_link(image_src) + image_html = f'

' + if description: + description += image_html + else: + description = image_html + + if description: + description = description.replace('src="/', f'src="{self.url}/') + description = description.replace('href="/', f'href="{self.url}/') item = Item( title=f"{date} – {title}", -- cgit v1.3