Improvements all around

5 files changed, 96 insertions, 32 deletions

diff --git a/plugins/DigestPlugin.py b/plugins/DigestPlugin.py
index b56af0d..f1debeb 100644
--- a/plugins/DigestPlugin.py
+++ b/plugins/DigestPlugin.py
@@ -4,7 +4,7 @@ from datetime import datetime, timedelta
 import re
 
 from tinydb import Query
-from app.Item import Item
+from app.Item import Item, ItemGUID
 from app.PluginInterface import Params, PluginInterface
 from app.DatabaseManager import database_manager
 from app.utils import dict_to_item, get_config_or_default, item_to_dict, get_config
@@ -183,7 +183,7 @@ class Plugin(PluginInterface):
                 category=None,
                 comments=None,
                 enclosures=[],
-                guid=f"aggro__{self.id}__{digest_pub_date.isoformat()}",
+                guid=ItemGUID(f"aggro__{self.id}__{digest_pub_date.isoformat()}"),
             )
 
             digest_items.append(digest_item)
diff --git a/plugins/FacebookSourcePlugin.py b/plugins/FacebookSourcePlugin.py
index 24784e3..a43aee7 100644
--- a/plugins/FacebookSourcePlugin.py
+++ b/plugins/FacebookSourcePlugin.py
@@ -5,7 +5,7 @@ import re
 import urllib.parse
 from datetime import datetime, timedelta
 from bs4 import BeautifulSoup, Tag
-from app.Item import Item
+from app.Item import Item, ItemGUID
 from app.PluginInterface import Params, PluginInterface
 from app.utils import get_config, get_config_or_default
 
@@ -83,7 +83,9 @@ def fetch_page_posts(email: str, password: str, page_id: str, limit: int) -> lis
         if cookie_page_resp.status_code >= 400:
             raise Exception(cookie_page_resp.text)
 
-        cookie_page = BeautifulSoup(cookie_page_resp.text, features=["xml", "lxml"])
+        cookie_page = BeautifulSoup(
+            cookie_page_resp.text, features=["xml", "lxml", "lxml-xml"]
+        )
         lsd: str = cookie_page.find("input", {"name": "lsd"})["value"]  # type: ignore
         jazoest: str = cookie_page.find("input", {"name": "jazoest"})["value"]  # type: ignore
 
@@ -106,7 +108,9 @@ def fetch_page_posts(email: str, password: str, page_id: str, limit: int) -> lis
         if login_page_resp.status_code >= 400:
             raise Exception(login_page_resp.text)
 
-        login_page = BeautifulSoup(login_page_resp.text, features=["xml", "lxml"])
+        login_page = BeautifulSoup(
+            login_page_resp.text, features=["xml", "lxml", "lxml-xml"]
+        )
 
         lsd: str = login_page.find("input", {"name": "lsd"})["value"]  # type: ignore
         jazoest: str = login_page.find("input", {"name": "jazoest"})["value"]  # type: ignore
@@ -158,13 +162,22 @@ def fetch_page_posts(email: str, password: str, page_id: str, limit: int) -> lis
             if timeline_resp.status_code >= 400:
                 raise Exception(timeline_resp.text)
 
-            timeline = BeautifulSoup(timeline_resp.text, features=["xml", "lxml"])
+            timeline = BeautifulSoup(
+                timeline_resp.text, features=["xml", "lxml", "lxml-xml"]
+            )
             posts = timeline.select("section > article")
 
             for post in posts:
-                link_tag: Tag | None = post.find("a", string="Full Story")  # type: ignore
                 time_tag: Tag = post.find("abbr")  # type: ignore
-                link = f"{base_url}{link_tag['href']}" if link_tag is not None else None
+
+                link_tag: Tag | None = post.find("a", string="Full Story")  # type: ignore
+                if link_tag is not None:
+                    link = f"{base_url}{link_tag['href']}"
+                    # drop tracking parameters that change at a whim
+                    link = link.split("&eav")[0]
+                else:
+                    link = None
+
                 pub_date_str: str = time_tag.get_text()
                 pub_date = parse_custom_date(pub_date_str)
                 story_body_container = str(post.find("div"))
@@ -175,7 +188,10 @@ def fetch_page_posts(email: str, password: str, page_id: str, limit: int) -> lis
                     title=title,
                     description=story_body_container,
                     link=link,
-                    guid=link if link is not None else f"aggro__facebook__{title}",
+                    guid=ItemGUID(
+                        link if link is not None else f"aggro__facebook__{title}",
+                        is_perma_link=link is not None,
+                    ),
                     author=page_id,
                     category=None,
                     comments=None,
diff --git a/plugins/FeedSinkPlugin.py b/plugins/FeedSinkPlugin.py
index 623bb49..f126b6d 100644
--- a/plugins/FeedSinkPlugin.py
+++ b/plugins/FeedSinkPlugin.py
@@ -56,8 +56,12 @@ class Plugin(PluginInterface):
                 item_author = ET.SubElement(item_elem, "author")
                 item_author.text = item.author
 
-            item_guid = ET.SubElement(item_elem, "guid")
-            item_guid.text = item.guid
+            item_guid = ET.SubElement(
+                item_elem,
+                "guid",
+                {"isPermaLink": "true" if item.guid.is_perma_link else "false"},
+            )
+            item_guid.text = item.guid.value
 
             for enc in item.enclosures:
                 ET.SubElement(
diff --git a/plugins/FeedSourcePlugin.py b/plugins/FeedSourcePlugin.py
index fd27081..9eba538 100644
--- a/plugins/FeedSourcePlugin.py
+++ b/plugins/FeedSourcePlugin.py
@@ -2,7 +2,7 @@ import feedparser  # type: ignore
 import time
 from datetime import datetime, timezone
 from typing import Any
-from app.Item import Item, ItemEnclosure
+from app.Item import Item, ItemEnclosure, ItemGUID
 from app.PluginInterface import Params, PluginInterface
 from app.utils import ItemDict, get_config
 
@@ -56,17 +56,18 @@ class Plugin(PluginInterface):
                 else datetime_1970
             )
 
+            link: str = d["link"]  # type: ignore
             result_items.append(
                 Item(
                     title=d.get("title", None),  # type: ignore
-                    link=d["link"],  # type: ignore
+                    link=link,
                     description=d.get("description", None),  # type: ignore
                     author=d.get("author", None),  # type: ignore
                     pub_date=published_datetime,
                     category=d.get("category", None),  # type: ignore
                     comments=d.get("comments"),  # type: ignore
                     enclosures=item_enclosures,
-                    guid=d["link"],  # type: ignore
+                    guid=ItemGUID(link, is_perma_link=True),
                 )
             )
 
diff --git a/plugins/ScraperSourcePlugin.py b/plugins/ScraperSourcePlugin.py
index ea28f97..75aa376 100644
--- a/plugins/ScraperSourcePlugin.py
+++ b/plugins/ScraperSourcePlugin.py
@@ -1,26 +1,18 @@
 from bs4 import BeautifulSoup
-import feedparser  # type: ignore
 import time
 from datetime import datetime, timezone
 from typing import Any
-
+import re
 import requests
-from app.Item import Item, ItemEnclosure
+from app.Item import Item, ItemEnclosure, ItemGUID
 from app.PluginInterface import Params, PluginInterface
 from app.utils import ItemDict, get_config, get_config_or_default
 
 
-def struct_time_to_utc_datetime(struct_time: time.struct_time) -> datetime:
-    timestamp = time.mktime(struct_time)
-    naive_datetime = datetime.fromtimestamp(timestamp)
-    aware_datetime = naive_datetime.replace(tzinfo=timezone.utc)
-    return aware_datetime
-
-
 class Plugin(PluginInterface):
     def __init__(self, id: str, params: Params) -> None:
         super().__init__(id, params)
-        self.url: str = get_config(params, "url")
+        self.url: str = get_config(params, "url").strip("/")
         self.selector_post: str = get_config(params, "selector_post")
         self.selector_title: str | None = get_config_or_default(
             params, "selector_title", None
@@ -37,16 +29,27 @@ class Plugin(PluginInterface):
         self.selector_author: str | None = get_config_or_default(
             params, "selector_author", None
         )
+        self.selector_image: str | None = get_config_or_default(
+            params, "selector_image", None
+        )
 
         print(f"[ScraperSourcePlugin#{self.id}] initialized")
 
+    def absolute_link(self, link: str) -> str:
+        if link.startswith("/"):
+            link = self.url + link
+
+        return link
+
     def process(self, source_id: str | None, items: list[Item]) -> list[Item]:
         print(f"[ScraperSourcePlugin#{self.id}] process called")
 
         result_items: list[Item] = []
         with requests.session() as session:
             page_resp = session.get(self.url, allow_redirects=True)
-            page_elem = BeautifulSoup(page_resp.text, features="lxml")
+            page_elem = BeautifulSoup(
+                page_resp.text, features=["xml", "lxml", "lxml-xml"]
+            )
             post_elems = eval(self.selector_post, {"page": page_elem})
 
             for post_elem in post_elems:
@@ -54,9 +57,8 @@ class Plugin(PluginInterface):
                     link_elem = eval(
                         self.selector_link, {"page": page_elem, "post": post_elem}
                     )[0]
-                    detail_page_url = link_elem["href"]
-                    if detail_page_url.startswith("/"):
-                        detail_page_url = self.url.strip("/") + detail_page_url
+                    detail_page_url = self.absolute_link(link_elem["href"])
+
                 else:
                     detail_page_url = None
 
@@ -65,9 +67,9 @@ class Plugin(PluginInterface):
                         detail_page_url, allow_redirects=True
                     )
                     detail_page_elem = BeautifulSoup(
-                        detail_page_resp.text, features="lxml"
+                        detail_page_resp.text, features=["xml", "lxml", "lxml-xml"]
                     )
-                    guid = detail_page_url
+                    guid = ItemGUID(detail_page_url, is_perma_link=True)
                 else:
                     detail_page_elem = None
                     guid = None
@@ -124,8 +126,49 @@ class Plugin(PluginInterface):
                 else:
                     author = None
 
+                if self.selector_image:
+                    image_elem = eval(
+                        self.selector_image,
+                        {
+                            "page": page_elem,
+                            "post": post_elem,
+                            "detail_page": detail_page_elem,
+                        },
+                    )[0]
+                    if image_elem.has_attr("src"):
+                        image_src = image_elem["src"]
+                    elif (
+                        image_elem.has_attr("style")
+                        and "background-image:" in image_elem["style"]
+                    ):
+                        match = re.match(
+                            r"background-image:\s*url\((?P<url>.*?)\)",
+                            image_elem["style"],
+                        )
+                        if match is None:
+                            raise Exception(
+                                f"[ScraperSourcePlugin#{self.id}] weird regex result when looking for background-image"
+                            )
+                        image_src = match.group("url")
+                    else:
+                        image_src = None
+                else:
+                    image_src = None
+
                 if guid is None:
-                    guid = f"aggro__{self.id}__{title}"
+                    guid = ItemGUID(f"aggro__{self.id}__{title}")
+
+                if image_src is not None:
+                    image_src = self.absolute_link(image_src)
+                    image_html = f'<br><br><img src="{image_src}">'
+                    if description:
+                        description += image_html
+                    else:
+                        description = image_html
+
+                if description:
+                    description = description.replace('src="/', f'src="{self.url}/')
+                    description = description.replace('href="/', f'href="{self.url}/')
 
                 item = Item(
                     title=f"{date} – {title}",