From a8aed014dbbfed67ef56fdda6219e7f17922f383 Mon Sep 17 00:00:00 2001
From: Jan Tuomi <jans.tuomi@gmail.com>
Date: Mon, 11 Sep 2023 09:51:05 +0300
Subject: Improvements all around

---
 plugins/ScraperSourcePlugin.py | 79 ++++++++++++++++++++++++++++++++----------
 1 file changed, 61 insertions(+), 18 deletions(-)

(limited to 'plugins/ScraperSourcePlugin.py')

diff --git a/plugins/ScraperSourcePlugin.py b/plugins/ScraperSourcePlugin.py
index ea28f97..75aa376 100644
--- a/plugins/ScraperSourcePlugin.py
+++ b/plugins/ScraperSourcePlugin.py
@@ -1,26 +1,18 @@
 from bs4 import BeautifulSoup
-import feedparser  # type: ignore
 import time
 from datetime import datetime, timezone
 from typing import Any
-
+import re
 import requests
-from app.Item import Item, ItemEnclosure
+from app.Item import Item, ItemEnclosure, ItemGUID
 from app.PluginInterface import Params, PluginInterface
 from app.utils import ItemDict, get_config, get_config_or_default
 
 
-def struct_time_to_utc_datetime(struct_time: time.struct_time) -> datetime:
-    timestamp = time.mktime(struct_time)
-    naive_datetime = datetime.fromtimestamp(timestamp)
-    aware_datetime = naive_datetime.replace(tzinfo=timezone.utc)
-    return aware_datetime
-
-
 class Plugin(PluginInterface):
     def __init__(self, id: str, params: Params) -> None:
         super().__init__(id, params)
-        self.url: str = get_config(params, "url")
+        self.url: str = get_config(params, "url").strip("/")
         self.selector_post: str = get_config(params, "selector_post")
         self.selector_title: str | None = get_config_or_default(
             params, "selector_title", None
@@ -37,16 +29,27 @@ class Plugin(PluginInterface):
         self.selector_author: str | None = get_config_or_default(
             params, "selector_author", None
         )
+        self.selector_image: str | None = get_config_or_default(
+            params, "selector_image", None
+        )
 
         print(f"[ScraperSourcePlugin#{self.id}] initialized")
 
+    def absolute_link(self, link: str) -> str:
+        if link.startswith("/"):
+            link = self.url + link
+
+        return link
+
     def process(self, source_id: str | None, items: list[Item]) -> list[Item]:
         print(f"[ScraperSourcePlugin#{self.id}] process called")
 
         result_items: list[Item] = []
         with requests.session() as session:
             page_resp = session.get(self.url, allow_redirects=True)
-            page_elem = BeautifulSoup(page_resp.text, features="lxml")
+            page_elem = BeautifulSoup(
+                page_resp.text, features=["xml", "lxml", "lxml-xml"]
+            )
             post_elems = eval(self.selector_post, {"page": page_elem})
 
             for post_elem in post_elems:
@@ -54,9 +57,8 @@ class Plugin(PluginInterface):
                     link_elem = eval(
                         self.selector_link, {"page": page_elem, "post": post_elem}
                     )[0]
-                    detail_page_url = link_elem["href"]
-                    if detail_page_url.startswith("/"):
-                        detail_page_url = self.url.strip("/") + detail_page_url
+                    detail_page_url = self.absolute_link(link_elem["href"])
+
                 else:
                     detail_page_url = None
 
@@ -65,9 +67,9 @@ class Plugin(PluginInterface):
                         detail_page_url, allow_redirects=True
                     )
                     detail_page_elem = BeautifulSoup(
-                        detail_page_resp.text, features="lxml"
+                        detail_page_resp.text, features=["xml", "lxml", "lxml-xml"]
                     )
-                    guid = detail_page_url
+                    guid = ItemGUID(detail_page_url, is_perma_link=True)
                 else:
                     detail_page_elem = None
                     guid = None
@@ -124,8 +126,49 @@ class Plugin(PluginInterface):
                 else:
                     author = None
 
+                if self.selector_image:
+                    image_elem = eval(
+                        self.selector_image,
+                        {
+                            "page": page_elem,
+                            "post": post_elem,
+                            "detail_page": detail_page_elem,
+                        },
+                    )[0]
+                    if image_elem.has_attr("src"):
+                        image_src = image_elem["src"]
+                    elif (
+                        image_elem.has_attr("style")
+                        and "background-image:" in image_elem["style"]
+                    ):
+                        match = re.match(
+                            r"background-image:\s*url\((?P<url>.*?)\)",
+                            image_elem["style"],
+                        )
+                        if match is None:
+                            raise Exception(
+                                f"[ScraperSourcePlugin#{self.id}] weird regex result when looking for background-image"
+                            )
+                        image_src = match.group("url")
+                    else:
+                        image_src = None
+                else:
+                    image_src = None
+
                 if guid is None:
-                    guid = f"aggro__{self.id}__{title}"
+                    guid = ItemGUID(f"aggro__{self.id}__{title}")
+
+                if image_src is not None:
+                    image_src = self.absolute_link(image_src)
+                    image_html = f'<br><br><img src="{image_src}">'
+                    if description:
+                        description += image_html
+                    else:
+                        description = image_html
+
+                if description:
+                    description = description.replace('src="/', f'src="{self.url}/')
+                    description = description.replace('href="/', f'href="{self.url}/')
 
                 item = Item(
                     title=f"{date} – {title}",
-- 
cgit v1.3