diff options
| -rw-r--r-- | Aggrofile | 23 | ||||
| -rw-r--r-- | aggro.py | 8 | ||||
| -rw-r--r-- | plugins/ScraperSourcePlugin.py | 144 |
3 files changed, 169 insertions, 6 deletions
@@ -3,13 +3,29 @@ "plugins": { "plutonium74_fb": { "plugin": "FacebookSourcePlugin", - "schedule_expr": "schedule.every(30).seconds", + "schedule_expr": "schedule.every(24).hours", "login_email": "${AGGRO_FB_LOGIN_EMAIL}", "login_password": "${AGGRO_FB_LOGIN_PASSWORD}", - "page_id": "Batushkaband", + "page_id": "Plutonium74", "limit": 10, "__comment": "https://www.facebook.com/Plutonium74" }, + "aaniwalli_scraper": { + "plugin": "ScraperSourcePlugin", + "schedule_expr": "schedule.every(10).seconds", + "url": "https://www.ääniwalli.fi/", + "selector_post": "page.select('article')", + "selector_title": "post.select('h1')", + "selector_date": "post.select('.date')", + "selector_link": "post.select('a.event-link')", + "selector_description": "detail_page.select('.event-page')" + }, + "aaniwalli_feed": { + "plugin": "FeedSinkPlugin", + "feed_id": "aaniwalli", + "feed_title": "Ääniwalli", + "feed_description": "Ääniwalli events" + }, "plutonium74_fb_feed": { "plugin": "FeedSinkPlugin", "feed_id": "plutonium74_fb", @@ -79,6 +95,9 @@ ], "plutonium74_fb": [ "plutonium74_fb_feed" + ], + "aaniwalli_scraper": [ + "aaniwalli_feed" ] } }
\ No newline at end of file @@ -79,15 +79,15 @@ if __name__ == "__main__": memory_state.running = True + server_thread = Thread(target=run_server_thread, args=[aggro_config]) + server_thread.daemon = True + server_thread.start() + manager.initial_run_scheduled_plugins() plugin_thread = Thread(target=run_plugin_thread, args=[manager, aggro_config]) plugin_thread.start() - server_thread = Thread(target=run_server_thread, args=[aggro_config]) - server_thread.daemon = True - server_thread.start() - try: while memory_state.running: time.sleep(0.1) diff --git a/plugins/ScraperSourcePlugin.py b/plugins/ScraperSourcePlugin.py new file mode 100644 index 0000000..caa4f75 --- /dev/null +++ b/plugins/ScraperSourcePlugin.py @@ -0,0 +1,144 @@ +from bs4 import BeautifulSoup +import feedparser # type: ignore +import time +from datetime import datetime, timezone +from typing import Any + +import requests +from app.Item import Item, ItemEnclosure +from app.PluginInterface import Params, PluginInterface +from app.utils import ItemDict, get_config, get_config_or_default + + +def struct_time_to_utc_datetime(struct_time: time.struct_time) -> datetime: + timestamp = time.mktime(struct_time) + naive_datetime = datetime.fromtimestamp(timestamp) + aware_datetime = naive_datetime.replace(tzinfo=timezone.utc) + return aware_datetime + + +class Plugin(PluginInterface): + def __init__(self, id: str, params: Params) -> None: + super().__init__(id, params) + self.url: str = get_config(params, "url") + self.selector_post: str = get_config(params, "selector_post") + self.selector_title: str | None = get_config_or_default( + params, "selector_title", None + ) + self.selector_date: str | None = get_config_or_default( + params, "selector_date", None + ) + self.selector_link: str | None = get_config_or_default( + params, "selector_link", None + ) + self.selector_description: str | None = get_config_or_default( + params, "selector_description", None + ) + self.selector_author: str | None = get_config_or_default( + params, "selector_author", None + ) + + print(f"[ScraperSourcePlugin#{self.id}] initialized") + + def process(self, source_id: str | None, items: list[Item]) -> list[Item]: + print(f"[ScraperSourcePlugin#{self.id}] process called") + + result_items: list[Item] = [] + with requests.session() as session: + page_resp = session.get(self.url, allow_redirects=True) + page_elem = BeautifulSoup(page_resp.text) + post_elems = eval(self.selector_post, {"page": page_elem}) + + for post_elem in post_elems: + if self.selector_link: + link_elem = eval( + self.selector_link, {"page": page_elem, "post": post_elem} + )[0] + detail_page_url = link_elem["href"] + if detail_page_url.startswith("/"): + detail_page_url = self.url.strip("/") + detail_page_url + else: + detail_page_url = None + + if detail_page_url: + detail_page_resp = session.get( + detail_page_url, allow_redirects=True + ) + detail_page_elem = BeautifulSoup(detail_page_resp.text) + guid = detail_page_url + else: + detail_page_elem = None + guid = None + + if self.selector_title: + title_elem = eval( + self.selector_title, + { + "page": page_elem, + "post": post_elem, + "detail_page": detail_page_elem, + }, + )[0] + title = title_elem.get_text() + else: + title = None + + if self.selector_description: + description_elem = eval( + self.selector_description, + { + "page": page_elem, + "post": post_elem, + "detail_page": detail_page_elem, + }, + )[0] + description = str(description_elem) + else: + description = None + + if self.selector_date: + date_elem = eval( + self.selector_date, + { + "page": page_elem, + "post": post_elem, + "detail_page": detail_page_elem, + }, + )[0] + date = date_elem.get_text() + else: + date = None + + if self.selector_author: + author_elem = eval( + self.selector_author, + { + "page": page_elem, + "post": post_elem, + "detail_page": detail_page_elem, + }, + )[0] + author = author_elem.get_text() + else: + author = None + + if guid is None: + guid = f"aggro__{self.id}__{title}" + + item = Item( + title=f"{date} – {title}", + link=detail_page_url, + description=description, + pub_date=datetime.now(), + author=author, + category=None, + comments=None, + enclosures=[], + guid=guid, + ) + result_items.append(item) + + print( + f"[ScraperSourcePlugin#{self.id}] process returns items, n={len(result_items)}" + ) + return result_items |