diff options
| -rw-r--r-- | plugins/JsonApiSourcePlugin.py | 146 | ||||
| -rw-r--r-- | plugins/ScraperSourcePlugin.py | 9 |
2 files changed, 154 insertions, 1 deletions
diff --git a/plugins/JsonApiSourcePlugin.py b/plugins/JsonApiSourcePlugin.py new file mode 100644 index 0000000..48209f2 --- /dev/null +++ b/plugins/JsonApiSourcePlugin.py @@ -0,0 +1,146 @@ +import json +from typing import Any +import requests +from app.Item import Item, ItemGUID, ItemMediaContent +from app.PluginInterface import Params, PluginInterface +from app.utils import ItemDict, get_config, get_config_or_default + + +class Plugin(PluginInterface): + def __init__(self, id: str, params: Params) -> None: + super().__init__(id, params) + self.url: str = get_config(params, "url").strip("/") + self.selector_post: str = get_config(params, "selector_post") + self.selector_title: str | None = get_config_or_default( + params, "selector_title", None + ) + self.selector_date: str | None = get_config_or_default( + params, "selector_date", None + ) + self.selector_link: str | None = get_config_or_default( + params, "selector_link", None + ) + self.selector_description: str | None = get_config_or_default( + params, "selector_description", None + ) + self.selector_author: str | None = get_config_or_default( + params, "selector_author", None + ) + self.selector_image: str | None = get_config_or_default( + params, "selector_image", None + ) + self.show_image_in_description: bool = get_config_or_default( + params, "show_image_in_description", True + ) + + print(f"[JsonApiSourcePlugin#{self.id}] initialized") + + def absolute_link(self, link: str) -> str: + if link.startswith("/") or link.startswith("#"): + link = self.url + link + + return link + + def process(self, source_id: str | None, items: list[Item]) -> list[Item]: + print(f"[JsonApiSourcePlugin#{self.id}] process called") + + result_items: list[Item] = [] + with requests.session() as session: + api_resp = session.get(self.url, allow_redirects=True) + data = json.loads(api_resp.text) + post_items = eval(self.selector_post, {"data": data}) + + for post in post_items: + if self.selector_link: + link = eval(self.selector_link, {"post": post, "data": data})[ + 0 + ].strip() + guid = ItemGUID(link, is_perma_link=True) + else: + link = None + guid = None + + if self.selector_title: + title = eval(self.selector_title, {"post": post, "data": data})[ + 0 + ].strip() + else: + title = None + + if self.selector_description: + description = eval( + self.selector_description, {"post": post, "data": data} + )[0].strip() + else: + description = None + + if self.selector_date: + date = eval(self.selector_date, {"post": post, "data": data})[ + 0 + ].strip() + else: + date = None + + if self.selector_author: + author = eval(self.selector_author, {"post": post, "data": data})[ + 0 + ].strip() + else: + author = None + + if self.selector_image: + image_src = eval(self.selector_image, {"post": post, "data": data})[ + 0 + ].strip() + else: + image_src = None + + if guid is None: + if title: + guid = ItemGUID(f"aggro__{self.id}__{title}") + elif description: + guid = ItemGUID(f"aggro__{self.id}__{description}") + else: + raise Exception( + f"[ScraperSourcePlugin#{self.id}] both title and description are None" + ) + + if image_src is not None: + image_src = self.absolute_link(image_src) + image_html = f'<img src="{image_src}"><br><br>' + if description and self.show_image_in_description: + description = image_html + description + elif description is None: + description = image_html + + media_content = ItemMediaContent( + url=image_src, + medium="image", + ) + else: + media_content = None + + if description: + description = description.replace('src="/', f'src="{self.url}/') + description = description.replace('src="#', f'src="{self.url}#') + description = description.replace('href="/', f'href="{self.url}/') + description = description.replace('href="#', f'href="{self.url}#') + + item = Item( + title=f"{date} – {title}", + link=link if link else self.url, + description=description, + pub_date=None, # TODO pub_date parsing + author=author, + category=None, + comments=None, + enclosures=[], + guid=guid, + media_content=media_content, + ) + result_items.append(item) + + print( + f"[JsonApiSourcePlugin#{self.id}] process returns items, n={len(result_items)}" + ) + return result_items diff --git a/plugins/ScraperSourcePlugin.py b/plugins/ScraperSourcePlugin.py index 0013695..3ee78b7 100644 --- a/plugins/ScraperSourcePlugin.py +++ b/plugins/ScraperSourcePlugin.py @@ -159,7 +159,14 @@ class Plugin(PluginInterface): image_src = None if guid is None: - guid = ItemGUID(f"aggro__{self.id}__{title}") + if title: + guid = ItemGUID(f"aggro__{self.id}__{title}") + elif description: + guid = ItemGUID(f"aggro__{self.id}__{description}") + else: + raise Exception( + f"[ScraperSourcePlugin#{self.id}] both title and description are None" + ) if image_src is not None: image_src = self.absolute_link(image_src) |