Implement RSS spec stuff

8 files changed, 114 insertions, 32 deletions

diff --git a/app/ConcatPlugin.py b/app/ConcatPlugin.py
index 28e3773..43d7db2 100644
--- a/app/ConcatPlugin.py
+++ b/app/ConcatPlugin.py
@@ -1,17 +1,24 @@
 from typing import Any
+import time
 
 from tinydb import Query
 from app.Item import Item
-from app.PluginInterface import PluginInterface
+from app.PluginInterface import Params, PluginInterface
 from app.DatabaseManager import database_manager
-from app.utils import dict_to_item, item_to_dict
+from app.utils import ItemDict, dict_to_item, item_to_dict
 
 
 class Plugin(PluginInterface):
-    def __init__(self, id: str, params: dict[str, Any]) -> None:
+    def __init__(self, id: str, params: Params) -> None:
         super().__init__(id, params)
         print(f"[ConcatPlugin#{self.id}] initialized")
 
+    def item_sort_key(self, item: Item) -> time.struct_time:
+        if item.pub_date is None:
+            return time.localtime()
+
+        return item.pub_date
+
     def process(self, source_id: str | None, items: list[Item]) -> list[Item]:
         print(f"[ConcatPlugin#{self.id}] process called, n={len(items)}")
         if source_id is None:
@@ -31,12 +38,13 @@ class Plugin(PluginInterface):
             {"plugin_id": self.id, "state": state}, plugin_state_q
         )
 
-        aggregated_item_dicts: list[dict[str, str]] = []
+        aggregated_item_dicts: list[ItemDict] = []
         for data_source_id in state:
             item_dicts = state[data_source_id]
             aggregated_item_dicts += item_dicts
 
-        ret = list(map(dict_to_item, aggregated_item_dicts))
+        ret_items = list(map(dict_to_item, aggregated_item_dicts))
+        ret = sorted(ret_items, key=self.item_sort_key)
 
         print(f"[ConcatPlugin#{self.id}] process returning items, n={len(ret)}")
         return ret
diff --git a/app/FeedSinkPlugin.py b/app/FeedSinkPlugin.py
index 237214f..c0fc4c9 100644
--- a/app/FeedSinkPlugin.py
+++ b/app/FeedSinkPlugin.py
@@ -1,3 +1,4 @@
+import time
 from typing import Any
 import xml.etree.ElementTree as ET
 
@@ -32,12 +33,40 @@ class Plugin(PluginInterface):
 
         for item in items:
             item_elem = ET.SubElement(channel, "item")
-            item_title = ET.SubElement(item_elem, "title")
-            item_title.text = item.title
-            item_link = ET.SubElement(item_elem, "link")
-            item_link.text = item.link
-            item_description = ET.SubElement(item_elem, "description")
-            item_description.text = item.description
+            if item.title is not None:
+                item_title = ET.SubElement(item_elem, "title")
+                item_title.text = item.title
+            if item.link is not None:
+                item_link = ET.SubElement(item_elem, "link")
+                item_link.text = item.link
+            if item.description is not None:
+                item_description = ET.SubElement(item_elem, "description")
+                item_description.text = item.description
+            if item.category is not None:
+                item_category = ET.SubElement(item_elem, "category")
+                item_category.text = item.category
+            if item.comments is not None:
+                item_comments = ET.SubElement(item_elem, "comments")
+                item_comments.text = item.comments
+            if item.pub_date is not None:
+                item_pub_date = ET.SubElement(item_elem, "pubDate")
+                item_pub_date.text = time.strftime(
+                    "%a, %d %b %Y %H:%M:%S +0000", item.pub_date
+                )
+            if item.author is not None:
+                item_author = ET.SubElement(item_elem, "author")
+                item_author.text = item.author
+
+            for enc in item.enclosures:
+                ET.SubElement(
+                    item_elem,
+                    "enclosure",
+                    {
+                        "length": enc.length,
+                        "type": enc.type,
+                        "url": enc.url,
+                    },
+                )
 
         return ET.tostring(rss, "unicode")
 
diff --git a/app/FeedSourcePlugin.py b/app/FeedSourcePlugin.py
index 7690c8d..575e5d6 100644
--- a/app/FeedSourcePlugin.py
+++ b/app/FeedSourcePlugin.py
@@ -1,12 +1,12 @@
 import feedparser  # type: ignore
 from typing import Any
-from app.Item import Item
-from app.PluginInterface import PluginInterface
-from app.utils import get_param
+from app.Item import Item, ItemEnclosure
+from app.PluginInterface import Params, PluginInterface
+from app.utils import ItemDict, get_param
 
 
 class Plugin(PluginInterface):
-    def __init__(self, id: str, params: dict[str, Any]) -> None:
+    def __init__(self, id: str, params: Params) -> None:
         super().__init__(id, params)
         self.feed_url: str = get_param("feed_url", params)
 
@@ -16,7 +16,7 @@ class Plugin(PluginInterface):
         print(f"[FeedSourcePlugin#{self.id}] process called")
         if source_id is not None:
             raise Exception(
-                f"FeedSourcePlugin#{self.id} can only be scheduled, trying to process items from source {source_id}"
+                f"FeedSourcePlugin#{self.id} can only be scheduled, trying to process items from ItemSource {source_id}"
             )
 
         feed: Any = feedparser.parse(self.feed_url)  # type: ignore
@@ -26,9 +26,30 @@ class Plugin(PluginInterface):
                 f"[FeedSourcePlugin#{self.id}] malformed XML in feed {self.feed_url}"
             )
 
-        for d in feed.entries:
+        for _d in feed.entries:
+            d: ItemDict = _d
+            item_enclosures: list[ItemEnclosure] = []
+            for _e in d["enclosures"]:
+                e: dict[str, str] = _e  # type: ignore
+                item_enclosures.append(
+                    ItemEnclosure(
+                        length=e["length"],
+                        type=e["type"],
+                        url=e["href"],
+                    )
+                )
+
             result_items.append(
-                Item(title=d["title"], link=d["link"], description=d["description"])
+                Item(
+                    title=d.get("title", None),  # type: ignore
+                    link=d.get("link", None),  # type: ignore
+                    description=d.get("description", None),  # type: ignore
+                    author=d.get("author", None),  # type: ignore
+                    pub_date=d.get("published_parsed", None),  # type: ignore
+                    category=d.get("category", None),  # type: ignore
+                    comments=d.get("comments"),  # type: ignore
+                    enclosures=item_enclosures,
+                )
             )
 
         print(
diff --git a/app/FilterPlugin.py b/app/FilterPlugin.py
index 4e6ee3b..0f2b500 100644
--- a/app/FilterPlugin.py
+++ b/app/FilterPlugin.py
@@ -1,11 +1,10 @@
-from typing import Any
 from app.Item import Item
-from app.PluginInterface import PluginInterface
+from app.PluginInterface import Params, PluginInterface
 from app.utils import get_param
 
 
 class Plugin(PluginInterface):
-    def __init__(self, id: str, params: dict[str, Any]) -> None:
+    def __init__(self, id: str, params: Params) -> None:
         super().__init__(id, params)
         self.filter_expr: str = get_param("filter_expr", params)
         print(f"[FilterPlugin#{self.id}] initialized")
diff --git a/app/Item.py b/app/Item.py
index e74239f..23a841e 100644
--- a/app/Item.py
+++ b/app/Item.py
@@ -1,10 +1,25 @@
 from dataclasses import dataclass
+import time
+
+
+@dataclass
+class ItemEnclosure:
+    """RSS Item Enclosure object"""
+
+    length: str
+    type: str
+    url: str
 
 
 @dataclass
 class Item:
     """RSS Item"""
 
-    title: str
-    link: str
-    description: str
+    title: str | None
+    link: str | None
+    description: str | None
+    author: str | None
+    pub_date: time.struct_time | None
+    category: str | None
+    comments: str | None
+    enclosures: list[ItemEnclosure]
diff --git a/app/PluginInterface.py b/app/PluginInterface.py
index 6492ed1..6bc5513 100644
--- a/app/PluginInterface.py
+++ b/app/PluginInterface.py
@@ -1,11 +1,13 @@
 from abc import ABC, abstractmethod
-from typing import Any
+from typing import TypeAlias
 
 from app.Item import Item
 
+Params: TypeAlias = dict[str, str]
+
 
 class PluginInterface(ABC):
-    def __init__(self, id: str, params: dict[str, Any]):
+    def __init__(self, id: str, params: Params):
         self.id = id
         self.params = params
 
diff --git a/app/PluginManager.py b/app/PluginManager.py
index da85428..f50196e 100644
--- a/app/PluginManager.py
+++ b/app/PluginManager.py
@@ -4,7 +4,7 @@ import time
 from types import ModuleType
 from typing import Any
 from app.Item import Item
-from app.PluginInterface import PluginInterface
+from app.PluginInterface import Params, PluginInterface
 from app.AggroConfig import AggroConfig
 from app.utils import get_param
 from app.MemoryState import memory_state
@@ -42,7 +42,7 @@ class PluginManager:
     def build_plugin_instances(self):
         self.graph: dict[str, PluginInterface] = {}
         for id in self.config.plugins:
-            params: dict[str, str] = self.config.plugins[id]
+            params: Params = self.config.plugins[id]
 
             plugin_name = get_param("plugin", params)
             schedule_expr: str | None = params.get("schedule_expr", None)
diff --git a/app/utils.py b/app/utils.py
index 824d370..77ea775 100644
--- a/app/utils.py
+++ b/app/utils.py
@@ -1,17 +1,25 @@
+import time
+from typing import TypeAlias, Union
 from app.Item import Item
 from dataclasses import asdict
 
+from app.PluginInterface import Params
 
-def get_param(key: str, params: dict[str, str]) -> str:
+ItemDictValue: TypeAlias = Union[str, dict[str, "ItemDictValue"], list["ItemDictValue"]]
+ItemDict: TypeAlias = dict[str, ItemDictValue]
+
+
+def get_param(key: str, params: Params) -> str:
     if key not in params:
         raise Exception(f"no {key} field in config entry: " + str(params))
 
     return params[key]
 
 
-def item_to_dict(item: Item) -> dict[str, str]:
+def item_to_dict(item: Item) -> ItemDict:
     return asdict(item)
 
 
-def dict_to_item(d: dict[str, str]) -> Item:
-    return Item(**d)
+def dict_to_item(d: ItemDict) -> Item:
+    pub_date, rest = (lambda pub_date, **rest: (pub_date, rest))(**d)  # type: ignore
+    return Item(pub_date=time.struct_time(pub_date), **rest)  # type: ignore