diff options
| -rw-r--r-- | Aggrofile | 12 | ||||
| -rw-r--r-- | aggro.py | 26 | ||||
| -rw-r--r-- | app/AggroConfig.py | 2 | ||||
| -rw-r--r-- | app/FeedSinkPlugin.py | 52 | ||||
| -rw-r--r-- | app/database.py | 12 | ||||
| -rw-r--r-- | app/server.py | 31 | ||||
| -rw-r--r-- | requirements.txt | 1 |
7 files changed, 93 insertions, 43 deletions
@@ -6,19 +6,29 @@ "schedule_expr": "schedule.every(10).seconds", "feed_url": "https://juusomikkonen.com/feed.xml" }, + "mariusschulz_blog_atom": { + "plugin": "FeedSourcePlugin", + "schedule_expr": "schedule.every(10).seconds", + "feed_url": "https://feeds.feedburner.com/mariusschulz" + }, "title_filter": { "plugin": "FilterPlugin", "filter_expr": "'TypeScript' in item.title" }, "output_feed": { "plugin": "FeedSinkPlugin", - "feed_name": "test_feed" + "feed_id": "test_feed", + "feed_title": "Only Typescript posts", + "feed_description": "Feed where all posts have Typescript in the title" } }, "graph": { "juusomikkonen_blog_rss": [ "title_filter" ], + "mariusschulz_blog_atom": [ + "title_filter" + ], "title_filter": [ "output_feed" ] @@ -1,19 +1,28 @@ import json import os -import threading + +# from multiprocessing import Process +from threading import Thread import time from app.AggroConfig import AggroConfig from app.MemoryState import memory_state from app.PluginManager import PluginManager -from app.database import setup_db +from app.database import database_manager +from app.server import run_web_server -def run_plugin_thread(): +def run_plugin_thread(manager: PluginManager, config: AggroConfig): print("Plugin thread starting...") manager.run() print("Plugin thread exiting...") +def run_server_thread(config: AggroConfig): + print("Server thread starting...") + run_web_server(config.server_host, config.server_port) + print("Server thread exiting...") + + if __name__ == "__main__": print("Starting aggro. Press CTRL-C to exit.") @@ -23,21 +32,27 @@ if __name__ == "__main__": aggrofile_content = json.loads(f.read()) aggro_config = AggroConfig( + server_host=aggrofile_content.get("server_host", "localhost"), + server_port=aggrofile_content.get("server_port", 8080), db_path=aggrofile_content.get("db_path", "db.json"), plugins=aggrofile_content["plugins"], graph=aggrofile_content["graph"], ) - setup_db(aggro_config) + database_manager.setup(aggro_config) manager = PluginManager(aggro_config) manager.build_plugin_instances() memory_state.running = True - plugin_thread = threading.Thread(target=run_plugin_thread) + plugin_thread = Thread(target=run_plugin_thread, args=[manager, aggro_config]) plugin_thread.start() + server_thread = Thread(target=run_server_thread, args=[aggro_config]) + server_thread.daemon = True + server_thread.start() + try: while memory_state.running: time.sleep(0.1) @@ -45,4 +60,5 @@ if __name__ == "__main__": memory_state.running = False print("Waiting for threads to exit...") plugin_thread.join() + print("Server thread exiting...") print("Main thread exiting...") diff --git a/app/AggroConfig.py b/app/AggroConfig.py index 6ff6801..7e6a5f3 100644 --- a/app/AggroConfig.py +++ b/app/AggroConfig.py @@ -3,6 +3,8 @@ from dataclasses import dataclass @dataclass class AggroConfig: + server_host: str + server_port: int db_path: str plugins: dict[str, dict[str, str]] graph: dict[str, list[str]] diff --git a/app/FeedSinkPlugin.py b/app/FeedSinkPlugin.py index 6b0125d..2c85167 100644 --- a/app/FeedSinkPlugin.py +++ b/app/FeedSinkPlugin.py @@ -1,47 +1,34 @@ from typing import Any -from app.Item import Item -from app.PluginInterface import PluginInterface -from app.utils import get_param import xml.etree.ElementTree as ET +from tinydb import Query -# <?xml version="1.0" encoding="UTF-8" ?> -# <rss version="2.0"> - -# <channel> -# <title>W3Schools Home Page</title> -# <link>https://www.w3schools.com</link> -# <description>Free web building tutorials</description> -# <item> -# <title>RSS Tutorial</title> -# <link>https://www.w3schools.com/xml/xml_rss.asp</link> -# <description>New RSS tutorial on W3Schools</description> -# </item> -# <item> -# <title>XML Tutorial</title> -# <link>https://www.w3schools.com/xml</link> -# <description>New XML tutorial on W3Schools</description> -# </item> -# </channel> - -# </rss> +from app.Item import Item +from app.PluginInterface import PluginInterface +from app.utils import get_param +from app.database import database_manager class Plugin(PluginInterface): def __init__(self, id: str, params: dict[str, Any]) -> None: super().__init__(id, params) - self.feed_name: str = get_param("feed_name", params) + self.feed_id: str = get_param("feed_id", params) + self.feed_title: str = get_param("feed_title", params) + self.feed_link: str | None = params.get("feed_link", None) + self.feed_description: str = get_param("feed_description", params) print(f"[FeedSinkPlugin#{self.id}] initialized") + print(f"[FeedSinkPlugin#{self.id}] feed will be served at path /{self.feed_id}") def build_xml(self, items: list[Item]): rss = ET.Element("rss", {"version": "2.0"}) channel = ET.SubElement(rss, "channel") title = ET.SubElement(channel, "title") - title.text = "channel title" - link = ET.SubElement(channel, "link") - link.text = "channel link" + title.text = self.feed_title + if self.feed_link is not None: + link = ET.SubElement(channel, "link") + link.text = self.feed_link description = ET.SubElement(channel, "description") - description.text = "channel description" + description.text = self.feed_description for item in items: item_elem = ET.SubElement(channel, "item") @@ -56,10 +43,15 @@ class Plugin(PluginInterface): def process(self, source_id: str | None, items: list[Item]) -> list[Item]: print(f"[FeedSinkPlugin#{self.id}] process called, n={len(items)}") + if source_id is None: raise Exception(f"FeedSinkPlugin#{self.id} can not be scheduled") + if database_manager.db is None: + raise Exception("Database is not initialized") ret = self.build_xml(items) - print(f"[FeedSinkPlugin#{self.id}] process returning XML:") - print(ret) + + Q = Query() + database_manager.db.upsert({"feed_id": self.feed_id, "feed_xml": ret}, Q.feed_id == self.feed_id) # type: ignore + print(f"[FeedSinkPlugin#{self.id}] processed") return [] diff --git a/app/database.py b/app/database.py index fdeac1a..b796dc8 100644 --- a/app/database.py +++ b/app/database.py @@ -3,14 +3,12 @@ from tinydb import TinyDB from app.AggroConfig import AggroConfig -def setup_db(config: AggroConfig): - global database_manager - database_manager = DatabaseManager(config) - - class DatabaseManager: - def __init__(self, config: AggroConfig): + def __init__(self): + self.db: TinyDB | None = None + + def setup(self, config: AggroConfig): self.db = TinyDB(config.db_path) -database_manager: DatabaseManager +database_manager: DatabaseManager = DatabaseManager() diff --git a/app/server.py b/app/server.py new file mode 100644 index 0000000..785e30f --- /dev/null +++ b/app/server.py @@ -0,0 +1,31 @@ +from typing import Any +import bottle as _bottle # type: ignore +from tinydb import Query + +from app.database import database_manager + +bottle: Any = _bottle + + +@bottle.route("/<feed_id>") +def index(feed_id: str): + if database_manager.db is None: + raise Exception("Database is not initialized") + + Q = Query() + res = database_manager.db.search(Q.feed_id == feed_id) + if len(res) == 0: + bottle.abort(400, f"No feed found with id {feed_id}") + + if len(res) > 1: + bottle.abort(400, f"Weird number of feeds found with id {feed_id}: {len(res)}") + + bottle.response.set_header("content-type", "application/xml") + + feed: Any = res[0] # type: ignore + feed_xml: str = feed["feed_xml"] + return feed_xml + + +def run_web_server(host: str, port: int): + bottle.run(host=host, port=port) diff --git a/requirements.txt b/requirements.txt index 878f71c..95c005c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +bottle==0.12.25 feedparser==6.0.10 schedule==1.2.0 sgmllib3k==1.0.0 |