diff options
| -rw-r--r-- | Aggrofile | 88 | ||||
| -rw-r--r-- | README.md | 22 | ||||
| -rw-r--r-- | app/server.py | 12 | ||||
| -rw-r--r-- | docker-compose.yml | 2 | ||||
| -rw-r--r-- | plugins/FeedSinkPlugin.py | 2 |
5 files changed, 46 insertions, 80 deletions
@@ -3,15 +3,6 @@ "server_host": "0.0.0.0", "server_port": 8080, "plugins": { - "plutonium74_fb": { - "plugin": "FacebookSourcePlugin", - "schedule_expr": "schedule.every(24).hours", - "login_email": "${AGGRO_FB_LOGIN_EMAIL}", - "login_password": "${AGGRO_FB_LOGIN_PASSWORD}", - "page_id": "Plutonium74", - "limit": 10, - "__comment": "https://www.facebook.com/Plutonium74" - }, "aaniwalli_scraper": { "plugin": "ScraperSourcePlugin", "schedule_expr": "schedule.every(10).seconds", @@ -20,85 +11,30 @@ "selector_title": "post.select('h1')", "selector_date": "post.select('.date')", "selector_link": "post.select('a.event-link')", - "selector_description": "detail_page.select('.event-page')", + "selector_description": "detail_page.select('.right-pane')", "selector_image": "post.select('.event-link > .image')" }, "aaniwalli_feed": { "plugin": "FeedSinkPlugin", "feed_id": "aaniwalli", "feed_title": "Ääniwalli", + "feed_link": "https://www.ääniwalli.fi/", "feed_description": "Ääniwalli events" }, - "plutonium74_fb_feed": { - "plugin": "FeedSinkPlugin", - "feed_id": "plutonium74_fb", - "feed_title": "Plutonium74 FB", - "feed_description": "Plutonium74 FB posts" - }, - "juusomikkonen_blog_rss": { - "plugin": "FeedSourcePlugin", - "schedule_expr": "schedule.every(10).seconds", - "feed_url": "https://juusomikkonen.com/feed.xml" - }, - "juusomikkonen_add_author": { - "plugin": "MapPlugin", - "map_expr": "set_field(item, 'author', 'Juuso Mikkonen')" - }, - "mariusschulz_blog_atom": { - "plugin": "FeedSourcePlugin", + "barloose_scraper": { + "plugin": "JsonApiSourcePlugin", "schedule_expr": "schedule.every(10).seconds", - "feed_url": "https://feeds.feedburner.com/mariusschulz" - }, - "mariusschulz_add_author": { - "plugin": "MapPlugin", - "map_expr": "set_field(item, 'author', 'Marius Schulz')" - }, - "typescript_digest": { - "plugin": "DigestPlugin", - "from_datetime": "2010-01-01T00:00:00Z", - "interval": "2m", - "max_length": 1000, - "digest_title_prefix": "Only Typescript Bimonthly" - }, - "concat": { - "plugin": "ConcatPlugin" - }, - "title_filter": { - "plugin": "FilterPlugin", - "filter_expr": "'TypeScript' in item.title" - }, - "typescript_digest_feed": { - "plugin": "FeedSinkPlugin", - "feed_id": "typescript_digest", - "feed_title": "Only Typescript Digest", - "feed_description": "Bimonthly digest where all posts have Typescript in the title" + "url": "https://barloose.com/wp-json/tribe/events/v1/events", + "selector_post": "data['events']", + "selector_title": "post['title']", + "selector_link": "post['url']", + "selector_date": "post['start_date']", + "selector_description": "post['description']", + "selector_image": "post['image']['url']", + "show_image_in_description": true } }, "graph": { - "juusomikkonen_blog_rss": [ - "juusomikkonen_add_author" - ], - "juusomikkonen_add_author": [ - "concat" - ], - "mariusschulz_blog_atom": [ - "mariusschulz_add_author" - ], - "mariusschulz_add_author": [ - "concat" - ], - "concat": [ - "title_filter" - ], - "title_filter": [ - "typescript_digest" - ], - "typescript_digest": [ - "typescript_digest_feed" - ], - "plutonium74_fb": [ - "plutonium74_fb_feed" - ], "aaniwalli_scraper": [ "aaniwalli_feed" ] @@ -1,2 +1,22 @@ # aggro -A feed manipulator & aggregator +A feed manipulator & aggregator built with Python. + +Aggro is a tool for creating, manipulating and serving syndicated feeds based on data from arbitrary web pages with a post-like structure. Aggro is capable of sourcing content by scraping HTML, using JSON APIs, using existing RSS/Atom feeds and reading Facebook posts. These content streams can be manipulated (filtered, mapped over, concatenated, digested) to create new, transformed feeds. + +Aggro is based on a plugin system that is configured using a declarative JSON configuration file called Aggrofile. An Aggrofile defines the used plugin instances and a directed acyclic graph (DAG) between the plugins. Plugins are triggered either on a defined schedule or by propagating data through the DAG. + +Aggro was born from a concrete need to aggregate post-like data from various websites, such as event venues in my city and artists on social media. The design of Aggro is very pragmatic: features are implemented if I need them for my personal use case. As such, Aggro is a very personal project. Aggro is not mature and probably never will be. + +Aggro is free and open source, MIT licensed software. + +## Installation and running + +Aggro is intended to be run as a long-lived process on a server. I personally run it as a docker container on a DigitalOcean VPS, built with the Dockerfile in the repository root. See `docker-compose.yml` for a usage example. See the `Aggrofile` for a configuration example. + +## Plugins + +Plugin documentation is a work in progress. For now, check out the implementation (see `plugins -> *Plugin.py -> Plugin.__init__`) to figure out what parameters to give to each plugin. + +## Author + +Jan Tuomi <<jans.tuomi@gmail.com>> diff --git a/app/server.py b/app/server.py index 081c961..3ff981e 100644 --- a/app/server.py +++ b/app/server.py @@ -33,7 +33,8 @@ def index(): raise Exception("Database is not initialized") Q = Query() - feeds = database_manager.feeds.all() + _feeds = database_manager.feeds.all() + feeds: list[dict[str, str]] = _feeds # type: ignore bottle.response.set_header("content-type", "text/html") page = f""" @@ -67,6 +68,10 @@ def index(): th, td {{ padding: 8px; }} + + footer {{ + margin-top: 32px; + }} </style> </head> <body> @@ -74,6 +79,11 @@ def index(): <i>Feed manipulator</i> <h2>Feeds served at this address</h2> {feeds_to_table(feeds)} + <footer> + <p> + <a href="https://github.com/jantuomi/aggro">Aggro</a> is MIT licensed open source software. + </p> + </footer> </body> </html> """ diff --git a/docker-compose.yml b/docker-compose.yml index 97b3d78..b0b5007 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: aggro: - image: registry.digitalocean.com/jan-systems-registry/aggro:latest + image: aggro:latest build: context: . ports: diff --git a/plugins/FeedSinkPlugin.py b/plugins/FeedSinkPlugin.py index b51d64d..a49e0ae 100644 --- a/plugins/FeedSinkPlugin.py +++ b/plugins/FeedSinkPlugin.py @@ -121,5 +121,5 @@ class Plugin(PluginInterface): Q = Query() database_manager.feeds.upsert(ret, Q.feed_id == self.feed_id) # type: ignore - self.log('build complete for feed id "{self.feed_id}"') + self.log(f'build complete for feed id "{self.feed_id}"') return [] |