Add dotenv and env var in config support

13 files changed, 93 insertions, 43 deletions

diff --git a/.gitignore b/.gitignore
index 15eca83..e74de77 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 __pycache__/
 .DS_Store
 db.json
+.env
diff --git a/Aggrofile b/Aggrofile
index f78dfd8..3828653 100644
--- a/Aggrofile
+++ b/Aggrofile
@@ -4,8 +4,8 @@
         "plutonium74_fb": {
             "plugin": "FacebookSourcePlugin",
             "schedule_expr": "schedule.every(30).seconds",
-            "login_email": "",
-            "login_password": "",
+            "login_email": "${AGGRO_FB_LOGIN_EMAIL}",
+            "login_password": "${AGGRO_FB_LOGIN_PASSWORD}",
             "page_id": "Batushkaband",
             "limit": 10,
             "__comment": "https://www.facebook.com/Plutonium74"
diff --git a/aggro.py b/aggro.py
index 032956d..6d42687 100644
--- a/aggro.py
+++ b/aggro.py
@@ -1,8 +1,7 @@
 import json
 import os
 import hashlib
-
-# from multiprocessing import Process
+from dotenv import load_dotenv
 from threading import Thread
 import time
 from typing import Any
@@ -13,6 +12,9 @@ from app.MemoryState import memory_state
 from app.PluginManager import PluginManager
 from app.DatabaseManager import database_manager
 from app.server import run_web_server
+from app.utils import get_config, get_config_or_default
+
+load_dotenv()
 
 
 def run_plugin_thread(manager: PluginManager, config: AggroConfig):
@@ -37,11 +39,11 @@ if __name__ == "__main__":
         aggrofile = json.loads(aggrofile_content)
 
     aggro_config = AggroConfig(
-        server_host=aggrofile.get("server_host", "localhost"),
-        server_port=aggrofile.get("server_port", 8080),
-        db_path=aggrofile.get("db_path", "db.json"),
-        plugins=aggrofile["plugins"],
-        graph=aggrofile["graph"],
+        server_host=get_config_or_default(aggrofile, "server_host", "localhost"),
+        server_port=get_config_or_default(aggrofile, "server_port", 8080),
+        db_path=get_config_or_default(aggrofile, "db_path", "db.json"),
+        plugins=get_config(aggrofile, "plugins"),
+        graph=get_config(aggrofile, "graph"),
     )
 
     database_manager.setup(aggro_config)
diff --git a/app/AggroConfig.py b/app/AggroConfig.py
index 7e6a5f3..789fd6b 100644
--- a/app/AggroConfig.py
+++ b/app/AggroConfig.py
@@ -1,10 +1,12 @@
 from dataclasses import dataclass
 
+from app.PluginInterface import Params
+
 
 @dataclass
 class AggroConfig:
     server_host: str
     server_port: int
     db_path: str
-    plugins: dict[str, dict[str, str]]
+    plugins: dict[str, Params]
     graph: dict[str, list[str]]
diff --git a/app/PluginManager.py b/app/PluginManager.py
index 68149fd..fb6b4c3 100644
--- a/app/PluginManager.py
+++ b/app/PluginManager.py
@@ -6,7 +6,7 @@ from typing import Any
 from app.Item import Item
 from app.PluginInterface import Params, PluginInterface
 from app.AggroConfig import AggroConfig
-from app.utils import get_param
+from app.utils import get_config
 from app.MemoryState import memory_state
 
 
@@ -44,7 +44,7 @@ class PluginManager:
         for id in self.config.plugins:
             params: Params = self.config.plugins[id]
 
-            plugin_name = get_param("plugin", params)
+            plugin_name: str = get_config(params, "plugin")
             schedule_expr: str | None = params.get("schedule_expr", None)
 
             if plugin_name not in self._plugins:
diff --git a/app/utils.py b/app/utils.py
index 626ee24..f0c175f 100644
--- a/app/utils.py
+++ b/app/utils.py
@@ -1,19 +1,36 @@
+import os
 from datetime import datetime
-from typing import TypeAlias, Union
+from typing import Any, TypeAlias, Union
 from app.Item import Item
 from dataclasses import asdict
 
-from app.PluginInterface import Params
-
 ItemDictValue: TypeAlias = Union[str, dict[str, "ItemDictValue"], list["ItemDictValue"]]
 ItemDict: TypeAlias = dict[str, ItemDictValue]
 
 
-def get_param(key: str, params: Params) -> str:
-    if key not in params:
-        raise Exception(f"no {key} field in config entry: " + str(params))
+def evaluate_env_ref(v: Any) -> str:
+    if type(v) == str and v.startswith("${") and v.endswith("}"):
+        return os.environ[v[2:-1]]  # type: ignore
+    else:
+        return v
+
+
+def get_config(config: dict[str, Any], key: str) -> Any:
+    v: Any
+    try:
+        v = config[key]
+    except KeyError:
+        raise Exception(f"no {key} field in config: " + str(config))
+
+    return evaluate_env_ref(v)
+
+
+def get_config_or_default(
+    config: dict[str, Any], key: str, default: Any | None = None
+) -> Any:
+    v: Any = config.get(key, default)
 
-    return params[key]
+    return evaluate_env_ref(v)
 
 
 def item_to_dict(item: Item) -> ItemDict:
diff --git a/plugins/DigestPlugin.py b/plugins/DigestPlugin.py
index d6e8eb4..b56af0d 100644
--- a/plugins/DigestPlugin.py
+++ b/plugins/DigestPlugin.py
@@ -7,7 +7,7 @@ from tinydb import Query
 from app.Item import Item
 from app.PluginInterface import Params, PluginInterface
 from app.DatabaseManager import database_manager
-from app.utils import dict_to_item, get_param, item_to_dict
+from app.utils import dict_to_item, get_config_or_default, item_to_dict, get_config
 
 
 def add_days(start: datetime, days: int):
@@ -99,14 +99,16 @@ def parse_interval(interval: str) -> tuple[int, str]:
 class Plugin(PluginInterface):
     def __init__(self, id: str, params: Params) -> None:
         super().__init__(id, params)
-        self.digest_title_prefix = get_param("digest_title_prefix", params)
-        self.digest_description = params.get("digest_description", None)
-        self.digest_link = params.get("digest_link", None)
-        from_datatime_str = get_param("from_datetime", params)
+        self.digest_title_prefix = get_config(params, "digest_title_prefix")
+        self.digest_description = get_config_or_default(
+            params, "digest_description", None
+        )
+        self.digest_link = get_config_or_default(params, "digest_link", None)
+        from_datatime_str = get_config(params, "from_datetime")
         self.from_datetime = datetime.fromisoformat(from_datatime_str)
-        interval = get_param("interval", params)
+        interval = get_config(params, "interval")
         self.interval_pair = parse_interval(interval)
-        self.max_length = int(params.get("max_length", "1000"))
+        self.max_length = int(get_config_or_default(params, "max_length", "1000"))
         print(f"[DigestPlugin#{self.id}] initialized")
 
     def process(self, source_id: str | None, items: list[Item]) -> list[Item]:
diff --git a/plugins/FacebookSourcePlugin.py b/plugins/FacebookSourcePlugin.py
index d4f6f16..e7ce1ab 100644
--- a/plugins/FacebookSourcePlugin.py
+++ b/plugins/FacebookSourcePlugin.py
@@ -7,7 +7,7 @@ from datetime import datetime, timedelta
 from bs4 import BeautifulSoup, Tag
 from app.Item import Item
 from app.PluginInterface import Params, PluginInterface
-from app.utils import get_param
+from app.utils import get_config, get_config_or_default
 
 
 def replace_lm_links(text: str) -> str:
@@ -201,10 +201,10 @@ def fetch_page_posts(email: str, password: str, page_id: str, limit: int) -> lis
 class Plugin(PluginInterface):
     def __init__(self, id: str, params: Params) -> None:
         super().__init__(id, params)
-        self.login_email = get_param("login_email", params)
-        self.login_password = get_param("login_password", params)
-        self.page_id = get_param("page_id", params)
-        self.limit = int(params.get("limit", "10"))
+        self.login_email = get_config(params, "login_email")
+        self.login_password = get_config(params, "login_password")
+        self.page_id = get_config(params, "page_id")
+        self.limit = int(get_config_or_default(params, "limit", "10"))
 
         print(f"[FacebookSourcePlugin#{self.id}] initialized")
 
diff --git a/plugins/FeedSinkPlugin.py b/plugins/FeedSinkPlugin.py
index 02e20e9..623bb49 100644
--- a/plugins/FeedSinkPlugin.py
+++ b/plugins/FeedSinkPlugin.py
@@ -5,17 +5,17 @@ from tinydb import Query
 
 from app.Item import Item
 from app.PluginInterface import PluginInterface
-from app.utils import get_param
+from app.utils import get_config, get_config_or_default
 from app.DatabaseManager import database_manager
 
 
 class Plugin(PluginInterface):
     def __init__(self, id: str, params: dict[str, Any]) -> None:
         super().__init__(id, params)
-        self.feed_id: str = get_param("feed_id", params)
-        self.feed_title: str = get_param("feed_title", params)
-        self.feed_link: str | None = params.get("feed_link", None)
-        self.feed_description: str = get_param("feed_description", params)
+        self.feed_id: str = get_config(params, "feed_id")
+        self.feed_title: str = get_config(params, "feed_title")
+        self.feed_link: str | None = get_config_or_default(params, "feed_link", None)
+        self.feed_description: str = get_config(params, "feed_description")
         print(f"[FeedSinkPlugin#{self.id}] initialized")
         print(f"[FeedSinkPlugin#{self.id}] feed will be served at path /{self.feed_id}")
 
diff --git a/plugins/FeedSourcePlugin.py b/plugins/FeedSourcePlugin.py
index 2f0aa94..fd27081 100644
--- a/plugins/FeedSourcePlugin.py
+++ b/plugins/FeedSourcePlugin.py
@@ -4,7 +4,7 @@ from datetime import datetime, timezone
 from typing import Any
 from app.Item import Item, ItemEnclosure
 from app.PluginInterface import Params, PluginInterface
-from app.utils import ItemDict, get_param
+from app.utils import ItemDict, get_config
 
 
 def struct_time_to_utc_datetime(struct_time: time.struct_time) -> datetime:
@@ -17,7 +17,7 @@ def struct_time_to_utc_datetime(struct_time: time.struct_time) -> datetime:
 class Plugin(PluginInterface):
     def __init__(self, id: str, params: Params) -> None:
         super().__init__(id, params)
-        self.feed_url: str = get_param("feed_url", params)
+        self.feed_url: str = get_config(params, "feed_url")
 
         print(f"[FeedSourcePlugin#{self.id}] initialized")
 
diff --git a/plugins/FilterPlugin.py b/plugins/FilterPlugin.py
index 0f2b500..11cbc68 100644
--- a/plugins/FilterPlugin.py
+++ b/plugins/FilterPlugin.py
@@ -1,12 +1,12 @@
 from app.Item import Item
 from app.PluginInterface import Params, PluginInterface
-from app.utils import get_param
+from app.utils import get_config
 
 
 class Plugin(PluginInterface):
     def __init__(self, id: str, params: Params) -> None:
         super().__init__(id, params)
-        self.filter_expr: str = get_param("filter_expr", params)
+        self.filter_expr: str = get_config(params, "filter_expr")
         print(f"[FilterPlugin#{self.id}] initialized")
 
     def process(self, source_id: str | None, items: list[Item]) -> list[Item]:
diff --git a/plugins/MapPlugin.py b/plugins/MapPlugin.py
index 8b5dae7..d924f2f 100644
--- a/plugins/MapPlugin.py
+++ b/plugins/MapPlugin.py
@@ -2,7 +2,7 @@ from dataclasses import fields
 from typing import Any
 from app.Item import Item
 from app.PluginInterface import Params, PluginInterface
-from app.utils import get_param
+from app.utils import get_config
 
 
 def set_field(item: Item, k: str, v: Any):
@@ -17,7 +17,7 @@ def set_field(item: Item, k: str, v: Any):
 class Plugin(PluginInterface):
     def __init__(self, id: str, params: Params) -> None:
         super().__init__(id, params)
-        self.map_expr: str = get_param("map_expr", params)
+        self.map_expr: str = get_config(params, "map_expr")
         print(f"[MapItemPlugin#{self.id}] initialized")
 
     def process(self, source_id: str | None, items: list[Item]) -> list[Item]:
diff --git a/requirements.txt b/requirements.txt
index 9fc810c..2e697ae 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,10 +1,36 @@
+appdirs==1.4.4
+beautifulsoup4==4.12.2
 bottle==0.12.25
+bs4==0.0.1
 certifi==2023.7.22
 charset-normalizer==3.2.0
+cssselect==1.2.0
+dateparser==1.1.8
+demjson3==3.0.6
+facebook-scraper==0.2.59
+fake-useragent==1.2.1
 feedparser==6.0.10
 idna==3.4
+importlib-metadata==6.8.0
+lxml==4.9.3
+parse==1.19.1
+pyee==8.2.2
+pyppeteer==1.0.2
+pyquery==2.0.0
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+pytz==2023.3
+regex==2023.8.8
 requests==2.31.0
+requests-html==0.10.0
 schedule==1.2.0
 sgmllib3k==1.0.0
+six==1.16.0
+soupsieve==2.5
 tinydb==4.8.0
-urllib3==2.0.4
+tqdm==4.66.1
+tzlocal==5.0.1
+urllib3==1.26.16
+w3lib==2.1.2
+websockets==10.4
+zipp==3.16.2