diff options
| -rw-r--r-- | Aggrofile | 2 | ||||
| -rw-r--r-- | Dockerfile | 11 | ||||
| -rw-r--r-- | docker-compose.yml | 13 | ||||
| -rw-r--r-- | plugins/FacebookSourcePlugin.py | 6 | ||||
| -rw-r--r-- | plugins/ScraperSourcePlugin.py | 14 |
5 files changed, 37 insertions, 9 deletions
@@ -1,5 +1,7 @@ { "db_path": "db.json", + "server_host": "0.0.0.0", + "server_port": 8080, "plugins": { "plutonium74_fb": { "plugin": "FacebookSourcePlugin", diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..64a6c96 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.11-alpine + +WORKDIR /app +COPY requirements.txt ./ +RUN pip install -r requirements.txt + +COPY app ./app +COPY plugins ./plugins +COPY aggro.py ./aggro.py + +CMD ["python", "aggro.py"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..97b3d78 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,13 @@ +version: '3' + +services: + aggro: + image: registry.digitalocean.com/jan-systems-registry/aggro:latest + build: + context: . + ports: + - 8080:8080 + volumes: + - ${PWD}/db.json:/app/db.json + - ${PWD}/.env:/app/.env + - ${PWD}/Aggrofile:/app/Aggrofile diff --git a/plugins/FacebookSourcePlugin.py b/plugins/FacebookSourcePlugin.py index e7ce1ab..24784e3 100644 --- a/plugins/FacebookSourcePlugin.py +++ b/plugins/FacebookSourcePlugin.py @@ -83,7 +83,7 @@ def fetch_page_posts(email: str, password: str, page_id: str, limit: int) -> lis if cookie_page_resp.status_code >= 400: raise Exception(cookie_page_resp.text) - cookie_page = BeautifulSoup(cookie_page_resp.text, features="xml") + cookie_page = BeautifulSoup(cookie_page_resp.text, features=["xml", "lxml"]) lsd: str = cookie_page.find("input", {"name": "lsd"})["value"] # type: ignore jazoest: str = cookie_page.find("input", {"name": "jazoest"})["value"] # type: ignore @@ -106,7 +106,7 @@ def fetch_page_posts(email: str, password: str, page_id: str, limit: int) -> lis if login_page_resp.status_code >= 400: raise Exception(login_page_resp.text) - login_page = BeautifulSoup(login_page_resp.text, features="lxml") + login_page = BeautifulSoup(login_page_resp.text, features=["xml", "lxml"]) lsd: str = login_page.find("input", {"name": "lsd"})["value"] # type: ignore jazoest: str = login_page.find("input", {"name": "jazoest"})["value"] # type: ignore @@ -158,7 +158,7 @@ def fetch_page_posts(email: str, password: str, page_id: str, limit: int) -> lis if timeline_resp.status_code >= 400: raise Exception(timeline_resp.text) - timeline = BeautifulSoup(timeline_resp.text, features="lxml") + timeline = BeautifulSoup(timeline_resp.text, features=["xml", "lxml"]) posts = timeline.select("section > article") for post in posts: diff --git a/plugins/ScraperSourcePlugin.py b/plugins/ScraperSourcePlugin.py index caa4f75..ea28f97 100644 --- a/plugins/ScraperSourcePlugin.py +++ b/plugins/ScraperSourcePlugin.py @@ -46,7 +46,7 @@ class Plugin(PluginInterface): result_items: list[Item] = [] with requests.session() as session: page_resp = session.get(self.url, allow_redirects=True) - page_elem = BeautifulSoup(page_resp.text) + page_elem = BeautifulSoup(page_resp.text, features="lxml") post_elems = eval(self.selector_post, {"page": page_elem}) for post_elem in post_elems: @@ -64,7 +64,9 @@ class Plugin(PluginInterface): detail_page_resp = session.get( detail_page_url, allow_redirects=True ) - detail_page_elem = BeautifulSoup(detail_page_resp.text) + detail_page_elem = BeautifulSoup( + detail_page_resp.text, features="lxml" + ) guid = detail_page_url else: detail_page_elem = None @@ -79,7 +81,7 @@ class Plugin(PluginInterface): "detail_page": detail_page_elem, }, )[0] - title = title_elem.get_text() + title = title_elem.get_text().strip() else: title = None @@ -92,7 +94,7 @@ class Plugin(PluginInterface): "detail_page": detail_page_elem, }, )[0] - description = str(description_elem) + description = str(description_elem).strip() else: description = None @@ -105,7 +107,7 @@ class Plugin(PluginInterface): "detail_page": detail_page_elem, }, )[0] - date = date_elem.get_text() + date = date_elem.get_text().strip() else: date = None @@ -118,7 +120,7 @@ class Plugin(PluginInterface): "detail_page": detail_page_elem, }, )[0] - author = author_elem.get_text() + author = author_elem.get_text().strip() else: author = None |