From bd77148bef39829350670856706b48ad35837df1 Mon Sep 17 00:00:00 2001
From: Jan Tuomi <jans.tuomi@gmail.com>
Date: Mon, 25 Sep 2023 10:11:28 +0300
Subject: Improve http error logging

---
 build-and-push.sh               |  7 +++++++
 plugins/FacebookSourcePlugin.py | 24 ++++++++++++++++++++----
 plugins/ScraperSourcePlugin.py  |  4 +++-
 3 files changed, 30 insertions(+), 5 deletions(-)
 create mode 100755 build-and-push.sh

diff --git a/build-and-push.sh b/build-and-push.sh
new file mode 100755
index 0000000..a6b389d
--- /dev/null
+++ b/build-and-push.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+IMAGE=registry.digitalocean.com/jan-systems-registry/aggro:latest
+
+set -euxo pipefail
+docker build --platform linux/amd64 -t $IMAGE .
+docker push $IMAGE
diff --git a/plugins/FacebookSourcePlugin.py b/plugins/FacebookSourcePlugin.py
index 9e91f84..4af54d7 100644
--- a/plugins/FacebookSourcePlugin.py
+++ b/plugins/FacebookSourcePlugin.py
@@ -82,7 +82,11 @@ def fetch_page_posts(email: str, password: str, page_id: str, limit: int) -> lis
             f"{base_url}/login/", headers=headers, allow_redirects=True
         )
         if cookie_page_resp.status_code >= 400:
-            raise Exception(cookie_page_resp.text)
+            ex = Exception(
+                f"status_code >= 400, got {cookie_page_resp.status_code} when fetching cookies consent page"
+            )
+            ex.add_note(cookie_page_resp.text[0:1000])
+            raise ex
 
         cookie_page = BeautifulSoup(cookie_page_resp.text, "html.parser")
         lsd: str = cookie_page.find("input", {"name": "lsd"})["value"]  # type: ignore
@@ -105,7 +109,11 @@ def fetch_page_posts(email: str, password: str, page_id: str, limit: int) -> lis
         )
 
         if login_page_resp.status_code >= 400:
-            raise Exception(login_page_resp.text)
+            ex = Exception(
+                f"status_code >= 400, got {login_page_resp.status_code} when fetching login page"
+            )
+            ex.add_note(login_page_resp.text[0:1000])
+            raise ex
 
         login_page = BeautifulSoup(login_page_resp.text, "html.parser")
 
@@ -144,7 +152,11 @@ def fetch_page_posts(email: str, password: str, page_id: str, limit: int) -> lis
         )
 
         if login_post_resp.status_code >= 400:
-            raise Exception(login_post_resp.text)
+            ex = Exception(
+                f"status_code >= 400, got {login_post_resp.status_code} when sending login POST"
+            )
+            ex.add_note(login_post_resp.text[0:1000])
+            raise ex
 
         page_timeline_url = f"{base_url}/{page_id}?v=timeline"
         items: list[Item] = []
@@ -157,7 +169,11 @@ def fetch_page_posts(email: str, password: str, page_id: str, limit: int) -> lis
             )
 
             if timeline_resp.status_code >= 400:
-                raise Exception(timeline_resp.text)
+                ex = Exception(
+                    f"status_code >= 400, got {timeline_resp.status_code} when fetching timeline page"
+                )
+                ex.add_note(timeline_resp.text[0:1000])
+                raise ex
 
             timeline = BeautifulSoup(timeline_resp.text, "html.parser")
             posts = timeline.select("section > article")
diff --git a/plugins/ScraperSourcePlugin.py b/plugins/ScraperSourcePlugin.py
index eb5e526..dd7e757 100644
--- a/plugins/ScraperSourcePlugin.py
+++ b/plugins/ScraperSourcePlugin.py
@@ -53,7 +53,9 @@ class Plugin(PluginInterface):
         try:
             resp = session.get(url, allow_redirects=True)
             if resp.status_code >= 400:
-                raise Exception("status_code >= 400")
+                ex = Exception(f"status_code >= 400, got {resp.status_code}")
+                ex.add_note(resp.text[0:1000])
+                raise ex
         except Exception as ex:
             ex.add_note(f"{self.log_prefix} failed to fetch page, url: {url}")
             raise
-- 
cgit v1.3