From 4844c3f4baf97d9ef7a194ab1f24ce922b927b7b Mon Sep 17 00:00:00 2001
From: Jan Tuomi <jans.tuomi@gmail.com>
Date: Fri, 22 Sep 2023 18:38:15 +0300
Subject: Improve scraper errors

---
 app/PluginManager.py | 13 +++++++++----
 app/utils.py         |  8 ++++++++
 2 files changed, 17 insertions(+), 4 deletions(-)

(limited to 'app')

diff --git a/app/PluginManager.py b/app/PluginManager.py
index 86da22f..fdc81a9 100644
--- a/app/PluginManager.py
+++ b/app/PluginManager.py
@@ -42,10 +42,15 @@ class PluginManager:
             return
 
         try:
-            plugin: PluginInterface = self.plugin_instances[id]
-            ret_items: list[Item] = plugin.process(source_id, items)
-            self.propagate(id, ret_items)
-        except Exception as ex:
+            try:
+                plugin: PluginInterface = self.plugin_instances[id]
+                ret_items: list[Item] = plugin.process(source_id, items)
+                self.propagate(id, ret_items)
+            except Exception as ex:
+                ex.add_note(f"context: plugin id: {id}, source id: {source_id}")
+                raise
+
+        except:
             exc = traceback.format_exc()
             print(exc, file=sys.stderr)
             if self.email_alerter:
diff --git a/app/utils.py b/app/utils.py
index 5401c5c..8dc4972 100644
--- a/app/utils.py
+++ b/app/utils.py
@@ -50,3 +50,11 @@ def item_to_dict(item: Item) -> ItemDict:
 def dict_to_item(d: ItemDict) -> Item:
     pub_date, rest = (lambda pub_date, **rest: (pub_date, rest))(**d)  # type: ignore
     return Item(pub_date=datetime.fromisoformat(pub_date), **rest)  # type: ignore
+
+
+def dump_to_file(prefix: str, text: str):
+    now = datetime.now().isoformat()
+    dump_file_name = f"dump/{prefix}__{now}"
+    with open(dump_file_name, "w") as f:
+        f.write(text)
+    return dump_file_name
-- 
cgit v1.3