Improve scraper errors

2 files changed, 17 insertions, 4 deletions

diff --git a/app/PluginManager.py b/app/PluginManager.py
index 86da22f..fdc81a9 100644
--- a/app/PluginManager.py
+++ b/app/PluginManager.py
@@ -42,10 +42,15 @@ class PluginManager:
             return
 
         try:
-            plugin: PluginInterface = self.plugin_instances[id]
-            ret_items: list[Item] = plugin.process(source_id, items)
-            self.propagate(id, ret_items)
-        except Exception as ex:
+            try:
+                plugin: PluginInterface = self.plugin_instances[id]
+                ret_items: list[Item] = plugin.process(source_id, items)
+                self.propagate(id, ret_items)
+            except Exception as ex:
+                ex.add_note(f"context: plugin id: {id}, source id: {source_id}")
+                raise
+
+        except:
             exc = traceback.format_exc()
             print(exc, file=sys.stderr)
             if self.email_alerter:
diff --git a/app/utils.py b/app/utils.py
index 5401c5c..8dc4972 100644
--- a/app/utils.py
+++ b/app/utils.py
@@ -50,3 +50,11 @@ def item_to_dict(item: Item) -> ItemDict:
 def dict_to_item(d: ItemDict) -> Item:
     pub_date, rest = (lambda pub_date, **rest: (pub_date, rest))(**d)  # type: ignore
     return Item(pub_date=datetime.fromisoformat(pub_date), **rest)  # type: ignore
+
+
+def dump_to_file(prefix: str, text: str):
+    now = datetime.now().isoformat()
+    dump_file_name = f"dump/{prefix}__{now}"
+    with open(dump_file_name, "w") as f:
+        f.write(text)
+    return dump_file_name