From 861333c28a8f0eb5663c5b744212d4d1a5eaf4ab Mon Sep 17 00:00:00 2001 From: Jan Tuomi Date: Sat, 20 Apr 2024 22:08:05 +0300 Subject: Improve scraper logs --- plugins/ScraperSourcePlugin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/ScraperSourcePlugin.py b/plugins/ScraperSourcePlugin.py index e33b625..a1930f9 100644 --- a/plugins/ScraperSourcePlugin.py +++ b/plugins/ScraperSourcePlugin.py @@ -87,7 +87,7 @@ class Plugin(PluginInterface): dump_file_name = dump_to_file(self.id, str(ctx)) ex.add_note( f"{self.log_prefix} failed to eval selector:\n {selector}\n" - + f"HTML dumped to {dump_file_name}" + + f"HTML dumped to {dump_file_name}.\n" ) raise @@ -152,6 +152,7 @@ class Plugin(PluginInterface): # Fetch and parse post detail page if detail_page_url: # Fetch + self.log(f"scraping detail page: {detail_page_url}") detail_page_resp = self.get_url(session, url=detail_page_url) detail_page_elem = self.parse_html( from_url=detail_page_url, html=detail_page_resp.text -- cgit v1.3