From b482facc9e8f5599cad176885b06bba6eef055d3 Mon Sep 17 00:00:00 2001 From: Jan Tuomi Date: Sat, 22 Mar 2025 10:09:10 +0200 Subject: Add crawl_detail_page option to Scraper --- plugins/ScraperSourcePlugin.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/ScraperSourcePlugin.py b/plugins/ScraperSourcePlugin.py index 52e3e3f..cb3fe5e 100644 --- a/plugins/ScraperSourcePlugin.py +++ b/plugins/ScraperSourcePlugin.py @@ -37,6 +37,9 @@ class Plugin(PluginInterface): self.show_image_in_description: bool = get_config_or_default( params, "show_image_in_description", True ) + self.crawl_detail_page: bool = get_config_or_default( + params, "crawl_detail_page", True + ) print(f"[ScraperSourcePlugin#{self.id}] initialized") @@ -150,7 +153,7 @@ class Plugin(PluginInterface): ) # Fetch and parse post detail page - if detail_page_url: + if detail_page_url and self.crawl_detail_page: # Fetch self.log(f"scraping detail page: {detail_page_url}") detail_page_resp = self.get_url(session, url=detail_page_url) -- cgit v1.3