diff options
Diffstat (limited to 'plugins/ScraperSourcePlugin.py')
| -rw-r--r-- | plugins/ScraperSourcePlugin.py | 19 |
1 files changed, 18 insertions, 1 deletions
diff --git a/plugins/ScraperSourcePlugin.py b/plugins/ScraperSourcePlugin.py index 4a01d19..058dbc9 100644 --- a/plugins/ScraperSourcePlugin.py +++ b/plugins/ScraperSourcePlugin.py @@ -53,8 +53,26 @@ class Plugin(PluginInterface): self.log(f'starting to scrape posts from URL "{self.url}"') + headers = { + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/114.0", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + "DNT": "1", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + "Pragma": "no-cache", + "Cache-Control": "no-cache", + "TE": "trailers", + } + result_items: list[Item] = [] with requests.session() as session: + session.headers.update(headers) + page_resp = session.get(self.url, allow_redirects=True) page_elem = BeautifulSoup(page_resp.text, "html.parser") post_elems = eval(self.selector_post, {"page": page_elem}) @@ -65,7 +83,6 @@ class Plugin(PluginInterface): self.selector_link, {"page": page_elem, "post": post_elem} )[0] detail_page_url = self.absolute_link(link_elem["href"]) - else: detail_page_url = None |