diff options
| author | Jan Tuomi <jans.tuomi@gmail.com> | 2023-09-13 16:17:10 +0300 |
|---|---|---|
| committer | Jan Tuomi <jans.tuomi@gmail.com> | 2023-09-13 16:17:10 +0300 |
| commit | 82901e29439c2ddd29a20ad993db8fa1ec918d9c (patch) | |
| tree | 3426e0cffab08d8b86e99fe9752feb2ff1a9d5bc | |
| parent | fa2c9120c47ce61f61444c62fda637f3ed2678ed (diff) | |
Add doctype to html, browser-like headers to scraper
| -rw-r--r-- | app/server.py | 1 | ||||
| -rw-r--r-- | plugins/ScraperSourcePlugin.py | 19 |
2 files changed, 19 insertions, 1 deletions
diff --git a/app/server.py b/app/server.py index 038760e..8e90dea 100644 --- a/app/server.py +++ b/app/server.py @@ -41,6 +41,7 @@ def index(): bottle.response.set_header("content-type", "text/html") page = f""" + <!doctype html> <html lang="en"> <head> <title>Aggro – Feed manipulator</title> diff --git a/plugins/ScraperSourcePlugin.py b/plugins/ScraperSourcePlugin.py index 4a01d19..058dbc9 100644 --- a/plugins/ScraperSourcePlugin.py +++ b/plugins/ScraperSourcePlugin.py @@ -53,8 +53,26 @@ class Plugin(PluginInterface): self.log(f'starting to scrape posts from URL "{self.url}"') + headers = { + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/114.0", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + "DNT": "1", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + "Pragma": "no-cache", + "Cache-Control": "no-cache", + "TE": "trailers", + } + result_items: list[Item] = [] with requests.session() as session: + session.headers.update(headers) + page_resp = session.get(self.url, allow_redirects=True) page_elem = BeautifulSoup(page_resp.text, "html.parser") post_elems = eval(self.selector_post, {"page": page_elem}) @@ -65,7 +83,6 @@ class Plugin(PluginInterface): self.selector_link, {"page": page_elem, "post": post_elem} )[0] detail_page_url = self.absolute_link(link_elem["href"]) - else: detail_page_url = None |