From 0acb5b7047f5687a93c8e124c8e3a13cd57746b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Moreira?= Date: Wed, 8 Jan 2020 15:48:14 -0300 Subject: [PATCH] [improvement][web] not using HTTP sessions anymore to perform the searches --- CHANGELOG.md | 4 ++++ bauh/api/http.py | 36 ++++++++++++++++++++++-------------- bauh/gems/web/controller.py | 14 +++++++------- bauh/gems/web/worker.py | 2 +- 4 files changed, 34 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea6391eb..fecbd3a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [0.8.1] +### Improvements +- Web: + - not using HTTP sessions anymore to perform the searches. It seems to avoid URLs not being found after an internet drop event. + ### Fixes - not verifying if an icon path is a file - Web: diff --git a/bauh/api/http.py b/bauh/api/http.py index 31e90c90..2a1c91be 100644 --- a/bauh/api/http.py +++ b/bauh/api/http.py @@ -17,7 +17,7 @@ class HttpClient: self.sleep = sleep self.logger = logger - def get(self, url: str, params: dict = None, headers: dict = None, allow_redirects: bool = True, ignore_ssl: bool = False, single_call: bool = False) -> requests.Request: + def get(self, url: str, params: dict = None, headers: dict = None, allow_redirects: bool = True, ignore_ssl: bool = False, single_call: bool = False, session: bool = True) -> requests.Response: cur_attempts = 1 while cur_attempts <= self.max_attempts: @@ -35,7 +35,10 @@ class HttpClient: if ignore_ssl: args['verify'] = False - res = self.session.get(url, **args) + if session: + res = self.session.get(url, **args) + else: + res = requests.get(url, **args) if res.status_code == 200: return res @@ -56,20 +59,20 @@ class HttpClient: self.logger.warning("Could not retrieve data from '{}'".format(url)) - def get_json(self, url: str, params: dict = None, headers: dict = None, allow_redirects: bool = True): - res = self.get(url, params=params, headers=headers, allow_redirects=allow_redirects) + def get_json(self, url: str, params: dict = None, headers: dict = None, allow_redirects: bool = True, session: bool = True): + res = self.get(url, params=params, headers=headers, allow_redirects=allow_redirects, session=session) return res.json() if res else None - def get_yaml(self, url: str, params: dict = None, headers: dict = None, allow_redirects: bool = True): - res = self.get(url, params=params, headers=headers, allow_redirects=allow_redirects) + def get_yaml(self, url: str, params: dict = None, headers: dict = None, allow_redirects: bool = True, session: bool = True): + res = self.get(url, params=params, headers=headers, allow_redirects=allow_redirects, session=session) return yaml.safe_load(res.text) if res else None - def get_content_length(self, url: str) -> str: - """ - :param url: - :return: - """ - res = self.session.get(url, allow_redirects=True, stream=True) + def get_content_length(self, url: str, session: bool = True) -> str: + params = {'url': url, 'allow_redirects': True, 'stream': True} + if session: + res = self.session.get(**params) + else: + res = requests.get(**params) if res.status_code == 200: size = res.headers.get('Content-Length') @@ -77,6 +80,11 @@ class HttpClient: if size is not None: return system.get_human_size_str(size) - def exists(self, url: str) -> bool: - res = self.session.head(url=url, allow_redirects=True, verify=False, timeout=5) + def exists(self, url: str, session: bool = True) -> bool: + params = {'url': url, 'allow_redirects': True, 'verify': False, 'timeout': 5} + if session: + res = self.session.head(**params) + else: + res = self.session.get(**params) + return res.status_code in (200, 403) diff --git a/bauh/gems/web/controller.py b/bauh/gems/web/controller.py index e45f9df7..0e1d9c14 100644 --- a/bauh/gems/web/controller.py +++ b/bauh/gems/web/controller.py @@ -137,10 +137,10 @@ class WebApplicationManager(SoftwareManager): return description def _get_fix_for(self, url_no_protocol: str) -> str: - try: - fix_url = URL_FIX_PATTERN.format(url=url_no_protocol) - res = self.http_client.get(fix_url) + fix_url = URL_FIX_PATTERN.format(url=url_no_protocol) + try: + res = self.http_client.get(fix_url, session=False) if res: return res.text except Exception as e: @@ -156,7 +156,7 @@ class WebApplicationManager(SoftwareManager): headers = {'Accept-language': self._get_lang_header(), 'User-Agent': UA_CHROME} try: - url_res = self.http_client.get(url, headers=headers, ignore_ssl=True, single_call=True) + url_res = self.http_client.get(url, headers=headers, ignore_ssl=True, single_call=True, session=False) if url_res: return BeautifulSoup(url_res.text, 'lxml', parse_only=SoupStrainer('head')) @@ -529,11 +529,11 @@ class WebApplicationManager(SoftwareManager): def _download_suggestion_icon(self, pkg: WebApplication, app_dir: str) -> Tuple[str, bytes]: try: - if self.http_client.exists(pkg.icon_url): + if self.http_client.exists(pkg.icon_url, session=False): icon_path = '{}/{}'.format(app_dir, pkg.icon_url.split('/')[-1]) try: - res = self.http_client.get(pkg.icon_url) + res = self.http_client.get(pkg.icon_url, session=False) if not res: self.logger.info('Could not download the icon {}'.format(pkg.icon_url)) else: @@ -756,7 +756,7 @@ class WebApplicationManager(SoftwareManager): if not app.description: app.description = self._get_app_description(app.url, soup) - find_url = not app.icon_url or (app.icon_url and not self.http_client.exists(app.icon_url)) + find_url = not app.icon_url or (app.icon_url and not self.http_client.exists(app.icon_url, session=False)) if find_url: app.icon_url = self._get_app_icon_url(app.url, soup) diff --git a/bauh/gems/web/worker.py b/bauh/gems/web/worker.py index c6b68d6e..48182222 100644 --- a/bauh/gems/web/worker.py +++ b/bauh/gems/web/worker.py @@ -19,7 +19,7 @@ class SuggestionsDownloader: def download(self) -> dict: self.logger.info("Reading suggestions from {}".format(URL_SUGGESTIONS)) try: - suggestions = self.http_client.get_yaml(URL_SUGGESTIONS) + suggestions = self.http_client.get_yaml(URL_SUGGESTIONS, session=False) if suggestions: self.logger.info("{} suggestions successfully read".format(len(suggestions)))