spalencsar/bearhub
1
0
Fork 0
mirror of https://github.com/spalencsar/bearhub.git synced 2026-07-07 01:14:15 +02:00
Code Issues Packages Projects Releases Wiki Activity

[improvement][web] not using HTTP sessions anymore to perform the searches

Browse Source
This commit is contained in:
Vinícius Moreira
2020-01-08 15:48:14 -03:00
parent 74ddfc03ef
commit 0acb5b7047
4 changed files with 34 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
## [0.8.1]
### Improvements
- Web:
- not using HTTP sessions anymore to perform the searches. It seems to avoid URLs not being found after an internet drop event.
### Fixes
- not verifying if an icon path is a file
- Web:

36
bauh/api/http.py
View File

@@ -17,7 +17,7 @@ class HttpClient:
self.sleep = sleep
self.logger = logger
def get(self, url: str, params: dict = None, headers: dict = None, allow_redirects: bool = True, ignore_ssl: bool = False, single_call: bool = False) -> requests.Request:
def get(self, url: str, params: dict = None, headers: dict = None, allow_redirects: bool = True, ignore_ssl: bool = False, single_call: bool = False, session: bool = True) -> requests.Response:
cur_attempts = 1
while cur_attempts <= self.max_attempts:
@@ -35,7 +35,10 @@ class HttpClient:
if ignore_ssl:
args['verify'] = False
res = self.session.get(url, **args)
if session:
res = self.session.get(url, **args)
else:
res = requests.get(url, **args)
if res.status_code == 200:
return res
@@ -56,20 +59,20 @@ class HttpClient:
self.logger.warning("Could not retrieve data from '{}'".format(url))
def get_json(self, url: str, params: dict = None, headers: dict = None, allow_redirects: bool = True):
res = self.get(url, params=params, headers=headers, allow_redirects=allow_redirects)
def get_json(self, url: str, params: dict = None, headers: dict = None, allow_redirects: bool = True, session: bool = True):
res = self.get(url, params=params, headers=headers, allow_redirects=allow_redirects, session=session)
return res.json() if res else None
def get_yaml(self, url: str, params: dict = None, headers: dict = None, allow_redirects: bool = True):
res = self.get(url, params=params, headers=headers, allow_redirects=allow_redirects)
def get_yaml(self, url: str, params: dict = None, headers: dict = None, allow_redirects: bool = True, session: bool = True):
res = self.get(url, params=params, headers=headers, allow_redirects=allow_redirects, session=session)
return yaml.safe_load(res.text) if res else None
def get_content_length(self, url: str) -> str:
"""
:param url:
:return:
"""
res = self.session.get(url, allow_redirects=True, stream=True)
def get_content_length(self, url: str, session: bool = True) -> str:
params = {'url': url, 'allow_redirects': True, 'stream': True}
if session:
res = self.session.get(**params)
else:
res = requests.get(**params)
if res.status_code == 200:
size = res.headers.get('Content-Length')
@@ -77,6 +80,11 @@ class HttpClient:
if size is not None:
return system.get_human_size_str(size)
def exists(self, url: str) -> bool:
res = self.session.head(url=url, allow_redirects=True, verify=False, timeout=5)
def exists(self, url: str, session: bool = True) -> bool:
params = {'url': url, 'allow_redirects': True, 'verify': False, 'timeout': 5}
if session:
res = self.session.head(**params)
else:
res = self.session.get(**params)
return res.status_code in (200, 403)

14
bauh/gems/web/controller.py
View File

@@ -137,10 +137,10 @@ class WebApplicationManager(SoftwareManager):
return description
def _get_fix_for(self, url_no_protocol: str) -> str:
try:
fix_url = URL_FIX_PATTERN.format(url=url_no_protocol)
res = self.http_client.get(fix_url)
fix_url = URL_FIX_PATTERN.format(url=url_no_protocol)
try:
res = self.http_client.get(fix_url, session=False)
if res:
return res.text
except Exception as e:
@@ -156,7 +156,7 @@ class WebApplicationManager(SoftwareManager):
headers = {'Accept-language': self._get_lang_header(), 'User-Agent': UA_CHROME}
try:
url_res = self.http_client.get(url, headers=headers, ignore_ssl=True, single_call=True)
url_res = self.http_client.get(url, headers=headers, ignore_ssl=True, single_call=True, session=False)
if url_res:
return BeautifulSoup(url_res.text, 'lxml', parse_only=SoupStrainer('head'))
@@ -529,11 +529,11 @@ class WebApplicationManager(SoftwareManager):
def _download_suggestion_icon(self, pkg: WebApplication, app_dir: str) -> Tuple[str, bytes]:
try:
if self.http_client.exists(pkg.icon_url):
if self.http_client.exists(pkg.icon_url, session=False):
icon_path = '{}/{}'.format(app_dir, pkg.icon_url.split('/')[-1])
try:
res = self.http_client.get(pkg.icon_url)
res = self.http_client.get(pkg.icon_url, session=False)
if not res:
self.logger.info('Could not download the icon {}'.format(pkg.icon_url))
else:
@@ -756,7 +756,7 @@ class WebApplicationManager(SoftwareManager):
if not app.description:
app.description = self._get_app_description(app.url, soup)
find_url = not app.icon_url or (app.icon_url and not self.http_client.exists(app.icon_url))
find_url = not app.icon_url or (app.icon_url and not self.http_client.exists(app.icon_url, session=False))
if find_url:
app.icon_url = self._get_app_icon_url(app.url, soup)

2
bauh/gems/web/worker.py
View File

@@ -19,7 +19,7 @@ class SuggestionsDownloader:
def download(self) -> dict:
self.logger.info("Reading suggestions from {}".format(URL_SUGGESTIONS))
try:
suggestions = self.http_client.get_yaml(URL_SUGGESTIONS)
suggestions = self.http_client.get_yaml(URL_SUGGESTIONS, session=False)
if suggestions:
self.logger.info("{} suggestions successfully read".format(len(suggestions)))