[improvement][web] not using HTTP sessions anymore to perform the searches

CHANGELOG.md

@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [0.8.1]
+### Improvements
+- Web:
+    - not using HTTP sessions anymore to perform the searches. It seems to avoid URLs not being found after an internet drop event. 
+    
 ### Fixes
 - not verifying if an icon path is a file
 - Web:

bauh/api/http.py

@@ -17,7 +17,7 @@ class HttpClient:
         self.sleep = sleep
         self.logger = logger
 
-    def get(self, url: str, params: dict = None, headers: dict = None, allow_redirects: bool = True, ignore_ssl: bool = False, single_call: bool = False) -> requests.Request:
+    def get(self, url: str, params: dict = None, headers: dict = None, allow_redirects: bool = True, ignore_ssl: bool = False, single_call: bool = False, session: bool = True) -> requests.Response:
         cur_attempts = 1
 
         while cur_attempts <= self.max_attempts:
@@ -35,7 +35,10 @@ class HttpClient:
                 if ignore_ssl:
                     args['verify'] = False
 
-                res = self.session.get(url, **args)
+                if session:
+                    res = self.session.get(url, **args)
+                else:
+                    res = requests.get(url, **args)
 
                 if res.status_code == 200:
                     return res
@@ -56,20 +59,20 @@ class HttpClient:
 
             self.logger.warning("Could not retrieve data from '{}'".format(url))
 
-    def get_json(self, url: str, params: dict = None, headers: dict = None, allow_redirects: bool = True):
+    def get_json(self, url: str, params: dict = None, headers: dict = None, allow_redirects: bool = True, session: bool = True):
-        res = self.get(url, params=params, headers=headers, allow_redirects=allow_redirects)
+        res = self.get(url, params=params, headers=headers, allow_redirects=allow_redirects, session=session)
         return res.json() if res else None
 
-    def get_yaml(self, url: str, params: dict = None, headers: dict = None, allow_redirects: bool = True):
+    def get_yaml(self, url: str, params: dict = None, headers: dict = None, allow_redirects: bool = True, session: bool = True):
-        res = self.get(url, params=params, headers=headers, allow_redirects=allow_redirects)
+        res = self.get(url, params=params, headers=headers, allow_redirects=allow_redirects, session=session)
         return yaml.safe_load(res.text) if res else None
 
-    def get_content_length(self, url: str) -> str:
+    def get_content_length(self, url: str, session: bool = True) -> str:
-        """
+        params = {'url': url, 'allow_redirects': True, 'stream': True}
-        :param url:
+        if session:
-        :return:
+            res = self.session.get(**params)
-        """
+        else:
-        res = self.session.get(url, allow_redirects=True, stream=True)
+            res = requests.get(**params)
 
         if res.status_code == 200:
             size = res.headers.get('Content-Length')
@@ -77,6 +80,11 @@ class HttpClient:
             if size is not None:
                 return system.get_human_size_str(size)
 
-    def exists(self, url: str) -> bool:
+    def exists(self, url: str, session: bool = True) -> bool:
-        res = self.session.head(url=url, allow_redirects=True, verify=False, timeout=5)
+        params = {'url': url, 'allow_redirects': True, 'verify': False, 'timeout': 5}
+        if session:
+            res = self.session.head(**params)
+        else:
+            res = self.session.get(**params)
+
         return res.status_code in (200, 403)

bauh/gems/web/controller.py

@@ -137,10 +137,10 @@ class WebApplicationManager(SoftwareManager):
         return description
 
     def _get_fix_for(self, url_no_protocol: str) -> str:
-        try:
+        fix_url = URL_FIX_PATTERN.format(url=url_no_protocol)
-            fix_url = URL_FIX_PATTERN.format(url=url_no_protocol)
-            res = self.http_client.get(fix_url)
 
+        try:
+            res = self.http_client.get(fix_url, session=False)
             if res:
                 return res.text
         except Exception as e:
@@ -156,7 +156,7 @@ class WebApplicationManager(SoftwareManager):
         headers = {'Accept-language': self._get_lang_header(), 'User-Agent': UA_CHROME}
 
         try:
-            url_res = self.http_client.get(url, headers=headers, ignore_ssl=True, single_call=True)
+            url_res = self.http_client.get(url, headers=headers, ignore_ssl=True, single_call=True, session=False)
 
             if url_res:
                 return BeautifulSoup(url_res.text, 'lxml', parse_only=SoupStrainer('head'))
@@ -529,11 +529,11 @@ class WebApplicationManager(SoftwareManager):
 
     def _download_suggestion_icon(self, pkg: WebApplication, app_dir: str) -> Tuple[str, bytes]:
         try:
-            if self.http_client.exists(pkg.icon_url):
+            if self.http_client.exists(pkg.icon_url, session=False):
                 icon_path = '{}/{}'.format(app_dir, pkg.icon_url.split('/')[-1])
 
                 try:
-                    res = self.http_client.get(pkg.icon_url)
+                    res = self.http_client.get(pkg.icon_url, session=False)
                     if not res:
                         self.logger.info('Could not download the icon {}'.format(pkg.icon_url))
                     else:
@@ -756,7 +756,7 @@ class WebApplicationManager(SoftwareManager):
             if not app.description:
                 app.description = self._get_app_description(app.url, soup)
 
-            find_url = not app.icon_url or (app.icon_url and not self.http_client.exists(app.icon_url))
+            find_url = not app.icon_url or (app.icon_url and not self.http_client.exists(app.icon_url, session=False))
 
             if find_url:
                 app.icon_url = self._get_app_icon_url(app.url, soup)

bauh/gems/web/worker.py

@@ -19,7 +19,7 @@ class SuggestionsDownloader:
     def download(self) -> dict:
         self.logger.info("Reading suggestions from {}".format(URL_SUGGESTIONS))
         try:
-            suggestions = self.http_client.get_yaml(URL_SUGGESTIONS)
+            suggestions = self.http_client.get_yaml(URL_SUGGESTIONS, session=False)
 
             if suggestions:
                 self.logger.info("{} suggestions successfully read".format(len(suggestions)))