SSLError: HTTPSConnectionPool

[andreanuzzo]

Hello,

I was trying to run similarity searches by using:

import pandas as pd
import numpy as np
from chembl_webresource_client.new_client import new_client
from rdkit import Chem
from rdkit.Chem import Draw 
from time import sleep

[...]

for smile in SMILES:
  tmp = pd.DataFrame()
  tmp['SMILES'] = smile
  records = similarity.filter(smiles=smile, similarity=70)
  [...]
  hmp2_hmdb_chembl = pd.concat([hmp2_hmdb_chembl, tmp]
  sleep(10)

Unfortunately I ran across the dreaded urllib3 error SSLError: HTTPSConnectionPool. Now I cannot even run from chembl_webresource_client.new_client import new_client that I got this error thrown.

Error                                     Traceback (most recent call last)
/home/cdsw/.local/lib/python3.6/site-packages/urllib3/contrib/pyopenssl.py in wrap_socket(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname)
    484             try:
--> 485                 cnx.do_handshake()
    486             except OpenSSL.SSL.WantReadError:

/home/cdsw/.local/lib/python3.6/site-packages/OpenSSL/SSL.py in do_handshake(self)
   1914         result = _lib.SSL_do_handshake(self._ssl)
-> 1915         self._raise_ssl_error(self._ssl, result)
   1916 

/home/cdsw/.local/lib/python3.6/site-packages/OpenSSL/SSL.py in _raise_ssl_error(self, ssl, result)
   1646         else:
-> 1647             _raise_current_error()
   1648 

/home/cdsw/.local/lib/python3.6/site-packages/OpenSSL/_util.py in exception_from_error_queue(exception_type)
     53 
---> 54     raise exception_type(errors)
     55 

Error: [('SSL routines', 'tls_process_server_certificate', 'certificate verify failed')]

During handling of the above exception, another exception occurred:

SSLError                                  Traceback (most recent call last)
/home/cdsw/.local/lib/python3.6/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    661             if is_new_proxy_conn:
--> 662                 self._prepare_proxy(conn)
    663 

/home/cdsw/.local/lib/python3.6/site-packages/urllib3/connectionpool.py in _prepare_proxy(self, conn)
    947         conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers)
--> 948         conn.connect()
    949 

/home/cdsw/.local/lib/python3.6/site-packages/urllib3/connection.py in connect(self)
    393             server_hostname=server_hostname,
--> 394             ssl_context=context,
    395         )

/home/cdsw/.local/lib/python3.6/site-packages/urllib3/util/ssl_.py in ssl_wrap_socket(sock, keyfile, certfile, cert_reqs, ca_certs, server_hostname, ssl_version, ciphers, ssl_context, ca_cert_dir, key_password)
    369         if HAS_SNI and server_hostname is not None:
--> 370             return context.wrap_socket(sock, server_hostname=server_hostname)
    371 

/home/cdsw/.local/lib/python3.6/site-packages/urllib3/contrib/pyopenssl.py in wrap_socket(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname)
    490             except OpenSSL.SSL.Error as e:
--> 491                 raise ssl.SSLError("bad handshake: %r" % e)
    492             break

SSLError: ("bad handshake: Error([('SSL routines', 'tls_process_server_certificate', 'certificate verify failed')],)",)

During handling of the above exception, another exception occurred:

MaxRetryError                             Traceback (most recent call last)
/home/cdsw/.local/lib/python3.6/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
    448                     retries=self.max_retries,
--> 449                     timeout=timeout
    450                 )

/home/cdsw/.local/lib/python3.6/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    719             retries = retries.increment(
--> 720                 method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
    721             )

/home/cdsw/.local/lib/python3.6/site-packages/urllib3/util/retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
    435         if new_retry.is_exhausted():
--> 436             raise MaxRetryError(_pool, url, error or ResponseError(cause))
    437 

MaxRetryError: HTTPSConnectionPool(host='www.ebi.ac.uk', port=443): Max retries exceeded with url: /chembl/api/data/spore (Caused by SSLError(SSLError("bad handshake: Error([('SSL routines', 'tls_process_server_certificate', 'certificate verify failed')],)",),))

During handling of the above exception, another exception occurred:

SSLError                                  Traceback (most recent call last)
in engine
----> 1 from chembl_webresource_client.new_client import new_client

/home/cdsw/.local/lib/python3.6/site-packages/chembl_webresource_client/new_client.py in <module>()
     65 #-----------------------------------------------------------------------------------------------------------------------
     66 
---> 67 new_client = client_from_url(Settings.Instance().NEW_CLIENT_URL + '/spore')
     68 
     69 #-----------------------------------------------------------------------------------------------------------------------

/home/cdsw/.local/lib/python3.6/site-packages/chembl_webresource_client/new_client.py in client_from_url(url, base_url)
     25 
     26     """
---> 27     res = requests.get(url)
     28     if not res.ok:
     29         raise Exception('Error getting schema from url {0} with status {1} and msg {2}'.format(url, res.status_code, res.text))

/home/cdsw/.local/lib/python3.6/site-packages/requests/api.py in get(url, params, **kwargs)
     73 
     74     kwargs.setdefault('allow_redirects', True)
---> 75     return request('get', url, params=params, **kwargs)
     76 
     77 

/home/cdsw/.local/lib/python3.6/site-packages/requests/api.py in request(method, url, **kwargs)
     58     # cases, and look like a memory leak in others.
     59     with sessions.Session() as session:
---> 60         return session.request(method=method, url=url, **kwargs)
     61 
     62 

/home/cdsw/.local/lib/python3.6/site-packages/requests/sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
    531         }
    532         send_kwargs.update(settings)
--> 533         resp = self.send(prep, **send_kwargs)
    534 
    535         return resp

/home/cdsw/.local/lib/python3.6/site-packages/requests/sessions.py in send(self, request, **kwargs)
    644 
    645         # Send the request
--> 646         r = adapter.send(request, **kwargs)
    647 
    648         # Total elapsed time of the request (approximately)

/home/cdsw/.local/lib/python3.6/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
    512             if isinstance(e.reason, _SSLError):
    513                 # This branch is for urllib3 v1.22 and later.
--> 514                 raise SSLError(e, request=request)
    515 
    516             raise ConnectionError(e, request=request)

SSLError: HTTPSConnectionPool(host='www.ebi.ac.uk', port=443): Max retries exceeded with url: /chembl/api/data/spore (Caused by SSLError(SSLError("bad handshake: Error([('SSL routines', 'tls_process_server_certificate', 'certificate verify failed')],)",),))

I have used the same script months ago but without the similarity search (only identity search via SMILES string). Any idea on how to fix it? Rebooting, spawn new sessions or reinstalling urllib3 did not help. Similar to another thread I saw here, but in my case just waiting for a couple of days did not solve either.

Thanks

[andreanuzzo]

Discovered some temporary files in the /home folder from sqlite. Deleting them solves mitigates the issue. I am able to rerun the script and import the web service, but I still get the error after a while. What is the limit of requests?