| 1234567891011121314151617181920212223242526272829303132333435363738394041 |
- """ s3 support for remote file interactivity """
- from pandas import compat
- try:
- import s3fs
- from botocore.exceptions import NoCredentialsError
- except ImportError:
- raise ImportError("The s3fs library is required to handle s3 files")
- if compat.PY3:
- from urllib.parse import urlparse as parse_url
- else:
- from urlparse import urlparse as parse_url
- def _strip_schema(url):
- """Returns the url without the s3:// part"""
- result = parse_url(url)
- return result.netloc + result.path
- def get_filepath_or_buffer(filepath_or_buffer, encoding=None,
- compression=None, mode=None):
- if mode is None:
- mode = 'rb'
- fs = s3fs.S3FileSystem(anon=False)
- try:
- filepath_or_buffer = fs.open(_strip_schema(filepath_or_buffer), mode)
- except (compat.FileNotFoundError, NoCredentialsError):
- # boto3 has troubles when trying to access a public file
- # when credentialed...
- # An OSError is raised if you have credentials, but they
- # aren't valid for that bucket.
- # A NoCredentialsError is raised if you don't have creds
- # for that bucket.
- fs = s3fs.S3FileSystem(anon=True)
- filepath_or_buffer = fs.open(_strip_schema(filepath_or_buffer), mode)
- return filepath_or_buffer, None, compression, True
|
