Skip to content

Commit 04676b5

Browse files
authored
[sftp] Improve write & memory performance when saving files (#1194)
* Move seekability test to a separate helper function Having a third copy of this code would feel silly. * Stop calling content.open() from SFTPStorage._save The open method isn't available in all file-like objects one might want to save using the storage framework, so instead of relying on it calling seek(0) for us do it ourselves (conditionally). * Require at least version 1.10.0 of paramiko This version introduced the SFTPClient.putfo() method. It was released in early 2013, so this shouldn't be a great hardship. * Use SFTPClient.putfo() in SFTPStorage._save() Instead of using SFTPClient.open() to get a file-like object, reading the entire content into memory and then calling .write() on that. The measured performance difference is stunning, and the uploaded file data no longer needs to fit into memory.
1 parent b04de35 commit 04676b5

File tree

6 files changed

+22
-10
lines changed

6 files changed

+22
-10
lines changed

setup.cfg

+1-1
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ google =
4646
libcloud =
4747
apache-libcloud
4848
sftp =
49-
paramiko
49+
paramiko >= 1.10.0
5050

5151
[flake8]
5252
exclude =

storages/backends/gcloud.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
from storages.utils import check_location
1616
from storages.utils import clean_name
1717
from storages.utils import get_available_overwrite_name
18+
from storages.utils import is_seekable
1819
from storages.utils import safe_join
1920
from storages.utils import setting
2021
from storages.utils import to_bytes
@@ -194,7 +195,7 @@ def _save(self, name, content):
194195
for prop, val in blob_params.items():
195196
setattr(file_object.blob, prop, val)
196197

197-
rewind = not hasattr(content, 'seekable') or content.seekable()
198+
rewind = is_seekable(content)
198199
file_object.blob.upload_from_file(
199200
content,
200201
rewind=rewind,

storages/backends/s3boto3.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
from storages.compress import CompressStorageMixin
2525
from storages.utils import check_location
2626
from storages.utils import get_available_overwrite_name
27+
from storages.utils import is_seekable
2728
from storages.utils import lookup_env
2829
from storages.utils import safe_join
2930
from storages.utils import setting
@@ -445,7 +446,7 @@ def _save(self, name, content):
445446
name = self._normalize_name(cleaned_name)
446447
params = self._get_write_parameters(name, content)
447448

448-
if not hasattr(content, 'seekable') or content.seekable():
449+
if is_seekable(content):
449450
content.seek(0, os.SEEK_SET)
450451
if (self.gzip and
451452
params['ContentType'] in self.gzip_content_types and

storages/backends/sftpstorage.py

+4-4
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
from django.utils.deconstruct import deconstructible
1818

1919
from storages.base import BaseStorage
20+
from storages.utils import is_seekable
2021
from storages.utils import setting
2122

2223

@@ -123,15 +124,14 @@ def _mkdir(self, path):
123124

124125
def _save(self, name, content):
125126
"""Save file via SFTP."""
126-
content.open()
127+
if is_seekable(content):
128+
content.seek(0, os.SEEK_SET)
127129
path = self._remote_path(name)
128130
dirname = posixpath.dirname(path)
129131
if not self.exists(dirname):
130132
self._mkdir(dirname)
131133

132-
f = self.sftp.open(path, 'wb')
133-
f.write(content.file.read())
134-
f.close()
134+
self.sftp.putfo(content, path)
135135

136136
# set file permissions if configured
137137
if self._file_mode is not None:

storages/utils.py

+4
Original file line numberDiff line numberDiff line change
@@ -125,3 +125,7 @@ def get_available_overwrite_name(name, max_length):
125125
'allows sufficient "max_length".' % name
126126
)
127127
return os.path.join(dir_name, "{}{}".format(file_root, file_ext))
128+
129+
130+
def is_seekable(file_object):
131+
return not hasattr(file_object, 'seekable') or file_object.seekable()

tests/test_sftp.py

+9-3
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
from django.test import override_settings
1313

1414
from storages.backends import sftpstorage
15+
from tests.utils import NonSeekableContentFile
1516

1617

1718
class SFTPStorageTest(TestCase):
@@ -69,15 +70,20 @@ def test_mkdir_parent(self, mock_sftp):
6970
@patch('storages.backends.sftpstorage.SFTPStorage.sftp')
7071
def test_save(self, mock_sftp):
7172
self.storage._save('foo', File(io.BytesIO(b'foo'), 'foo'))
72-
self.assertTrue(mock_sftp.open.return_value.write.called)
73+
self.assertTrue(mock_sftp.putfo.called)
74+
75+
@patch('storages.backends.sftpstorage.SFTPStorage.sftp')
76+
def test_save_non_seekable(self, mock_sftp):
77+
self.storage._save('foo', NonSeekableContentFile('foo'))
78+
self.assertTrue(mock_sftp.putfo.called)
7379

7480
@patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{
7581
'stat.side_effect': (FileNotFoundError(), True)
7682
})
7783
def test_save_in_subdir(self, mock_sftp):
7884
self.storage._save('bar/foo', File(io.BytesIO(b'foo'), 'foo'))
7985
self.assertEqual(mock_sftp.mkdir.call_args_list[0][0], ('bar',))
80-
self.assertTrue(mock_sftp.open.return_value.write.called)
86+
self.assertTrue(mock_sftp.putfo.called)
8187

8288
@patch('storages.backends.sftpstorage.SFTPStorage.sftp')
8389
def test_delete(self, mock_sftp):
@@ -212,4 +218,4 @@ def test_write(self):
212218
def test_close(self, mock_sftp):
213219
self.file.write(b'foo')
214220
self.file.close()
215-
self.assertTrue(mock_sftp.open.return_value.write.called)
221+
self.assertTrue(mock_sftp.putfo.called)

0 commit comments

Comments
 (0)