From beb4f8c8f9f3cc9a1c2d9db9bc05afe4835b1b32 Mon Sep 17 00:00:00 2001 From: Calum Lind Date: Wed, 22 Feb 2017 12:53:20 +0000 Subject: [PATCH] [Common] Rename decode_string to decode_bytes - Switch to using Python 3 naming convention where str now refers to unicode and bytes are encoded strings. - Cleanup docs and code - Also rename convert_to_utf8 to utf8_encode_structure to clarify functionality. --- deluge/common.py | 45 +++++++++++++------------- deluge/config.py | 4 +-- deluge/core/alertmanager.py | 4 +-- deluge/core/torrent.py | 14 ++++---- deluge/core/torrentmanager.py | 24 +++++++------- deluge/httpdownloader.py | 4 +-- deluge/maketorrent.py | 4 +-- deluge/tests/test_torrent.py | 4 +-- deluge/tests/test_ui_entry.py | 4 +-- deluge/tests/test_webserver.py | 4 +-- deluge/ui/common.py | 10 +++--- deluge/ui/console/modes/addtorrents.py | 2 +- deluge/ui/gtkui/ipcinterface.py | 4 +-- deluge/ui/gtkui/preferences.py | 2 +- 14 files changed, 64 insertions(+), 65 deletions(-) diff --git a/deluge/common.py b/deluge/common.py index 7aea9b6af..048d48142 100644 --- a/deluge/common.py +++ b/deluge/common.py @@ -777,36 +777,36 @@ def is_ipv6(ip): return True else: try: - return ipaddress.IPv6Address(decode_string(ip)) + return ipaddress.IPv6Address(decode_bytes(ip)) except ipaddress.AddressValueError: pass return False -def decode_string(s, encoding='utf8'): - """ - Decodes a string and return unicode. If it cannot decode using - `:param:encoding` then it will try latin1, and if that fails, - try to detect the string encoding. If that fails, decode with - ignore. +def decode_bytes(byte_str, encoding='utf8'): + """Decodes a byte string and return unicode. - :param s: string to decode - :type s: string - :param encoding: the encoding to use in the decoding - :type encoding: string - :returns: s converted to unicode - :rtype: unicode + If it cannot decode using `encoding` then it will try latin1, + and if that fails, try to detect the string encoding. If that fails, + decode with ignore. + + Args: + byte_str (bytes): The byte string to decode. + encoding (str): The encoding to try first when decoding. + + Returns: + str: A unicode string. """ - if not s: + if not byte_str: return '' - elif isinstance(s, unicode): - return s + elif isinstance(byte_str, unicode): + return byte_str encodings = [lambda: ('utf8', 'strict'), lambda: ('iso-8859-1', 'strict'), - lambda: (chardet.detect(s)['encoding'], 'strict'), + lambda: (chardet.detect(byte_str)['encoding'], 'strict'), lambda: (encoding, 'ignore')] if encoding is not 'utf8': @@ -814,7 +814,7 @@ def decode_string(s, encoding='utf8'): for l in encodings: try: - return s.decode(*l()) + return byte_str.decode(*l()) except UnicodeDecodeError: pass return '' @@ -833,13 +833,13 @@ def utf8_encoded(s, encoding='utf8'): """ if isinstance(s, str): - s = decode_string(s, encoding).encode('utf8') + s = decode_bytes(s, encoding).encode('utf8') elif isinstance(s, unicode): s = s.encode('utf8') return s -def convert_to_utf8(data): +def utf8_encode_structure(data): """Recursively convert all unicode keys and values in a data structure to utf8. e.g. converting keys and values for a dict with nested dicts and lists etc. @@ -851,13 +851,12 @@ def convert_to_utf8(data): input type: The data with unicode keys and values converted to utf8. """ - if isinstance(data, unicode): return data.encode('utf8') elif isinstance(data, (list, tuple)): - return type(data)(map(convert_to_utf8, data)) + return type(data)(map(utf8_encode_structure, data)) elif isinstance(data, dict): - return dict(map(convert_to_utf8, data.items())) + return dict(map(utf8_encode_structure, data.items())) else: return data diff --git a/deluge/config.py b/deluge/config.py index a9d0dada8..a5bfa01c0 100644 --- a/deluge/config.py +++ b/deluge/config.py @@ -47,7 +47,7 @@ import logging import os import shutil -from deluge.common import decode_string, get_default_config_dir, utf8_encoded +from deluge.common import decode_bytes, get_default_config_dir, utf8_encoded log = logging.getLogger(__name__) callLater = None # Necessary for the config tests @@ -245,7 +245,7 @@ class Config(object): """ if isinstance(self.__config[key], basestring): - return decode_string(self.__config[key]) + return decode_bytes(self.__config[key]) else: return self.__config[key] diff --git a/deluge/core/alertmanager.py b/deluge/core/alertmanager.py index 7ebaecd65..ed17341a0 100644 --- a/deluge/core/alertmanager.py +++ b/deluge/core/alertmanager.py @@ -23,7 +23,7 @@ from twisted.internet import reactor import deluge.component as component from deluge._libtorrent import lt -from deluge.common import decode_string +from deluge.common import decode_bytes log = logging.getLogger(__name__) @@ -112,7 +112,7 @@ class AlertManager(component.Component): alert_type = type(alert).__name__ # Display the alert message if log.isEnabledFor(logging.DEBUG): - log.debug('%s: %s', alert_type, decode_string(alert.message())) + log.debug('%s: %s', alert_type, decode_bytes(alert.message())) # Call any handlers for this alert type if alert_type in self.handlers: for handler in self.handlers[alert_type]: diff --git a/deluge/core/torrent.py b/deluge/core/torrent.py index 1cb777b98..f4b9261fe 100644 --- a/deluge/core/torrent.py +++ b/deluge/core/torrent.py @@ -26,7 +26,7 @@ from twisted.internet.defer import Deferred, DeferredList import deluge.component as component from deluge._libtorrent import lt -from deluge.common import decode_string, utf8_encoded +from deluge.common import decode_bytes, utf8_encoded from deluge.configmanager import ConfigManager, get_config_dir from deluge.core.authmanager import AUTH_LEVEL_ADMIN from deluge.decorators import deprecated @@ -612,7 +612,7 @@ class Torrent(object): self.state = 'Error' # auto-manage status will be reverted upon resuming. self.handle.auto_managed(False) - self.set_status_message(decode_string(status_error)) + self.set_status_message(decode_bytes(status_error)) elif status.moving_storage: self.state = 'Moving' elif not session_paused and status.paused and status.auto_managed: @@ -761,7 +761,7 @@ class Torrent(object): if peer.flags & peer.connecting or peer.flags & peer.handshake: continue - client = decode_string(peer.client) + client = decode_bytes(peer.client) try: country = component.get('Core').geoip_instance.country_code_by_addr(peer.ip[0]) @@ -868,7 +868,7 @@ class Torrent(object): if not self.options['name']: handle_name = self.handle.name() if handle_name: - name = decode_string(handle_name) + name = decode_bytes(handle_name) else: name = self.torrent_id else: @@ -1013,7 +1013,7 @@ class Torrent(object): 'trackers': lambda: self.trackers, 'tracker_status': lambda: self.tracker_status, 'upload_payload_rate': lambda: self.status.upload_payload_rate, - 'comment': lambda: decode_string(self.torrent_info.comment()) if self.has_metadata else '', + 'comment': lambda: decode_bytes(self.torrent_info.comment()) if self.has_metadata else '', 'num_files': lambda: self.torrent_info.num_files() if self.has_metadata else 0, 'num_pieces': lambda: self.torrent_info.num_pieces() if self.has_metadata else 0, 'piece_length': lambda: self.torrent_info.piece_length() if self.has_metadata else 0, @@ -1114,7 +1114,7 @@ class Torrent(object): bool: True if successful, otherwise False """ - dest = decode_string(dest) + dest = decode_bytes(dest) if not os.path.exists(dest): try: @@ -1253,7 +1253,7 @@ class Torrent(object): """ for index, filename in filenames: # Make sure filename is a unicode object - filename = sanitize_filepath(decode_string(filename)) + filename = sanitize_filepath(decode_bytes(filename)) # libtorrent needs unicode object if wstrings are enabled, utf8 bytestring otherwise try: self.handle.rename_file(index, filename) diff --git a/deluge/core/torrentmanager.py b/deluge/core/torrentmanager.py index 3177dee7f..81d075117 100644 --- a/deluge/core/torrentmanager.py +++ b/deluge/core/torrentmanager.py @@ -24,7 +24,7 @@ from twisted.internet.task import LoopingCall import deluge.component as component from deluge._libtorrent import lt -from deluge.common import decode_string, get_magnet_info, utf8_encoded +from deluge.common import decode_bytes, get_magnet_info, utf8_encoded from deluge.configmanager import ConfigManager, get_config_dir from deluge.core.authmanager import AUTH_LEVEL_ADMIN from deluge.core.torrent import Torrent, TorrentOptions, sanitize_filepath @@ -358,7 +358,7 @@ class TorrentManager(component.Component): # Check for renamed files and if so, rename them in the torrent_info before adding. if options['mapped_files'] and torrent_info: for index, fname in options['mapped_files'].items(): - fname = sanitize_filepath(decode_string(fname)) + fname = sanitize_filepath(decode_bytes(fname)) if log.isEnabledFor(logging.DEBUG): log.debug('renaming file index %s to %s', index, fname) try: @@ -1045,7 +1045,7 @@ class TorrentManager(component.Component): except (RuntimeError, KeyError): return # Set the tracker status for the torrent - torrent.set_tracker_status('Warning: %s' % decode_string(alert.message())) + torrent.set_tracker_status('Warning: %s' % decode_bytes(alert.message())) def on_alert_tracker_error(self, alert): """Alert handler for libtorrent tracker_error_alert""" @@ -1054,10 +1054,10 @@ class TorrentManager(component.Component): except (RuntimeError, KeyError): return - error_message = decode_string(alert.error_message()) + error_message = decode_bytes(alert.error_message()) if not error_message: error_message = alert.error.message() - log.debug('Tracker Error Alert: %s [%s]', decode_string(alert.message()), error_message) + log.debug('Tracker Error Alert: %s [%s]', decode_bytes(alert.message()), error_message) torrent.set_tracker_status('Error: ' + error_message) def on_alert_storage_moved(self, alert): @@ -1085,9 +1085,9 @@ class TorrentManager(component.Component): except (RuntimeError, KeyError): return - log.warning('on_alert_storage_moved_failed: %s', decode_string(alert.message())) + log.warning('on_alert_storage_moved_failed: %s', decode_bytes(alert.message())) # Set an Error message and pause the torrent - alert_msg = decode_string(alert.message()).split(':', 1)[1].strip() + alert_msg = decode_bytes(alert.message()).split(':', 1)[1].strip() torrent.force_error_state('Failed to move download folder: %s' % alert_msg) if torrent_id in self.waiting_on_finish_moving: @@ -1145,7 +1145,7 @@ class TorrentManager(component.Component): return if torrent_id in self.waiting_on_resume_data: - self.waiting_on_resume_data[torrent_id].errback(Exception(decode_string(alert.message()))) + self.waiting_on_resume_data[torrent_id].errback(Exception(decode_bytes(alert.message()))) def on_alert_fastresume_rejected(self, alert): """Alert handler for libtorrent fastresume_rejected_alert""" @@ -1155,7 +1155,7 @@ class TorrentManager(component.Component): except (RuntimeError, KeyError): return - alert_msg = decode_string(alert.message()) + alert_msg = decode_bytes(alert.message()) log.error('on_alert_fastresume_rejected: %s', alert_msg) if alert.error.value() == 134: if not os.path.isdir(torrent.options['download_location']): @@ -1179,7 +1179,7 @@ class TorrentManager(component.Component): except (RuntimeError, KeyError): return - new_name = decode_string(alert.new_name) + new_name = decode_bytes(alert.new_name) log.debug('index: %s name: %s', alert.index, new_name) # We need to see if this file index is in a waiting_on_folder dict @@ -1251,13 +1251,13 @@ class TorrentManager(component.Component): 'external IP received: 0:0:0:0:0:0:0:0' """ - external_ip = decode_string(alert.message()).split(' ')[-1] + external_ip = decode_bytes(alert.message()).split(' ')[-1] log.info('on_alert_external_ip: %s', external_ip) component.get('EventManager').emit(ExternalIPEvent(external_ip)) def on_alert_performance(self, alert): """Alert handler for libtorrent performance_alert""" - log.warning('on_alert_performance: %s, %s', decode_string(alert.message()), alert.warning_code) + log.warning('on_alert_performance: %s, %s', decode_bytes(alert.message()), alert.warning_code) if alert.warning_code == lt.performance_warning_t.send_buffer_watermark_too_low: max_send_buffer_watermark = 3 * 1024 * 1024 # 3MiB settings = self.session.get_settings() diff --git a/deluge/httpdownloader.py b/deluge/httpdownloader.py index b3f1ce1a3..10e8ce0ea 100644 --- a/deluge/httpdownloader.py +++ b/deluge/httpdownloader.py @@ -19,7 +19,7 @@ from twisted.python.failure import Failure from twisted.web import client, http from twisted.web.error import PageRedirect -from deluge.common import convert_to_utf8, get_version +from deluge.common import get_version, utf8_encode_structure log = logging.getLogger(__name__) @@ -175,7 +175,7 @@ def _download_file(url, filename, callback=None, headers=None, force_filename=Fa url = url.encode('utf8') filename = filename.encode('utf8') - headers = convert_to_utf8(headers) if headers else headers + headers = utf8_encode_structure(headers) if headers else headers factory = HTTPDownloader(url, filename, callback, headers, force_filename, allow_compression) # In Twisted 13.1.0 _parse() function replaced by _URI class. diff --git a/deluge/maketorrent.py b/deluge/maketorrent.py index 2dcd6e905..82599ac0b 100644 --- a/deluge/maketorrent.py +++ b/deluge/maketorrent.py @@ -14,7 +14,7 @@ import sys from hashlib import sha1 as sha from deluge.bencode import bencode -from deluge.common import convert_to_utf8, get_path_size +from deluge.common import get_path_size, utf8_encode_structure class InvalidPath(Exception): @@ -194,7 +194,7 @@ class TorrentMetadata(object): # Write out the torrent file with open(torrent_path, 'wb') as _file: - _file.write(bencode(convert_to_utf8(torrent))) + _file.write(bencode(utf8_encode_structure(torrent))) def get_data_path(self): """Get the path to the files that the torrent will contain. diff --git a/deluge/tests/test_torrent.py b/deluge/tests/test_torrent.py index db38701d6..680eddd52 100644 --- a/deluge/tests/test_torrent.py +++ b/deluge/tests/test_torrent.py @@ -18,7 +18,7 @@ import deluge.component as component import deluge.core.torrent import deluge.tests.common as common from deluge._libtorrent import lt -from deluge.common import convert_to_utf8 +from deluge.common import utf8_encode_structure from deluge.core.core import Core from deluge.core.rpcserver import RPCServer from deluge.core.torrent import Torrent @@ -180,7 +180,7 @@ class TorrentTestCase(BaseTestCase): filename = common.get_test_data_file('test_torrent.file.torrent') with open(filename) as _file: filedump = _file.read() - resume_data = convert_to_utf8(resume_data) + resume_data = utf8_encode_structure(resume_data) torrent_id = yield self.core.torrentmanager.add(state=torrent_state, filedump=filedump, resume_data=lt.bencode(resume_data)) torrent = self.core.torrentmanager.torrents[torrent_id] diff --git a/deluge/tests/test_ui_entry.py b/deluge/tests/test_ui_entry.py index e5c1c428a..be5c61241 100644 --- a/deluge/tests/test_ui_entry.py +++ b/deluge/tests/test_ui_entry.py @@ -24,7 +24,7 @@ import deluge.ui.console import deluge.ui.console.cmdline.commands.quit import deluge.ui.console.main import deluge.ui.web.server -from deluge.common import convert_to_utf8 +from deluge.common import utf8_encode_structure from deluge.ui import ui_entry from deluge.ui.web.server import DelugeWeb @@ -168,7 +168,7 @@ class GtkUIDelugeScriptEntryTestCase(BaseTestCase, GtkUIBaseTestCase): self.var['cmd_name'] = 'deluge gtk' self.var['start_cmd'] = ui_entry.start_ui - self.var['sys_arg_cmd'] = convert_to_utf8(['./deluge', 'gtk']) + self.var['sys_arg_cmd'] = utf8_encode_structure(['./deluge', 'gtk']) def set_up(self): return GtkUIBaseTestCase.set_up(self) diff --git a/deluge/tests/test_webserver.py b/deluge/tests/test_webserver.py index a77e12b00..a98b4c090 100644 --- a/deluge/tests/test_webserver.py +++ b/deluge/tests/test_webserver.py @@ -18,7 +18,7 @@ from twisted.trial.unittest import SkipTest from twisted.web.client import Agent, FileBodyProducer from twisted.web.http_headers import Headers -from deluge.common import convert_to_utf8 +from deluge.common import utf8_encode_structure from . import common from .common import get_test_data_file @@ -46,7 +46,7 @@ class WebServerTestCase(WebServerTestBase, WebServerMockBase): 'Content-Type': ['application/json']} url = 'http://127.0.0.1:%s/json' % self.webserver_listen_port - d = yield agent.request(b'POST', url.encode('utf-8'), Headers(convert_to_utf8(headers)), + d = yield agent.request(b'POST', url.encode('utf-8'), Headers(utf8_encode_structure(headers)), FileBodyProducer(StringIO(input_file.encode('utf-8')))) try: body = yield twisted.web.client.readBody(d) diff --git a/deluge/ui/common.py b/deluge/ui/common.py index 6b7e812b2..cc5b92838 100644 --- a/deluge/ui/common.py +++ b/deluge/ui/common.py @@ -20,7 +20,7 @@ from hashlib import sha1 as sha import deluge.configmanager from deluge import bencode -from deluge.common import decode_string +from deluge.common import decode_bytes log = logging.getLogger(__name__) @@ -166,9 +166,9 @@ class TorrentInfo(object): # Check if 'name.utf-8' is in the torrent and if not try to decode the string # using the encoding found. if 'name.utf-8' in self.__m_metadata['info']: - self.__m_name = decode_string(self.__m_metadata['info']['name.utf-8']) + self.__m_name = decode_bytes(self.__m_metadata['info']['name.utf-8']) else: - self.__m_name = decode_string(self.__m_metadata['info']['name'], self.encoding) + self.__m_name = decode_bytes(self.__m_metadata['info']['name'], self.encoding) # Get list of files from torrent info paths = {} @@ -180,10 +180,10 @@ class TorrentInfo(object): for index, f in enumerate(self.__m_metadata['info']['files']): if 'path.utf-8' in f: - path = decode_string(os.path.join(prefix, *f['path.utf-8'])) + path = decode_bytes(os.path.join(prefix, *f['path.utf-8'])) del f['path.utf-8'] else: - path = os.path.join(prefix, decode_string(os.path.join(*f['path']), self.encoding)) + path = os.path.join(prefix, decode_bytes(os.path.join(*f['path']), self.encoding)) f['path'] = path f['index'] = index if 'sha1' in f and len(f['sha1']) == 20: diff --git a/deluge/ui/console/modes/addtorrents.py b/deluge/ui/console/modes/addtorrents.py index 3a7fa0640..e35af806c 100644 --- a/deluge/ui/console/modes/addtorrents.py +++ b/deluge/ui/console/modes/addtorrents.py @@ -181,7 +181,7 @@ class AddTorrents(BaseMode): self.formatted_rows = [] for row in self.raw_rows: - filename = deluge.common.decode_string(row[0]) + filename = deluge.common.decode_bytes(row[0]) size = row[1] time = row[2] diff --git a/deluge/ui/gtkui/ipcinterface.py b/deluge/ui/gtkui/ipcinterface.py index 18ab6ae95..50e0e2edd 100644 --- a/deluge/ui/gtkui/ipcinterface.py +++ b/deluge/ui/gtkui/ipcinterface.py @@ -23,7 +23,7 @@ from twisted.internet import reactor from twisted.internet.protocol import ClientFactory, Factory, Protocol, connectionDone import deluge.component as component -from deluge.common import decode_string, is_magnet, is_url, windows_check +from deluge.common import decode_bytes, is_magnet, is_url, windows_check from deluge.configmanager import ConfigManager, get_config_dir from deluge.ui.client import client @@ -206,7 +206,7 @@ def process_args(args): log.debug('Attempting to add file (%s) from external source...', arg) if urlparse(arg).scheme == 'file': arg = url2pathname(urlparse(arg).path) - path = os.path.abspath(decode_string(arg)) + path = os.path.abspath(decode_bytes(arg)) if not os.path.exists(path): log.error('No such file: %s', path) diff --git a/deluge/ui/gtkui/preferences.py b/deluge/ui/gtkui/preferences.py index 7e417f912..ab5954cc7 100644 --- a/deluge/ui/gtkui/preferences.py +++ b/deluge/ui/gtkui/preferences.py @@ -910,7 +910,7 @@ class Preferences(component.Component): response = chooser.run() if response == gtk.RESPONSE_OK: - filepath = deluge.common.decode_string(chooser.get_filename()) + filepath = deluge.common.decode_bytes(chooser.get_filename()) else: chooser.destroy() return