From 8a8859694cebb1a51ae599e5768875008d61404c Mon Sep 17 00:00:00 2001 From: albertosottile Date: Wed, 25 Jul 2018 14:04:58 +0200 Subject: [PATCH 01/12] Enable logging support on server for client versions --- syncplay/constants.py | 1 + syncplay/server.py | 31 +++++++++++++++++++++++++++++-- syncplayServer.py | 1 + 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/syncplay/constants.py b/syncplay/constants.py index 9cad6c0..62501f0 100755 --- a/syncplay/constants.py +++ b/syncplay/constants.py @@ -53,6 +53,7 @@ DIFFERENT_DURATION_THRESHOLD = 2.5 PROTOCOL_TIMEOUT = 12.5 RECONNECT_RETRIES = 999 SERVER_STATE_INTERVAL = 1 +SERVER_LOG_SNAPSHOT_INTERVAL = 3600 WARNING_OSD_MESSAGES_LOOP_INTERVAL = 1 AUTOPLAY_DELAY = 3.0 DO_NOT_RESET_POSITION_THRESHOLD = 1.0 diff --git a/syncplay/server.py b/syncplay/server.py index 4b8cdaa..a2c467d 100755 --- a/syncplay/server.py +++ b/syncplay/server.py @@ -4,6 +4,7 @@ import codecs import hashlib import os import random +import sqlite3 import time from string import Template @@ -18,11 +19,12 @@ from syncplay.utils import RoomPasswordProvider, NotControlledRoom, RandomString class SyncFactory(Factory): - def __init__(self, password='', motdFilePath=None, isolateRooms=False, salt=None, + def __init__(self, port='', password='', motdFilePath=None, isolateRooms=False, salt=None, disableReady=False, disableChat=False, maxChatMessageLength=constants.MAX_CHAT_MESSAGE_LENGTH, - maxUsernameLength=constants.MAX_USERNAME_LENGTH): + maxUsernameLength=constants.MAX_USERNAME_LENGTH, logDbFile=None): self.isolateRooms = isolateRooms print(getMessage("welcome-server-notification").format(syncplay.version)) + self.port = port if password: password = hashlib.md5(password).hexdigest() self.password = password @@ -39,6 +41,11 @@ class SyncFactory(Factory): self._roomManager = RoomManager() else: self._roomManager = PublicRoomManager() + if logDbFile is not None: + self.logDbHandle = self._connectToLogDb(logDbFile) + reactor.callLater(random.randint(1,60), self._scheduleVersionSnapshot, self.logDbHandle, self.port) + else: + self.logDbHandle = None def buildProtocol(self, addr): return SyncServerProtocol(self) @@ -185,6 +192,17 @@ class SyncFactory(Factory): else: watcher.setPlaylistIndex(room.getName(), room.getPlaylistIndex()) + def _connectToLogDb(self, dbPath): + conn = sqlite3.connect(dbPath) + c = conn.cursor() + c.execute('create table if not exists versionSnapshots (snapshotTime integer, port integer, version string)') + conn.commit() + return conn + + def _scheduleVersionSnapshot(self, dbHandler, portNumber): + self._versionSnapshotTimer = task.LoopingCall(self._roomManager.runVersionSnapshot, dbHandler, portNumber) + self._versionSnapshotTimer.start(constants.SERVER_LOG_SNAPSHOT_INTERVAL) + class RoomManager(object): def __init__(self): @@ -260,6 +278,14 @@ class PublicRoomManager(RoomManager): RoomManager.moveWatcher(self, watcher, room) watcher.setFile(watcher.getFile()) + def runVersionSnapshot(self, dbHandler, portNumber): + snapshotTime = str(int(time.time())) + c = dbHandler.cursor() + for room in self._rooms.values(): + for watcher in room.getWatchers(): + c.execute("INSERT INTO versionSnapshots VALUES (" + snapshotTime + ", " + str(portNumber) + ", '" + watcher.getVersion() + "')") + dbHandler.commit() + class Room(object): STATE_PAUSED = 0 @@ -562,3 +588,4 @@ class ConfigurationGetter(object): self._argparser.add_argument('--motd-file', metavar='file', type=str, nargs='?', help=getMessage("server-motd-argument")) self._argparser.add_argument('--max-chat-message-length', metavar='maxChatMessageLength', type=int, nargs='?', help=getMessage("server-chat-maxchars-argument").format(constants.MAX_CHAT_MESSAGE_LENGTH)) self._argparser.add_argument('--max-username-length', metavar='maxUsernameLength', type=int, nargs='?', help=getMessage("server-maxusernamelength-argument").format(constants.MAX_USERNAME_LENGTH)) + self._argparser.add_argument('--log-db-file', metavar='file', type=str, nargs='?', help=getMessage("server-log-db-file-argument")) diff --git a/syncplayServer.py b/syncplayServer.py index 3aa9866..bdd265c 100755 --- a/syncplayServer.py +++ b/syncplayServer.py @@ -22,6 +22,7 @@ if __name__ == '__main__': reactor.listenTCP( int(args.port), SyncFactory( + args.port, args.password, args.motd_file, args.isolate_rooms, From 9f52834ae48d0d2be44bef6d6afab76cedef33f3 Mon Sep 17 00:00:00 2001 From: albertosottile Date: Wed, 25 Jul 2018 14:29:44 +0200 Subject: [PATCH 02/12] Server logger: fix bug and sanitize version string --- syncplay/constants.py | 2 +- syncplay/server.py | 10 ++++++++-- syncplayServer.py | 4 +++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/syncplay/constants.py b/syncplay/constants.py index 62501f0..c514d97 100755 --- a/syncplay/constants.py +++ b/syncplay/constants.py @@ -53,7 +53,7 @@ DIFFERENT_DURATION_THRESHOLD = 2.5 PROTOCOL_TIMEOUT = 12.5 RECONNECT_RETRIES = 999 SERVER_STATE_INTERVAL = 1 -SERVER_LOG_SNAPSHOT_INTERVAL = 3600 +SERVER_LOG_SNAPSHOT_INTERVAL = 1 WARNING_OSD_MESSAGES_LOOP_INTERVAL = 1 AUTOPLAY_DELAY = 3.0 DO_NOT_RESET_POSITION_THRESHOLD = 1.0 diff --git a/syncplay/server.py b/syncplay/server.py index a2c467d..3efe65b 100755 --- a/syncplay/server.py +++ b/syncplay/server.py @@ -4,6 +4,7 @@ import codecs import hashlib import os import random +import re import sqlite3 import time from string import Template @@ -43,7 +44,7 @@ class SyncFactory(Factory): self._roomManager = PublicRoomManager() if logDbFile is not None: self.logDbHandle = self._connectToLogDb(logDbFile) - reactor.callLater(random.randint(1,60), self._scheduleVersionSnapshot, self.logDbHandle, self.port) + reactor.callLater(0.1, self._scheduleVersionSnapshot, self.logDbHandle, self.port) else: self.logDbHandle = None @@ -471,7 +472,12 @@ class Watcher(object): return self._name def getVersion(self): - return self._connector.getVersion() + pattern = r'\A[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}\Z' + versionString = self._connector.getVersion() + if re.match(pattern, versionString) is not None: + return versionString + else: + return None def getFile(self): return self._file diff --git a/syncplayServer.py b/syncplayServer.py index bdd265c..b6bbd89 100755 --- a/syncplayServer.py +++ b/syncplayServer.py @@ -29,5 +29,7 @@ if __name__ == '__main__': args.salt, args.disable_ready, args.disable_chat, - args.max_chat_message_length)) + args.max_chat_message_length, + args.max_username_length, + args.log_db_file)) reactor.run() From c32bd1191c73efd3f31b98739fd900bcde81f6dd Mon Sep 17 00:00:00 2001 From: albertosottile Date: Wed, 25 Jul 2018 15:40:37 +0200 Subject: [PATCH 03/12] Server logger: add roomIndex and playStatus --- syncplay/constants.py | 2 +- syncplay/server.py | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/syncplay/constants.py b/syncplay/constants.py index c514d97..62501f0 100755 --- a/syncplay/constants.py +++ b/syncplay/constants.py @@ -53,7 +53,7 @@ DIFFERENT_DURATION_THRESHOLD = 2.5 PROTOCOL_TIMEOUT = 12.5 RECONNECT_RETRIES = 999 SERVER_STATE_INTERVAL = 1 -SERVER_LOG_SNAPSHOT_INTERVAL = 1 +SERVER_LOG_SNAPSHOT_INTERVAL = 3600 WARNING_OSD_MESSAGES_LOOP_INTERVAL = 1 AUTOPLAY_DELAY = 3.0 DO_NOT_RESET_POSITION_THRESHOLD = 1.0 diff --git a/syncplay/server.py b/syncplay/server.py index 3efe65b..0e7c4f5 100755 --- a/syncplay/server.py +++ b/syncplay/server.py @@ -44,7 +44,8 @@ class SyncFactory(Factory): self._roomManager = PublicRoomManager() if logDbFile is not None: self.logDbHandle = self._connectToLogDb(logDbFile) - reactor.callLater(0.1, self._scheduleVersionSnapshot, self.logDbHandle, self.port) + logDelay = 5*(int(self.port)%10 + 1) + reactor.callLater(logDelay, self._scheduleVersionSnapshot, self.logDbHandle, self.port) else: self.logDbHandle = None @@ -196,7 +197,7 @@ class SyncFactory(Factory): def _connectToLogDb(self, dbPath): conn = sqlite3.connect(dbPath) c = conn.cursor() - c.execute('create table if not exists versionSnapshots (snapshotTime integer, port integer, version string)') + c.execute('create table if not exists versionSnapshots (snapshotTime integer, port integer, version string, roomIndex integer, playStatus integer)') conn.commit() return conn @@ -282,9 +283,11 @@ class PublicRoomManager(RoomManager): def runVersionSnapshot(self, dbHandler, portNumber): snapshotTime = str(int(time.time())) c = dbHandler.cursor() - for room in self._rooms.values(): + for idx, room in enumerate(self._rooms.values()): + playStatus = str(room.isPlaying()) for watcher in room.getWatchers(): - c.execute("INSERT INTO versionSnapshots VALUES (" + snapshotTime + ", " + str(portNumber) + ", '" + watcher.getVersion() + "')") + c.execute("INSERT INTO versionSnapshots VALUES (" + snapshotTime + ", " + str(portNumber) + + ", '" + watcher.getVersion() + "', " + str(idx) + ", " + playStatus + ")") dbHandler.commit() From 14863fbbe7fe2e976fedf8dda1e5811577b9c9a3 Mon Sep 17 00:00:00 2001 From: albertosottile Date: Wed, 25 Jul 2018 16:09:22 +0200 Subject: [PATCH 04/12] Server Stats: use parametrized query and change column definitions --- syncplay/server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/syncplay/server.py b/syncplay/server.py index 0e7c4f5..8e001c4 100755 --- a/syncplay/server.py +++ b/syncplay/server.py @@ -197,7 +197,7 @@ class SyncFactory(Factory): def _connectToLogDb(self, dbPath): conn = sqlite3.connect(dbPath) c = conn.cursor() - c.execute('create table if not exists versionSnapshots (snapshotTime integer, port integer, version string, roomIndex integer, playStatus integer)') + c.execute('create table if not exists clients_snapshots (snapshot_time integer, port integer, version string, room_index integer, play_status integer)') conn.commit() return conn @@ -286,8 +286,8 @@ class PublicRoomManager(RoomManager): for idx, room in enumerate(self._rooms.values()): playStatus = str(room.isPlaying()) for watcher in room.getWatchers(): - c.execute("INSERT INTO versionSnapshots VALUES (" + snapshotTime + ", " + str(portNumber) + - ", '" + watcher.getVersion() + "', " + str(idx) + ", " + playStatus + ")") + content = (snapshotTime, str(portNumber), watcher.getVersion(), str(idx), playStatus, ) + c.execute("INSERT INTO clients_snapshots VALUES (?, ?, ?, ?, ?)", content) dbHandler.commit() From a630860d2e53c35095d4bd4769e7d8fc74c32aa0 Mon Sep 17 00:00:00 2001 From: albertosottile Date: Wed, 25 Jul 2018 16:19:12 +0200 Subject: [PATCH 05/12] Server Stats: rename feature --- syncplay/constants.py | 2 +- syncplay/messages_de.py | 3 ++- syncplay/messages_en.py | 5 +++-- syncplay/messages_it.py | 5 +++-- syncplay/messages_ru.py | 7 ++++--- syncplay/server.py | 22 +++++++++++----------- syncplayServer.py | 2 +- 7 files changed, 25 insertions(+), 21 deletions(-) diff --git a/syncplay/constants.py b/syncplay/constants.py index 62501f0..3b04d2d 100755 --- a/syncplay/constants.py +++ b/syncplay/constants.py @@ -53,7 +53,7 @@ DIFFERENT_DURATION_THRESHOLD = 2.5 PROTOCOL_TIMEOUT = 12.5 RECONNECT_RETRIES = 999 SERVER_STATE_INTERVAL = 1 -SERVER_LOG_SNAPSHOT_INTERVAL = 3600 +SERVER_STATS_SNAPSHOT_INTERVAL = 3600 WARNING_OSD_MESSAGES_LOOP_INTERVAL = 1 AUTOPLAY_DELAY = 3.0 DO_NOT_RESET_POSITION_THRESHOLD = 1.0 diff --git a/syncplay/messages_de.py b/syncplay/messages_de.py index 1dcab6d..318f0a3 100755 --- a/syncplay/messages_de.py +++ b/syncplay/messages_de.py @@ -434,7 +434,8 @@ de = { "server-motd-argument": "Pfad zur Datei, von der die Nachricht des Tages geladen wird", "server-chat-argument": "Should chat be disabled?", # TODO: Translate "server-chat-maxchars-argument": "Maximum number of characters in a chat message (default is {})", # TODO: Translate - "server-maxusernamelength-argument": "Maximum number of charactrs in a username (default is {})", # TODO: Translate + "server-maxusernamelength-argument": "Maximum number of characters in a username (default is {})", # TODO: Translate + "server-stats-db-file-argument": "Enable server stats using the SQLite db file provided", # TODO: Translate "server-messed-up-motd-unescaped-placeholders": "Die Nachricht des Tages hat unmaskierte Platzhalter. Alle $-Zeichen sollten verdoppelt werden ($$).", "server-messed-up-motd-too-long": "Die Nachricht des Tages ist zu lang - Maximal {} Zeichen, aktuell {}.", diff --git a/syncplay/messages_en.py b/syncplay/messages_en.py index f22015b..267cbce 100755 --- a/syncplay/messages_en.py +++ b/syncplay/messages_en.py @@ -437,8 +437,9 @@ en = { "server-disable-ready-argument": "disable readiness feature", "server-motd-argument": "path to file from which motd will be fetched", "server-chat-argument": "Should chat be disabled?", - "server-chat-maxchars-argument": "Maximum number of characters in a chat message (default is {})", # Default number of characters - "server-maxusernamelength-argument": "Maximum number of charactrs in a username (default is {})", + "server-chat-maxchars-argument": "Maximum number of characters in a chat message (default is {})", # Default number of characters + "server-maxusernamelength-argument": "Maximum number of characters in a username (default is {})", + "server-stats-db-file-argument": "Enable server stats using the SQLite db file provided", "server-messed-up-motd-unescaped-placeholders": "Message of the Day has unescaped placeholders. All $ signs should be doubled ($$).", "server-messed-up-motd-too-long": "Message of the Day is too long - maximum of {} chars, {} given.", diff --git a/syncplay/messages_it.py b/syncplay/messages_it.py index 8904733..0b804aa 100755 --- a/syncplay/messages_it.py +++ b/syncplay/messages_it.py @@ -437,8 +437,9 @@ it = { "server-disable-ready-argument": "disabilita la funzionalità \"pronto\"", "server-motd-argument": "percorso del file da cui verrà letto il messaggio del giorno", "server-chat-argument": "abilita o disabilita la chat", - "server-chat-maxchars-argument": "Numero massimo di caratteri in un messaggio di chat (default è {})", # Default number of characters - "server-maxusernamelength-argument": "Maximum number of charactrs in a username (default is {})", # TODO: Translate + "server-chat-maxchars-argument": "Numero massimo di caratteri in un messaggio di chat (default è {})", # Default number of characters + "server-maxusernamelength-argument": "Numero massimo di caratteri in un nome utente (default è {})", + "server-stats-db-file-argument": "Abilita la raccolta dei dati statistici nel file SQLite indicato", "server-messed-up-motd-unescaped-placeholders": "Il messaggio del giorno ha dei caratteri non 'escaped'. Tutti i simboli $ devono essere doppi ($$).", "server-messed-up-motd-too-long": "Il messaggio del giorno è troppo lungo - numero massimo di caratteri è {}, {} trovati.", diff --git a/syncplay/messages_ru.py b/syncplay/messages_ru.py index 9daecb6..6c31d62 100755 --- a/syncplay/messages_ru.py +++ b/syncplay/messages_ru.py @@ -440,9 +440,10 @@ ru = { "server-motd-argument": "путь к файлу, из которого будет извлекаться MOTD-сообщение", "server-chat-argument": "Should chat be disabled?", # TODO: Translate "server-chat-maxchars-argument": "Maximum number of characters in a chat message (default is {})", # TODO: Translate - "server-maxusernamelength-argument": "Maximum number of charactrs in a username (default is {})", # TODO: Translate - "server-messed-up-motd-unescaped-placeholders": "MOTD-сообщение содержит неэкранированные спец.символы. Все знаки $ должны быть продублированы ($$).", - "server-messed-up-motd-too-long": "MOTD-сообщение слишком длинное: максимальная длина - {} символ(ов), текущая длина - {} символ(ов).", + "server-maxusernamelength-argument": "Maximum number of characters in a username (default is {})", # TODO: Translate + "server-stats-db-file-argument": "Enable server stats using the SQLite db file provided", # TODO: Translate + "server-messed-up-motd-unescaped-placeholders" : "MOTD-сообщение содержит неэкранированные спец.символы. Все знаки $ должны быть продублированы ($$).", + "server-messed-up-motd-too-long" : "MOTD-сообщение слишком длинное: максимальная длина - {} символ(ов), текущая длина - {} символ(ов).", # Server errors "unknown-command-server-error": "Неизвестная команда: {}", # message diff --git a/syncplay/server.py b/syncplay/server.py index 8e001c4..77d99af 100755 --- a/syncplay/server.py +++ b/syncplay/server.py @@ -22,7 +22,7 @@ from syncplay.utils import RoomPasswordProvider, NotControlledRoom, RandomString class SyncFactory(Factory): def __init__(self, port='', password='', motdFilePath=None, isolateRooms=False, salt=None, disableReady=False, disableChat=False, maxChatMessageLength=constants.MAX_CHAT_MESSAGE_LENGTH, - maxUsernameLength=constants.MAX_USERNAME_LENGTH, logDbFile=None): + maxUsernameLength=constants.MAX_USERNAME_LENGTH, statsDbFile=None): self.isolateRooms = isolateRooms print(getMessage("welcome-server-notification").format(syncplay.version)) self.port = port @@ -42,12 +42,12 @@ class SyncFactory(Factory): self._roomManager = RoomManager() else: self._roomManager = PublicRoomManager() - if logDbFile is not None: - self.logDbHandle = self._connectToLogDb(logDbFile) + if statsDbFile is not None: + self.statsDbHandle = self._connectToStatsDb(statsDbFile) logDelay = 5*(int(self.port)%10 + 1) - reactor.callLater(logDelay, self._scheduleVersionSnapshot, self.logDbHandle, self.port) + reactor.callLater(logDelay, self._scheduleClientSnapshot, self.statsDbHandle, self.port) else: - self.logDbHandle = None + self.statsDbHandle = None def buildProtocol(self, addr): return SyncServerProtocol(self) @@ -194,16 +194,16 @@ class SyncFactory(Factory): else: watcher.setPlaylistIndex(room.getName(), room.getPlaylistIndex()) - def _connectToLogDb(self, dbPath): + def _connectToStatsDb(self, dbPath): conn = sqlite3.connect(dbPath) c = conn.cursor() c.execute('create table if not exists clients_snapshots (snapshot_time integer, port integer, version string, room_index integer, play_status integer)') conn.commit() return conn - def _scheduleVersionSnapshot(self, dbHandler, portNumber): - self._versionSnapshotTimer = task.LoopingCall(self._roomManager.runVersionSnapshot, dbHandler, portNumber) - self._versionSnapshotTimer.start(constants.SERVER_LOG_SNAPSHOT_INTERVAL) + def _scheduleClientSnapshot(self, dbHandler, portNumber): + self._clientSnapshotTimer = task.LoopingCall(self._roomManager.runClientSnapshot, dbHandler, portNumber) + self._clientSnapshotTimer.start(constants.SERVER_STATS_SNAPSHOT_INTERVAL) class RoomManager(object): @@ -280,7 +280,7 @@ class PublicRoomManager(RoomManager): RoomManager.moveWatcher(self, watcher, room) watcher.setFile(watcher.getFile()) - def runVersionSnapshot(self, dbHandler, portNumber): + def runClientSnapshot(self, dbHandler, portNumber): snapshotTime = str(int(time.time())) c = dbHandler.cursor() for idx, room in enumerate(self._rooms.values()): @@ -597,4 +597,4 @@ class ConfigurationGetter(object): self._argparser.add_argument('--motd-file', metavar='file', type=str, nargs='?', help=getMessage("server-motd-argument")) self._argparser.add_argument('--max-chat-message-length', metavar='maxChatMessageLength', type=int, nargs='?', help=getMessage("server-chat-maxchars-argument").format(constants.MAX_CHAT_MESSAGE_LENGTH)) self._argparser.add_argument('--max-username-length', metavar='maxUsernameLength', type=int, nargs='?', help=getMessage("server-maxusernamelength-argument").format(constants.MAX_USERNAME_LENGTH)) - self._argparser.add_argument('--log-db-file', metavar='file', type=str, nargs='?', help=getMessage("server-log-db-file-argument")) + self._argparser.add_argument('--stats-db-file', metavar='file', type=str, nargs='?', help=getMessage("server-stats-db-file-argument")) diff --git a/syncplayServer.py b/syncplayServer.py index b6bbd89..28cfd1b 100755 --- a/syncplayServer.py +++ b/syncplayServer.py @@ -31,5 +31,5 @@ if __name__ == '__main__': args.disable_chat, args.max_chat_message_length, args.max_username_length, - args.log_db_file)) + args.stats_db_file)) reactor.run() From bd766b4dfacf9a646762fbdb7356282a88fe746f Mon Sep 17 00:00:00 2001 From: albertosottile Date: Wed, 25 Jul 2018 17:31:17 +0200 Subject: [PATCH 06/12] Server Stats: remove unnecessary type casting before query --- syncplay/server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/syncplay/server.py b/syncplay/server.py index 77d99af..22f20b7 100755 --- a/syncplay/server.py +++ b/syncplay/server.py @@ -281,12 +281,12 @@ class PublicRoomManager(RoomManager): watcher.setFile(watcher.getFile()) def runClientSnapshot(self, dbHandler, portNumber): - snapshotTime = str(int(time.time())) + snapshotTime = int(time.time()) c = dbHandler.cursor() for idx, room in enumerate(self._rooms.values()): - playStatus = str(room.isPlaying()) + playStatus = room.isPlaying() for watcher in room.getWatchers(): - content = (snapshotTime, str(portNumber), watcher.getVersion(), str(idx), playStatus, ) + content = (snapshotTime, int(portNumber), watcher.getVersion(), idx, playStatus, ) c.execute("INSERT INTO clients_snapshots VALUES (?, ?, ?, ?, ?)", content) dbHandler.commit() From d1e0c974da3ef66ca4806ba4eb6636b03b801aea Mon Sep 17 00:00:00 2001 From: albertosottile Date: Thu, 26 Jul 2018 10:03:09 +0200 Subject: [PATCH 07/12] Server Stats: remove port, room_index, and play_status --- syncplay/server.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/syncplay/server.py b/syncplay/server.py index 22f20b7..f3e7859 100755 --- a/syncplay/server.py +++ b/syncplay/server.py @@ -4,7 +4,6 @@ import codecs import hashlib import os import random -import re import sqlite3 import time from string import Template @@ -45,7 +44,7 @@ class SyncFactory(Factory): if statsDbFile is not None: self.statsDbHandle = self._connectToStatsDb(statsDbFile) logDelay = 5*(int(self.port)%10 + 1) - reactor.callLater(logDelay, self._scheduleClientSnapshot, self.statsDbHandle, self.port) + reactor.callLater(logDelay, self._scheduleClientSnapshot, self.statsDbHandle) else: self.statsDbHandle = None @@ -197,12 +196,12 @@ class SyncFactory(Factory): def _connectToStatsDb(self, dbPath): conn = sqlite3.connect(dbPath) c = conn.cursor() - c.execute('create table if not exists clients_snapshots (snapshot_time integer, port integer, version string, room_index integer, play_status integer)') + c.execute('create table if not exists clients_snapshots (snapshot_time integer, version string)') conn.commit() return conn - def _scheduleClientSnapshot(self, dbHandler, portNumber): - self._clientSnapshotTimer = task.LoopingCall(self._roomManager.runClientSnapshot, dbHandler, portNumber) + def _scheduleClientSnapshot(self, dbHandler): + self._clientSnapshotTimer = task.LoopingCall(self._roomManager.runClientSnapshot, dbHandler) self._clientSnapshotTimer.start(constants.SERVER_STATS_SNAPSHOT_INTERVAL) @@ -280,14 +279,14 @@ class PublicRoomManager(RoomManager): RoomManager.moveWatcher(self, watcher, room) watcher.setFile(watcher.getFile()) - def runClientSnapshot(self, dbHandler, portNumber): + def runClientSnapshot(self, dbHandler): snapshotTime = int(time.time()) c = dbHandler.cursor() for idx, room in enumerate(self._rooms.values()): playStatus = room.isPlaying() for watcher in room.getWatchers(): - content = (snapshotTime, int(portNumber), watcher.getVersion(), idx, playStatus, ) - c.execute("INSERT INTO clients_snapshots VALUES (?, ?, ?, ?, ?)", content) + content = (snapshotTime, watcher.getVersion(), ) + c.execute("INSERT INTO clients_snapshots VALUES (?, ?)", content) dbHandler.commit() @@ -475,12 +474,7 @@ class Watcher(object): return self._name def getVersion(self): - pattern = r'\A[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}\Z' - versionString = self._connector.getVersion() - if re.match(pattern, versionString) is not None: - return versionString - else: - return None + return self._connector.getVersion() def getFile(self): return self._file From 6564f22d3afc5a02e96e495c9103d4715b965803 Mon Sep 17 00:00:00 2001 From: albertosottile Date: Thu, 26 Jul 2018 11:42:17 +0200 Subject: [PATCH 08/12] Server Stats: use twisted asynchronous adbapi --- syncplay/server.py | 63 ++++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/syncplay/server.py b/syncplay/server.py index f3e7859..ee2b2f0 100755 --- a/syncplay/server.py +++ b/syncplay/server.py @@ -8,6 +8,7 @@ import sqlite3 import time from string import Template +from twisted.enterprise import adbapi from twisted.internet import task, reactor from twisted.internet.protocol import Factory @@ -41,12 +42,11 @@ class SyncFactory(Factory): self._roomManager = RoomManager() else: self._roomManager = PublicRoomManager() - if statsDbFile is not None: - self.statsDbHandle = self._connectToStatsDb(statsDbFile) - logDelay = 5*(int(self.port)%10 + 1) - reactor.callLater(logDelay, self._scheduleClientSnapshot, self.statsDbHandle) - else: - self.statsDbHandle = None + if statsDbFile is not None: + statsDelay = 5*(int(self.port)%10 + 1) + StatsRecorder(statsDbFile, self._roomManager, statsDelay) + else: + self.statsDbHandle = None def buildProtocol(self, addr): return SyncServerProtocol(self) @@ -193,17 +193,33 @@ class SyncFactory(Factory): else: watcher.setPlaylistIndex(room.getName(), room.getPlaylistIndex()) - def _connectToStatsDb(self, dbPath): - conn = sqlite3.connect(dbPath) - c = conn.cursor() - c.execute('create table if not exists clients_snapshots (snapshot_time integer, version string)') - conn.commit() - return conn - - def _scheduleClientSnapshot(self, dbHandler): - self._clientSnapshotTimer = task.LoopingCall(self._roomManager.runClientSnapshot, dbHandler) - self._clientSnapshotTimer.start(constants.SERVER_STATS_SNAPSHOT_INTERVAL) - +class StatsRecorder(object): + def __init__(self, dbpath, roomManager, delay): + self._roomManagerHandle = roomManager + self._dbpool = self._initDatabase(dbpath) + reactor.callLater(delay, self._scheduleClientSnapshot) + + def __del__(self): + self._dbpool.close() + + def _initDatabase(self, dbPath): + dbpool = adbapi.ConnectionPool("sqlite3", dbPath) + query = 'create table if not exists clients_snapshots (snapshot_time integer, version string)' + dbpool.runQuery(query) + return dbpool + + def _scheduleClientSnapshot(self): + self._clientSnapshotTimer = task.LoopingCall(self._runClientSnapshot) + self._clientSnapshotTimer.start(constants.SERVER_STATS_SNAPSHOT_INTERVAL) + + def _runClientSnapshot(self): + snapshotTime = int(time.time()) + rooms = self._roomManagerHandle.exportRooms() + for room in rooms.values(): + for watcher in room.getWatchers(): + content = (snapshotTime, watcher.getVersion(), ) + self._dbpool.runQuery("INSERT INTO clients_snapshots VALUES (?, ?)", content) + class RoomManager(object): def __init__(self): @@ -263,6 +279,9 @@ class RoomManager(object): while username.lower() in allnames: username += '_' return username + + def exportRooms(self): + return self._rooms class PublicRoomManager(RoomManager): @@ -279,16 +298,6 @@ class PublicRoomManager(RoomManager): RoomManager.moveWatcher(self, watcher, room) watcher.setFile(watcher.getFile()) - def runClientSnapshot(self, dbHandler): - snapshotTime = int(time.time()) - c = dbHandler.cursor() - for idx, room in enumerate(self._rooms.values()): - playStatus = room.isPlaying() - for watcher in room.getWatchers(): - content = (snapshotTime, watcher.getVersion(), ) - c.execute("INSERT INTO clients_snapshots VALUES (?, ?)", content) - dbHandler.commit() - class Room(object): STATE_PAUSED = 0 From 6012a2b109b551b9cd8e9df31c4ba07e1ab24921 Mon Sep 17 00:00:00 2001 From: albertosottile Date: Thu, 26 Jul 2018 16:03:56 +0200 Subject: [PATCH 09/12] Server Stats: incapsulate collection code in try-catch --- syncplay/server.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/syncplay/server.py b/syncplay/server.py index ee2b2f0..32da0b0 100755 --- a/syncplay/server.py +++ b/syncplay/server.py @@ -195,15 +195,21 @@ class SyncFactory(Factory): class StatsRecorder(object): def __init__(self, dbpath, roomManager, delay): + self._dbPath = dbpath self._roomManagerHandle = roomManager - self._dbpool = self._initDatabase(dbpath) - reactor.callLater(delay, self._scheduleClientSnapshot) + try: + self._dbPool = self._initDatabase() + reactor.callLater(delay, self._scheduleClientSnapshot) + except: + self._dbPool = None + print("--- Error in initializing the stats database. Server Stats not enabled. ---") def __del__(self): - self._dbpool.close() + if self._dbPool is not None: + self._dbPool.close() - def _initDatabase(self, dbPath): - dbpool = adbapi.ConnectionPool("sqlite3", dbPath) + def _initDatabase(self): + dbpool = adbapi.ConnectionPool("sqlite3", self._dbPath, check_same_thread=False) query = 'create table if not exists clients_snapshots (snapshot_time integer, version string)' dbpool.runQuery(query) return dbpool @@ -213,12 +219,15 @@ class StatsRecorder(object): self._clientSnapshotTimer.start(constants.SERVER_STATS_SNAPSHOT_INTERVAL) def _runClientSnapshot(self): - snapshotTime = int(time.time()) - rooms = self._roomManagerHandle.exportRooms() - for room in rooms.values(): - for watcher in room.getWatchers(): - content = (snapshotTime, watcher.getVersion(), ) - self._dbpool.runQuery("INSERT INTO clients_snapshots VALUES (?, ?)", content) + try: + snapshotTime = int(time.time()) + rooms = self._roomManagerHandle.exportRooms() + for room in rooms.values(): + for watcher in room.getWatchers(): + content = (snapshotTime, watcher.getVersion(), ) + self._dbPool.runQuery("INSERT INTO clients_snapshots VALUES (?, ?)", content) + except: + pass class RoomManager(object): From c99c6e57acc77199576cdf05196390c1125f756b Mon Sep 17 00:00:00 2001 From: Alberto Sottile Date: Thu, 26 Jul 2018 23:53:16 +0200 Subject: [PATCH 10/12] Server Stats: remove sqlite3 dependency and enable stats for non-isolated servers --- syncplay/server.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/syncplay/server.py b/syncplay/server.py index 32da0b0..45ece4d 100755 --- a/syncplay/server.py +++ b/syncplay/server.py @@ -1,10 +1,8 @@ - import argparse import codecs import hashlib import os import random -import sqlite3 import time from string import Template @@ -42,11 +40,12 @@ class SyncFactory(Factory): self._roomManager = RoomManager() else: self._roomManager = PublicRoomManager() - if statsDbFile is not None: - statsDelay = 5*(int(self.port)%10 + 1) - StatsRecorder(statsDbFile, self._roomManager, statsDelay) - else: - self.statsDbHandle = None + if statsDbFile is not None: + statsDelay = 5*(int(self.port)%10 + 1) + self._statsDbHandle = StatsRecorder() + self._statsDbHandle.startRecorder(statsDbFile, self._roomManager, statsDelay) + else: + self._statsDbHandle = None def buildProtocol(self, addr): return SyncServerProtocol(self) @@ -194,7 +193,14 @@ class SyncFactory(Factory): watcher.setPlaylistIndex(room.getName(), room.getPlaylistIndex()) class StatsRecorder(object): - def __init__(self, dbpath, roomManager, delay): + def __init__(self): + self._dbPool = None + + def __del__(self): + if self._dbPool is not None: + self._dbPool.close() + + def startRecorder(self, dbpath, roomManager, delay): self._dbPath = dbpath self._roomManagerHandle = roomManager try: @@ -203,11 +209,7 @@ class StatsRecorder(object): except: self._dbPool = None print("--- Error in initializing the stats database. Server Stats not enabled. ---") - - def __del__(self): - if self._dbPool is not None: - self._dbPool.close() - + def _initDatabase(self): dbpool = adbapi.ConnectionPool("sqlite3", self._dbPath, check_same_thread=False) query = 'create table if not exists clients_snapshots (snapshot_time integer, version string)' From de4e9892f4034b286cc61d38bda2e1da86fe442f Mon Sep 17 00:00:00 2001 From: Alberto Sottile Date: Fri, 27 Jul 2018 00:20:28 +0200 Subject: [PATCH 11/12] Server Stats: addressing review comments --- syncplay/server.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/syncplay/server.py b/syncplay/server.py index 45ece4d..fc80a6a 100755 --- a/syncplay/server.py +++ b/syncplay/server.py @@ -42,8 +42,8 @@ class SyncFactory(Factory): self._roomManager = PublicRoomManager() if statsDbFile is not None: statsDelay = 5*(int(self.port)%10 + 1) - self._statsDbHandle = StatsRecorder() - self._statsDbHandle.startRecorder(statsDbFile, self._roomManager, statsDelay) + self._statsDbHandle = StatsRecorder(statsDbFile, self._roomManager) + self._statsDbHandle.startRecorder(statsDelay) else: self._statsDbHandle = None @@ -193,21 +193,21 @@ class SyncFactory(Factory): watcher.setPlaylistIndex(room.getName(), room.getPlaylistIndex()) class StatsRecorder(object): - def __init__(self): - self._dbPool = None - - def __del__(self): - if self._dbPool is not None: - self._dbPool.close() - - def startRecorder(self, dbpath, roomManager, delay): + def __init__(self, dbpath, roomManager): self._dbPath = dbpath self._roomManagerHandle = roomManager + self._connection = None + + def __del__(self): + if self._connection is not None: + self._connection.close() + + def startRecorder(self, delay): try: - self._dbPool = self._initDatabase() + self._connection = self._initDatabase() reactor.callLater(delay, self._scheduleClientSnapshot) except: - self._dbPool = None + self._connection = None print("--- Error in initializing the stats database. Server Stats not enabled. ---") def _initDatabase(self): @@ -227,7 +227,7 @@ class StatsRecorder(object): for room in rooms.values(): for watcher in room.getWatchers(): content = (snapshotTime, watcher.getVersion(), ) - self._dbPool.runQuery("INSERT INTO clients_snapshots VALUES (?, ?)", content) + self._connection.runQuery("INSERT INTO clients_snapshots VALUES (?, ?)", content) except: pass From 3234d9bc1a44be747a9fdecda0fa6f2c3c17d596 Mon Sep 17 00:00:00 2001 From: Alberto Sottile Date: Fri, 27 Jul 2018 00:54:44 +0200 Subject: [PATCH 12/12] Server Stats: use a separate class to manage the database --- syncplay/server.py | 51 ++++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/syncplay/server.py b/syncplay/server.py index fc80a6a..af06a50 100755 --- a/syncplay/server.py +++ b/syncplay/server.py @@ -41,9 +41,10 @@ class SyncFactory(Factory): else: self._roomManager = PublicRoomManager() if statsDbFile is not None: + self._statsDbHandle = DBManager(statsDbFile) + self._statsRecorder = StatsRecorder(self._statsDbHandle, self._roomManager) statsDelay = 5*(int(self.port)%10 + 1) - self._statsDbHandle = StatsRecorder(statsDbFile, self._roomManager) - self._statsDbHandle.startRecorder(statsDelay) + self._statsRecorder.startRecorder(statsDelay) else: self._statsDbHandle = None @@ -192,30 +193,19 @@ class SyncFactory(Factory): else: watcher.setPlaylistIndex(room.getName(), room.getPlaylistIndex()) + class StatsRecorder(object): - def __init__(self, dbpath, roomManager): - self._dbPath = dbpath + def __init__(self, dbHandle, roomManager): + self._dbHandle = dbHandle self._roomManagerHandle = roomManager - self._connection = None - - def __del__(self): - if self._connection is not None: - self._connection.close() def startRecorder(self, delay): try: - self._connection = self._initDatabase() + self._dbHandle.connect() reactor.callLater(delay, self._scheduleClientSnapshot) except: - self._connection = None print("--- Error in initializing the stats database. Server Stats not enabled. ---") - def _initDatabase(self): - dbpool = adbapi.ConnectionPool("sqlite3", self._dbPath, check_same_thread=False) - query = 'create table if not exists clients_snapshots (snapshot_time integer, version string)' - dbpool.runQuery(query) - return dbpool - def _scheduleClientSnapshot(self): self._clientSnapshotTimer = task.LoopingCall(self._runClientSnapshot) self._clientSnapshotTimer.start(constants.SERVER_STATS_SNAPSHOT_INTERVAL) @@ -226,11 +216,32 @@ class StatsRecorder(object): rooms = self._roomManagerHandle.exportRooms() for room in rooms.values(): for watcher in room.getWatchers(): - content = (snapshotTime, watcher.getVersion(), ) - self._connection.runQuery("INSERT INTO clients_snapshots VALUES (?, ?)", content) + self._dbHandle.addVersionLog(snapshotTime, watcher.getVersion()) except: pass - + + +class DBManager(object): + def __init__(self, dbpath): + self._dbPath = dbpath + self._connection = None + + def __del__(self): + if self._connection is not None: + self._connection.close() + + def connect(self): + self._connection = adbapi.ConnectionPool("sqlite3", self._dbPath, check_same_thread=False) + self._createSchema() + + def _createSchema(self): + initQuery = 'create table if not exists clients_snapshots (snapshot_time integer, version string)' + self._connection.runQuery(initQuery) + + def addVersionLog(self, timestamp, version): + content = (timestamp, version, ) + self._connection.runQuery("INSERT INTO clients_snapshots VALUES (?, ?)", content) + class RoomManager(object): def __init__(self):