From 092d48bca2615e8ccbe8a80b291726827f18f1be Mon Sep 17 00:00:00 2001 From: Jacob Stevens Date: Tue, 11 Oct 2022 09:57:49 -0500 Subject: [PATCH] More progress. Need to implement compression and such --- .gitignore | 1 + MediaManager.py | 25 +++++ Universal/network/sftpservice.py | 16 ++- Universal/tools.py | 6 +- Video/Services/tmdbservice.py | 2 +- Video/default.py | 2 +- Video/videoservice.py | 178 +++++++++++++++++++++++++------ 7 files changed, 185 insertions(+), 45 deletions(-) create mode 100644 MediaManager.py diff --git a/.gitignore b/.gitignore index b853414..88205eb 100644 --- a/.gitignore +++ b/.gitignore @@ -152,3 +152,4 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ +Settings.cfg diff --git a/MediaManager.py b/MediaManager.py new file mode 100644 index 0000000..6b1e549 --- /dev/null +++ b/MediaManager.py @@ -0,0 +1,25 @@ +import time +from Video import videoservice +from Universal import loggingservice + +tmp_RunScript = True +tmp_Initialized = False +tmp_Debug = True +tmp_OutputFile = False + +tmp_Library = 1 +tmp_Resume = False + +logger = loggingservice.Output(tmp_Debug, tmp_OutputFile) + +while tmp_RunScript: + if not tmp_Initialized: + video_process = videoservice.VideoService() + tmp_Initialized = True + if tmp_Resume: + #TODO Implement Resume to VideoService + pass + else: + video_process.startNewProcess(tmp_Library) + tmp_Library += 1 +print("Goodbye") \ No newline at end of file diff --git a/Universal/network/sftpservice.py b/Universal/network/sftpservice.py index 8c1d2af..a9a13ce 100644 --- a/Universal/network/sftpservice.py +++ b/Universal/network/sftpservice.py @@ -1,10 +1,8 @@ import time class SFTPService: - def __init__(self, logger, ioservice, universal): + def __init__(self, logger): self.logger = logger - self.ioservice = ioservice - self.universal = universal self.report(2, "Initializing") try: import paramiko @@ -17,6 +15,14 @@ class SFTPService: self.report(3, "Disabling SFTP features") self.connected = False + def isEnabled(self): + self.report(2, "isEnabled") + return self.enabled + + def isConnected(self): + self.report(2, "isConnected") + return self.connected + def connectSFTPServer(self, host, port, username, password, key_file, key_type): self.report(2, "connectSFTPServer") self.host = host @@ -32,7 +38,7 @@ class SFTPService: elif key_file.lower().find("rsa") != -1: tmp_key = self.paramiko.RSAKey.from_private_key_file(key_file) try: - tmp = self.paramiko.Transport((host, port)) + tmp = self.paramiko.Transport((host, int(port))) tmp.connect(None, username, password, tmp_key) except: self.connected = False @@ -71,7 +77,7 @@ class SFTPService: def getFileSize(self, path, retried=0): self.report(2, "getFileSize") try: - return str(self.sftp_socket.lstat(path)).split()[4] + return int(str(self.sftp_socket.lstat(path)).split()[4]) except IOError: return None except: diff --git a/Universal/tools.py b/Universal/tools.py index f1a4f6b..d034959 100644 --- a/Universal/tools.py +++ b/Universal/tools.py @@ -1,7 +1,3 @@ -from jmespath import search -from pyparsing import replace_with - - def replace(input_str, search_key, replace_str, checkspaces=False): if replace_str is None: input_str = replace(input_str, search_key, "", checkspaces) @@ -13,5 +9,5 @@ def replace(input_str, search_key, replace_str, checkspaces=False): else: input_str = input_str.replace(search_key, replace_str) else: - input_str = input_str.replace(search, replace_str) + input_str = input_str.replace(search_key, replace_str) return input_str diff --git a/Video/Services/tmdbservice.py b/Video/Services/tmdbservice.py index 5f7fc7a..703f365 100644 --- a/Video/Services/tmdbservice.py +++ b/Video/Services/tmdbservice.py @@ -2,12 +2,12 @@ from Universal import ioservice class TMDBService: def __init__(self, loggingservice): + self.logger = loggingservice self.report(2, "Initializing") try: import tmdbsimple self.tmdb = tmdbsimple self.report(2, "API Found") - self.logger = loggingservice self.enabled = True self.tmdb_key = None self.addDefaultConfiguration() diff --git a/Video/default.py b/Video/default.py index 82196a8..b1622c5 100644 --- a/Video/default.py +++ b/Video/default.py @@ -54,7 +54,7 @@ class VideoSettings: def updateLibrary(self, num): tmp_section = ("Video Library " + str(num)) - tmp_name = ioservice.getConfigurationStr(tmp_section, "Directory Name") + tmp_name = ioservice.getConfigurationStr(tmp_section, "Name") tmp_input = ioservice.getConfigurationStr(tmp_section, "Input") tmp_output = ioservice.getConfigurationStr(tmp_section, "Output") tmp_database = ioservice.getConfigurationStr(tmp_section, "Database Service") diff --git a/Video/videoservice.py b/Video/videoservice.py index 420da41..90646b3 100644 --- a/Video/videoservice.py +++ b/Video/videoservice.py @@ -1,18 +1,23 @@ import os import re import time -import default import subprocess +from Video import default from Universal import tools from Universal import ioservice from Universal import loggingservice from Video.Services import tmdbservice +from Universal.network import sftpservice class VideoService: def __init__(self): self.logger = loggingservice.Output(True, False) self.video_settings = default.VideoSettings() - self.tmdbservice = tmdbservice(self.logger) + # Movie/TV Show Database + self.tmdbservice = tmdbservice.TMDBService(self.logger) + # Network Service + self.sftpservice = sftpservice.SFTPService(self.logger) + self.failedFiles = [] self.video_settings.updateSettings() def startNewProcess(self, num=1): @@ -25,14 +30,16 @@ class VideoService: self.video_settings.updateServer(self.video_settings.library["Server"]) while self.is_processing: self.processLibraries() + self.startNewProcess(self.current_library) - def processLibraries(self, refreshLib = True): + def processLibraries(self, refreshLib=True): self.logger.debugReport("[Video] processLibraries") tmp_library_time = time.time() if self.video_settings.library is not None: if self.video_settings.library["Input"] is not None: if refreshLib: self.total_processed = 0 + time.sleep(2) self.logger.infoReport("[Video] Current Library: " + self.video_settings.library["Name"]) self.current_library_list = ioservice.listAllFiles(self.video_settings.library["Input"]) if self.current_library_list is not None: @@ -46,7 +53,10 @@ class VideoService: if self.srt_list is not None: for item_search in self.srt_list: self.current_library_list.remove(item_search) - self.logger.infoReport("[Video] Found a total of " + str(len(self.current_library_list)) + " files") + if len(self.current_library_list) == 0: + self.is_processing = False + else: + self.logger.infoReport("[Video] Found a total of " + str(len(self.current_library_list)) + " files") for self.current_file in self.current_library_list: if self.total_processed != 0: self.logger.infoReport("[Video] Total file remaining: " + str(len(self.current_library_list) - self.total_processed)) @@ -65,6 +75,8 @@ class VideoService: self.logger.debugReport("[Video] processVideo") self.logger.infoReport("[Video] Current File: " + self.current_file) tmp_isDeleted = False + tmp_isSuccess = True + tmp_serverPath = None if not ioservice.checkSizeChange(self.current_file): if ioservice.getFileSize(self.current_file) == 0: self.logger.infoReport("[Video] Assuming file has been deleted or moved... Skipping...") @@ -73,15 +85,94 @@ class VideoService: self.logger.infoReport("[Video] File size has changed") self.processVideo() else: - tmp_success = True if not tmp_isDeleted: - tmp_current_file_info = self.extractVideoInformation(self.current_file) -# self.constructOutputPath(tmp_current_file_info) - if tmp_current_file_info["Episode"] is None and tmp_current_file_info["Season"] is None: - self.logger.infoReport("[Video] " + tmp_current_file_info["Name"] + " " + str(tmp_current_file_info["Year"])) + tmp_currentFileDetails = self.extractVideoInformation(self.current_file) + self.constructOutputPath(tmp_currentFileDetails) + if tmp_currentFileDetails["Episode"] is None and tmp_currentFileDetails["Season"] is None: + self.logger.infoReport("[Video] " + tmp_currentFileDetails["Name"] + " " + str(tmp_currentFileDetails["Year"])) else: - self.logger.infoReport("[Video] " + tmp_current_file_info["Name"] + " Season: " + str(tmp_current_file_info["Season"]) + " Episode: " + str(tmp_current_file_info["Episode"])) - self.find_srts() + self.logger.infoReport("[Video] " + tmp_currentFileDetails["Name"] + " Season: " + str(tmp_currentFileDetails["Season"]) + " Episode: " + str(tmp_currentFileDetails["Episode"])) + self.findSRTs() + tmp_compress = False + if not self.video_settings.library["Server"].lower().find("none") != -1: + self.video_settings.updateServer(self.video_settings.library["Server"]) + tmp_serverPath = self.constructServerPath() + if self.video_settings.server["Type"].lower().find("sftp") != -1: + if self.sftpservice.isEnabled(): + if not self.sftpservice.isConnected(): + self.sftpservice.connectSFTPServer(self.video_settings.server["Host"], self.video_settings.server["Port"], self.video_settings.server["Username"], + self.video_settings.server["Password"], self.video_settings.server["Key File"], self.video_settings.server["Key Type"]) + if self.sftpservice.isConnected(): + if not ioservice.doesFileExist(self.currentFileOutputPath): + if self.video_settings.library["Server Overwrite"]: + tmp_compress = True + else: + if not self.sftpservice.doesFileExists(tmp_serverPath): + tmp_compress = True + else: + if not ioservice.doesFileExist(self.currentFileOutputPath): + tmp_compress = True + if tmp_compress and not self.video_settings.library["Compress"]: + if self.video_settings.library["Delete Input"]: + self.logger.infoReport("[Video] Moving file") + self.is_processing = True + ioservice.moveFile(self.current_file, self.currentFileOutputPath) + i = 0 + while i < len(self.srt_to_include): + ioservice.moveFile(self.srt_to_include[i], self.constructOutputPath(tmp_currentFileDetails, True, i)) + i += 1 + self.is_processing = False + tmp_isSuccess = True + self.logger.infoReport("[Video] Move complete") + else: + self.logger.infoReport("[Video] Copying file") + self.is_processing = True + ioservice.moveFile(self.current_file, self.currentFileOutputPath, True) + i = 0 + while i < len(self.srt_to_include): + ioservice.moveFile(self.srt_to_include[i], self.constructOutputPath(tmp_currentFileDetails, True, i), True) + i += 1 + self.is_processing = False + tmp_isSuccess = True + self.logger.infoReport("[Video] Copying complete") + elif tmp_compress and self.video_settings.library["Compress"]: + self.logger.infoReport("[Video] Compressing video") + self.startCompressing(self.current_file, self.currentFileOutputPath, self.srt_to_include) + tmp_isSuccess = True + self.logger.infoReport("[Video] Compressing completed") + if not self.video_settings.library["Server"].lower().find("none") != -1: + self.video_settings.updateServer(self.video_settings.library["Server"]) + if self.video_settings.server["Type"].lower().find("sftp") != -1: + if self.sftpservice.isEnabled(): + if not self.sftpservice.isConnected(): + self.sftpservice.connectSFTPServer(self.video_settings.server["Host"], self.video_settings.server["Port"], self.video_settings.server["Username"], + self.video_settings.server["Password"], self.video_settings.server["Key File"], self.video_settings.server["Key Type"]) + if self.sftpservice.isConnected(): + if self.sftpservice.doesFileExists(tmp_serverPath): + if self.video_settings.library["Server Overwrite"]: + if self.sftpservice.getFileSize(tmp_serverPath) != ioservice.getFileSize(self.currentFileOutputPath): + if self.sftpservice.doesFileExist(tmp_serverPath): + self.sftpservice.deleteFile(tmp_serverPath) + if self.sftpservice.uploadFile(self.currentFileOutputPath, tmp_serverPath): + tmp_isSuccess = True + self.logger.infoReport("[Video] Upload Complete") + else: + tmp_isSuccess = False + else: + if self.sftpservice.uploadFile(self.currentFileOutputPath, tmp_serverPath): + tmp_isSuccess = True + self.logger.infoReport("[Video] Upload Complete") + else: + tmp_isSuccess = False + else: + if self.sftpservice.uploadFile(self.currentFileOutputPath, tmp_serverPath): + tmp_isSuccess = True + else: + tmp_isSuccess = False + if not tmp_isSuccess: + self.failedFiles = self.failedFiles + [self.current_file] + else: + self.total_processed += 1 def extractVideoInformation(self, file): self.logger.debugReport("[Video] extractVideoInformation") @@ -158,21 +249,22 @@ class VideoService: tmp_name = tmp_data["name"] tmp_year = tmp_data["first_air_date"][:4] case "tvdb": - #NotImplemented + #TODO: Implement pass case _: + #TODO: Implement pass tmp_name = tmp_name.replace(":", "") if tmp_episode_title: tmp_episode_title = tmp_episode_title.replace(":", "") - return {"Name": tmp_name, "Season": tmp_season, "Episode": tmp_episode, "Episode Title": tmp_episode_title, "Episode Year": tmp_episode_year, "Input Format": tmp_input_format} + return {"Name": tmp_name, "Year": tmp_year, "Season": tmp_season, "Episode": tmp_episode, "Episode Title": tmp_episode_title, "Episode Year": tmp_episode_year, "Format": tmp_input_format} def constructOutputPath(self, file_info, subtitles=False, num=0): self.logger.debugReport("[Video] constructOutputPath") - self.current_output_path = self.video_settings.library["Output"] + self.video_settings.library["Directory"] - self.current_output_path = tools.replace(self.current_output_path, "${NAME}", file_info["Name"]) - self.current_output_path = tools.replace(self.current_output_path, "${YEAR}", file_info["Year"]) - self.current_output_path = tools.replace(self.current_output_path, "${SEASON}", file_info["Season"]) + self.currentFileOutputPath = self.video_settings.library["Output"] + self.video_settings.library["Directory"] + self.currentFileOutputPath = tools.replace(self.currentFileOutputPath, "${NAME}", file_info["Name"]) + self.currentFileOutputPath = tools.replace(self.currentFileOutputPath, "${YEAR}", file_info["Year"]) + self.currentFileOutputPath = tools.replace(self.currentFileOutputPath, "${SEASON}", file_info["Season"]) if isinstance(file_info["Episode"], list): tmp_episodes = None for eps in file_info["Episode"]: @@ -186,29 +278,37 @@ class VideoService: tmp_episodes = str(eps) else: tmp_episodes = tmp_episodes + str(eps) - self.current_output_path = tools.replace(self.current_output_path, "${EPISODE}", tmp_episodes) + self.currentFileOutputPath = tools.replace(self.currentFileOutputPath, "${EPISODE}", tmp_episodes) else: - self.current_output_path = tools.replace(self.current_output_path, "${EPISODE}", file_info["Episode"]) - self.current_output_path = tools.replace(self.current_output_path, "${EPISODE_YEAR}", file_info["Episode Year"]) - self.current_output_path = tools.replace(self.current_output_path, "${EPISODE_NAME}", file_info["Episode Name"]) + self.currentFileOutputPath = tools.replace(self.currentFileOutputPath, "${EPISODE}", file_info["Episode"]) + self.currentFileOutputPath = tools.replace(self.currentFileOutputPath, "${EPISODE_YEAR}", file_info["Episode Year"]) + self.currentFileOutputPath = tools.replace(self.currentFileOutputPath, "${EPISODE_NAME}", file_info["Episode Title"]) if not subtitles: - if self.current_output_path[self.current_output_path.index("${FORMAT}") - 1] != ".": + if self.currentFileOutputPath[self.currentFileOutputPath.index("${FORMAT}") - 1] != ".": if file_info["Format"].startswith("."): - self.current_output_path = tools.replace(self.current_output_path, "${FORMAT}", file_info["Format"]) + self.currentFileOutputPath = tools.replace(self.currentFileOutputPath, "${FORMAT}", file_info["Format"]) else: - self.current_output_path = tools.replace(self.current_output_path, "${FORMAT}", "." + file_info["Format"]) + self.currentFileOutputPath = tools.replace(self.currentFileOutputPath, "${FORMAT}", "." + file_info["Format"]) else: if file_info["Format"].startswith("."): - self.current_output_path = tools.replace(self.current_output_path, ".${FORMAT}", file_info["Format"]) + self.currentFileOutputPath = tools.replace(self.currentFileOutputPath, ".${FORMAT}", file_info["Format"]) else: - self.current_output_path = tools.replace(self.current_output_path, ".${FORMAT}", "." + file_info["Format"]) + self.currentFileOutputPath = tools.replace(self.currentFileOutputPath, ".${FORMAT}", "." + file_info["Format"]) else: - if self.current_output_path[self.current_output_path.index("${FORMAT}") - 1] != ".": - self.current_output_path = tools.replace(self.current_output_path, "${FORMAT}", " - " + self.srt_language[num] + ".srt") - elif self.current_output_path[self.current_output_path.index("${FORMAT}" - 1)] == ".": - self.current_output_path = tools.replace(self.current_output_path, ".${FORMAT}", " - " + self.srt_language[num] + ".srt") - self.logger.debugReport("[Video] Output Path: " + self.current_output_path) - return self.current_output_path + if self.currentFileOutputPath[self.currentFileOutputPath.index("${FORMAT}") - 1] != ".": + self.currentFileOutputPath = tools.replace(self.currentFileOutputPath, "${FORMAT}", " - " + self.srt_language[num] + ".srt") + elif self.currentFileOutputPath[self.currentFileOutputPath.index("${FORMAT}" - 1)] == ".": + self.currentFileOutputPath = tools.replace(self.currentFileOutputPath, ".${FORMAT}", " - " + self.srt_language[num] + ".srt") + self.logger.debugReport("[Video] Output Path: " + self.currentFileOutputPath) + return self.currentFileOutputPath + + def startCompressing(input_path, output_path, srt_list): + #TODO: Implement + pass + + def constructServerPath(self): + #TODO: Implement + pass def findSRTs(self): self.logger.debugReport("[Video] findSRTs") @@ -223,7 +323,19 @@ class VideoService: "sag", "san", "sin", "slo", "slv", "sme", "smo", "sna", "snd", "som", "sot", "spa", "srd", "srp", "ssw", "sun", "swa", "swe", "tah", "tam", "tat", "tel", "tgk", "tgl", "tha", "tib", "tir", "ton", "tsn", "tso", "tuk", "tur", "twi", "uig", "ukr", "urd", "uzb", "ven", "vie", "vol", "wel", "wln", "wol", "xho", "yid", "yor", "zha", "zul"] - + self.srt_to_include = [] + self.srt_language = [] + tmp_srt = self.current_file.replace(self.current_file.rsplit(".")[-1], "srt").lower() + if (ioservice.doesFileExist(tmp_srt)): + self.srt_to_include.append(tmp_srt) + self.srt_language.append("eng") + else: + for possible_srt in tmp_srt_lang: + tmp_srt = self.current_file.replace(self.current_file.rsplit(".")[-1], "")[:-1].lower() + tmp_srt = tmp_srt + "-" + possible_srt + ".srt" + if (ioservice.doesFileExist(tmp_srt)): + self.srt_to_include.append(tmp_srt) + self.srt_language.append(possible_srt) def getVideoLength(self): self.logger.debugReport("[Video] getVideoLength")