From 820d45742df3bedbb0477e06cb651f5d328e5ab2 Mon Sep 17 00:00:00 2001 From: Calin-Andrei Burloiu Date: Mon, 19 Dec 2011 15:33:02 +0200 Subject: [PATCH] CIS: transfer in, transcode, and thumbnail extraction bugs solved --- cis/api/avhandling.py | 15 +++++-- cis/api/base.py | 10 +++-- cis/api/file_transfer.py | 26 +++++++------ cis/cisd.py | 84 ++++++++++++++++++---------------------- cis/config.py | 2 +- 5 files changed, 70 insertions(+), 67 deletions(-) diff --git a/cis/api/avhandling.py b/cis/api/avhandling.py index e300075..808b572 100644 --- a/cis/api/avhandling.py +++ b/cis/api/avhandling.py @@ -96,7 +96,7 @@ class FFmpegTranscoder(base.BaseTranscoder): log.close() - return output_file + return self.output_file class FFmpegThumbExtractor(base.BaseThumbExtractor): @@ -144,16 +144,18 @@ class FFmpegThumbExtractor(base.BaseThumbExtractor): 'FFmpeg created an empty file.') def get_video_duration(self): - return FFmpegAVInfo.get_video_duration(self.input_file) + return FFprobeAVInfo.get_video_duration(self.input_file) -class FFmpegAVInfo(base.BaseAVInfo): +class FFprobeAVInfo(base.BaseAVInfo): prog_bin = "ffprobe" + log_file = 'log/FFprobeAVInfo.log' + @staticmethod def get_video_duration(input_file, formated=False): - args = FFmpegAVInfo.prog_bin + ' -show_format "' \ + args = FFprobeAVInfo.prog_bin + ' -show_format "' \ + input_file + '"' # READ handler for process's output. @@ -161,11 +163,16 @@ class FFmpegAVInfo(base.BaseAVInfo): stdout=subprocess.PIPE, stderr=open(os.devnull, 'w')) pipe = p.stdout + # WRITE handler for logging. + log = open(FFprobeAVInfo.log_file, 'w') + log.write(args + '\n') + # Parse ffprobe's output. while True: line = pipe.readline() if len(line) == 0: break + log.write(line) # Search for the line which contains duration information. m = re.match(r"duration=([\d\.]+)", line) diff --git a/cis/api/base.py b/cis/api/base.py index a2e11d1..36d490b 100644 --- a/cis/api/base.py +++ b/cis/api/base.py @@ -4,11 +4,13 @@ Base classes for the external programs API. """ -import api_exceptions +import os import re -import cis_util import random +import api_exceptions +import cis_util + class BaseTranscoder: """ Abstraction of the API class for the transcoder program. @@ -106,7 +108,7 @@ class BaseTranscoder: and re.match('[\d]+:[\d]+', v_dar) is None): raise ValueError('Video display aspect ratio must be a float or a string like :.') - self.output_file = self.dest_path + self.name + self.output_file = os.path.join(self.dest_path, self.name) if v_resolution is not None: self.output_file += '_' self.output_file += v_resolution[(v_resolution.rindex('x')+1):] @@ -238,7 +240,7 @@ class BaseThumbExtractor: def get_output_file_name(self, index): """ Returns the name required as output file name based on index. """ - output_file_name = self.dest_path + self.name \ + output_file_name = os.path.join(self.dest_path, self.name) \ + '_t' + ("%02d" % index) + '.jpg' return output_file_name diff --git a/cis/api/file_transfer.py b/cis/api/file_transfer.py index 14ef20c..b7435d6 100644 --- a/cis/api/file_transfer.py +++ b/cis/api/file_transfer.py @@ -30,43 +30,45 @@ class FTPFileTransferer(base.BaseFileTransferer): self.ftp.set_pasv(True) def get(self, files): - for crt_file in files: - crt_file = os.path.join(self.local_path, crt_file) + for crt_fn in files: + local_fn = os.path.join(self.local_path, crt_fn) + remote_fn = os.path.join(self.remote_path, crt_fn) try: - file_local = open(crt_file, 'wb') + file_local = open(local_fn, 'wb') except IOError as e: raise api_exceptions.FileTransferException( \ "Could not open local file '%s' for writing: %s" \ - % (crt_file, repr(e))) + % (local_fn, repr(e))) try: self.ftp.cwd(self.remote_path) - self.ftp.retrbinary('RETR %s' % crt_file, file_local.write) + self.ftp.retrbinary('RETR %s' % crt_fn, file_local.write) file_local.close() except ftplib.error_perm as e: raise api_exceptions.FileTransferException( \ "Could not get file '%s' from Web Server: %s" \ - % (crt_file, repr(e))) + % (remote_fn, repr(e))) def put(self, files): - for crt_file in files: - crt_file = os.path.join(self.local_path, crt_file) + for crt_fn in files: + local_fn = os.path.join(self.local_path, crt_fn) + remote_fn = os.path.join(self.local_path, crt_fn) try: - file_local = open(crt_file, 'rb') + file_local = open(local_fn, 'rb') except IOError as e: raise api_exceptions.FileTransferException( \ "Could not open local file '%s' for reading: %s" \ - % (crt_file, repr(e))) + % (local_fn, repr(e))) try: self.ftp.cwd(self.remote_path) - self.ftp.storbinary('STOR %s' % crt_file, file_local) + self.ftp.storbinary('STOR %s' % crt_fn, file_local) file_local.close() except ftplib.error_perm as e: raise api_exceptions.FileTransferException( \ "Could not get file '%s' from Web Server: %s" \ - % (crt_file, repr(e))) + % (remote_fn, repr(e))) def close(self): if self.ftp is not None: diff --git a/cis/cisd.py b/cis/cisd.py index 6a6c310..b042817 100755 --- a/cis/cisd.py +++ b/cis/cisd.py @@ -41,7 +41,7 @@ class CIWorker(threading.Thread): """ - threading.Thread.__init__(self) + threading.Thread.__init__(self, name='CIWorker') self.queue = queue self.bit_torrent = bit_torrent @@ -55,7 +55,7 @@ class CIWorker(threading.Thread): file_transfer = config.FILE_TRANSFERER_CLASS( \ self.raw_videos_dir, config.INPUT_PATH) - file_transfer.get(raw_video) + file_transfer.get([raw_video]) file_transfer.close() print '** Transfering in finished.' @@ -70,24 +70,15 @@ class CIWorker(threading.Thread): """ transcoder = config.TRANSCODER_CLASS( \ - input_file = video_name, \ + input_file = os.path.join(self.raw_videos_dir, input_video), \ name = video_name, prog_bin = config.TRANSCODER_BIN) transcoder.dest_path = self.transcoded_videos_dir # Transcode the raw video in each requested format. # TODO report partial errors for transcode_config in transcode_configs: - transcode_config['output_file'] = transcoder.transcode( \ - container = transcode_config['container'], \ - a_codec = transcode_config['a_codec'], \ - a_bitrate = transcode_config['a_bitrate'], \ - a_samplingrate = transcode_config['a_samplingrate'], \ - a_channels = transcode_config['a_channels'], \ - v_codec = transcode_config['v_codec'], \ - v_bitrate = transcode_config['v_bitrate'], \ - v_framerate = transcode_config['v_framerate'], \ - v_resolution = transcode_config['v_resolution'], \ - v_dar = transcode_config['dar']) + transcode_config['output_file'] = \ + transcoder.transcode(**transcode_config) print '** Transcoding finished.' @@ -104,7 +95,8 @@ class CIWorker(threading.Thread): # TODO report partial errors thumb_extractor = config.THUMB_EXTRACTOR_CLASS( \ - input_file = input_video, name = video_name, \ + input_file = os.path.join(self.raw_videos_dir, input_video), \ + name = video_name, \ prog_bin = config.THUMB_EXTRACTOR_BIN) thumb_extractor.dest_path = self.thumbs_dir if thumbs == 'random': @@ -179,35 +171,35 @@ class CIWorker(threading.Thread): self.extract_thumbs(job['raw_video'], job['name'], \ job['thumbs']) - # * CREATE TORRENTS AND START SEEDING OF TRANSCODED VIDEOS - self.seed(job['transcode_configs']) - - # Torrent files. - files = [f for f in os.listdir(self.torrents_dir) \ - if os.path.isfile(os.path.join( \ - self.torrents_dir, f))] - torrent_files = fnmatch.filter(files, name + "_*") - - # Thumbnail images files. - files = [f for f in os.listdir(self.thumbs_dir) \ - if os.path.isfile(os.path.join( \ - self.thumbs_dir, f))] - thumb_files = fnmatch.filter(files, name + "_*") - - # Raw video files. - raw_files = [f for f in os.listdir(self.raw_videos_dir) \ - if os.path.isfile(os.path.join( \ - self.raw_videos_dir, f))] - - # * TRANSFER TORRENTS AND THUMBNAIL IMAGES OUT - self.transfer_out(torrent_files, self.torrents_dir, \ - config.OUTPUT_TORRENTS_PATH) - self.transfer_out(thumb_files, self.thumbs_dir, \ - config.OUTPUT_THUMBS_PATH) - - # * CLEANUP RAW VIDEOS AND THUMBNAIL IMAGES - self.remove_files(raw_files, self.raw_videos_dir) - self.remove_files(thumb_files, self.thumbs_dir) +# # * CREATE TORRENTS AND START SEEDING OF TRANSCODED VIDEOS +# self.seed(job['transcode_configs']) +# +# # Torrent files. +# files = [f for f in os.listdir(self.torrents_dir) \ +# if os.path.isfile(os.path.join( \ +# self.torrents_dir, f))] +# torrent_files = fnmatch.filter(files, name + "_*") +# +# # Thumbnail images files. +# files = [f for f in os.listdir(self.thumbs_dir) \ +# if os.path.isfile(os.path.join( \ +# self.thumbs_dir, f))] +# thumb_files = fnmatch.filter(files, name + "_*") +# +# # Raw video files. +# raw_files = [f for f in os.listdir(self.raw_videos_dir) \ +# if os.path.isfile(os.path.join( \ +# self.raw_videos_dir, f))] +# +# # * TRANSFER TORRENTS AND THUMBNAIL IMAGES OUT +# self.transfer_out(torrent_files, self.torrents_dir, \ +# config.OUTPUT_TORRENTS_PATH) +# self.transfer_out(thumb_files, self.thumbs_dir, \ +# config.OUTPUT_THUMBS_PATH) +# +# # * CLEANUP RAW VIDEOS AND THUMBNAIL IMAGES +# self.remove_files(raw_files, self.raw_videos_dir) +# self.remove_files(thumb_files, self.thumbs_dir) # * JOB FINISHED queue.task_done() @@ -247,7 +239,7 @@ if __name__ == '__main__': 'v_bitrate': v_bitrate, 'v_resolution': v_resolution } - thumbs = 'random' + thumbs = 4 job = { 'raw_video': raw_video, @@ -255,7 +247,7 @@ if __name__ == '__main__': 'transcode_configs': [transcode_config], 'thumbs': thumbs } - + queue.put(job) queue.join() diff --git a/cis/config.py b/cis/config.py index 990f746..fcc629b 100644 --- a/cis/config.py +++ b/cis/config.py @@ -22,7 +22,7 @@ BT_TRACKER = "http://localhost:6969/announce" # === EXTERNAL PROGRAMS API CLASSES === # API class for a prgram which retrives audio/video information, like duration. -AVINFO_CLASS = avhandling.FFmpegAVInfo +AVINFO_CLASS = avhandling.FFprobeAVInfo # API class for a prgram which transcodes an audio/video file. TRANSCODER_CLASS = avhandling.FFmpegTranscoder # API class for a prgram which extracts thumbnail images from a file. -- 2.20.1