From 3cf4f616f7d4095623ad5eae5db9fb1149ee1238 Mon Sep 17 00:00:00 2001 From: Calin-Andrei Burloiu Date: Tue, 13 Dec 2011 13:51:52 +0200 Subject: [PATCH] CIS: BitTorrent support added; master, worker and queue threading facilities added; working at job worker --- cis/api/avhandling.py | 2 + cis/api/base.py | 26 ++++---- cis/bt.py | 70 +++++++++++++++++++++ cis/cisd.py | 140 ++++++++++++++++++++++++++++++++++++------ cis/config.py | 32 ++++++++-- 5 files changed, 237 insertions(+), 33 deletions(-) create mode 100644 cis/bt.py diff --git a/cis/api/avhandling.py b/cis/api/avhandling.py index 058c9b7..e300075 100644 --- a/cis/api/avhandling.py +++ b/cis/api/avhandling.py @@ -96,6 +96,8 @@ class FFmpegTranscoder(base.BaseTranscoder): log.close() + return output_file + class FFmpegThumbExtractor(base.BaseThumbExtractor): """ diff --git a/cis/api/base.py b/cis/api/base.py index dce1efb..6f41275 100644 --- a/cis/api/base.py +++ b/cis/api/base.py @@ -65,16 +65,18 @@ class BaseTranscoder: """ Transcodes the input file to an audio-video file. - container: possible values are listed in containers member as keys - a_codec: possible values are listed in a_codecs member as keys - v_codec: possible values are listed in v_codecs member as keys - a_bitrate: (numeric) audio bit rate - a_samplingrate: (numeric) audio sampling rate in Hz - a_channels: (numeric) number of audio channels - v_bitrate: (numeric) video bit rate - v_framerate: (numeric) number of frames per second for a video - v_resolution: (string) video image size as x - v_dar: video display aspect ratio as x or float + @param container: possible values are listed in containers member + as keys + @param a_codec possible values are listed in a_codecs member as keys + @param v_codec possible values are listed in v_codecs member as keys + @param a_bitrate (numeric) audio bit rate + @param a_samplingrate (numeric) audio sampling rate in Hz + @param a_channels (numeric) number of audio channels + @param v_bitrate (numeric) video bit rate + @param v_framerate (numeric) number of frames per second for a video + @param v_resolution (string) video image size as x + @param v_dar video display aspect ratio as x or float + @return output file name """ # Check parameters. @@ -113,7 +115,7 @@ class BaseTranscoder: if ext is not None: self.output_file += '.' + ext - self._transcode(self.tr_container(container), + return self._transcode(self.tr_container(container), self.tr_a_codec(a_codec), self.tr_v_codec(v_codec), a_bitrate, a_samplingrate, a_channels, v_bitrate, v_framerate, v_resolution, v_dar) @@ -124,6 +126,8 @@ class BaseTranscoder: """ Called by transcode; must be overridden by a child class which effectively transcodes the input file. + + @return output file name """ pass diff --git a/cis/bt.py b/cis/bt.py new file mode 100644 index 0000000..b6205b6 --- /dev/null +++ b/cis/bt.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python + +from BaseLib.Core.API import * +import tempfile +import random + +def create_torrent(source): + """ + Creates a torrent file for the video source file. + """ + + if isinstance(source, unicode): + usource = source + else: + usource = source.decode(sys.getfilesystemencoding()) + + duration = config.AVINFO_CLASS.get_video_duration(source, True) + + tdef = TorrentDef() + tdef.add_content(usource, playtime=duration) + tdef.set_tracker(config.BT_TRACKER) + + tdef.set_piece_length(32768) + + tdef.finalize() + tdef.save(source + '.tstream') + + +class BitTorrent: + """ + Implementation of BitTorrent operations that uses Next-Share library. + """ + + def __init__(self): + + port = random.randint(10000, 65535) + + # setup session + sscfg = SessionStartupConfig() + statedir = tempfile.mkdtemp() + sscfg.set_state_dir(statedir) + sscfg.set_listen_port(port) + sscfg.set_megacache(False) + sscfg.set_overlay(False) + sscfg.set_dialback(True) + sscfg.set_internal_tracker(False) + + self.session = Session(sscfg) + + def download(self, torrent, output_dir='.'): + """ + Download (leech or seed) a file via BitTorrent. + The code is adapted from Next-Share's 'BaseLib/Tools/cmdlinedl.py'. + + @param torrent .torrent file or URL + """ + + # setup and start download + dscfg = DownloadStartupConfig() + dscfg.set_dest_dir(output_dir); + + if torrent.startswith("http") or torrent.startswith(P2PURL_SCHEME): + tdef = TorrentDef.load_from_url(torrent) + else: + tdef = TorrentDef.load(torrent) + if tdef.get_live(): + raise ValueError("CIS does not support live torrents") + + d = self.session.start_download(tdef, dscfg) + #d.set_state_callback(state_callback, getpeerlist=False) diff --git a/cis/cisd.py b/cis/cisd.py index 3f0dc25..d4c20de 100755 --- a/cis/cisd.py +++ b/cis/cisd.py @@ -1,35 +1,131 @@ #!/usr/bin/env python import sys -import config import os +import time +import threading +from Queue import Queue + +import config +import bt + + +class CIWorker(threading.Thread): + """ + Content Ingestion Worker. A class which executes content ingestion jobs + on a separate thread. + + CIWorker shares a Queue with its master where jobs are submitted. + """ + + def __init__(self, queue): + threading.Thread.__init__(self) + + self.queue = queue + + def run(self): + while True: + job = self.queue.get() + + # * TRANSFER RAW VIDEO IN + file_transfer = config.FILE_TRANSFERER_CLASS( \ + 'tmp/raw', config.INPUT_PATH) + file_transfer.get([job.raw_video]) + file_transfer.close() + + # * TRANSCODE RAW VIDEO + transcoder = config.TRANSCODER_CLASS(input_file = job.raw_video, \ + name = job.name, prog_bin = config.TRANSCODER_BIN) + + # Transcode the raw video in each requested format. + for transcode_config in job.transcode_configs: + transcode_config['output_file'] = transcoder.transcode( \ + container = transcode_config.container, \ + a_codec = transcode_config.a_codec, \ + a_bitrate = transcode_config.a_bitrate, \ + a_samplingrate = transcode_config.a_samplingrate, \ + a_channels = transcode_config.a_channels, \ + v_codec = transcode_config.v_codec, \ + v_bitrate = transcode_config.v_bitrate, \ + v_framerate = transcode_config.v_framerate, \ + v_resolution = transcode_config.v_resolution, \ + v_dar = transcode_config.dar) + + # * EXTRACT THUMBNAIL IMAGES + thumb_extractor = config.THUMB_EXTRACTOR_CLASS( \ + input_file = job.raw_video, name = job.name, \ + prog_bin = config.THUMB_EXTRACTOR_BIN) + # TODO thumbnail extraction type must be got from input + thumb_extractor.extract_random_thumb() + print thumb_extractor.extract_summary_thumbs(5) + -from BaseLib.Core.API import * + queue.task_done() -def create_torrent(source): - """ Creates a torrent file for the video source file. """ - if isinstance(source, unicode): - usource = source - else: - usource = source.decode(sys.getfilesystemencoding()) +class TranscodeConfig: + """ + Structure that contains parameters for a transcoding procedure. + """ - duration = config.AVINFO_CLASS.get_video_duration(source, True) + def __init__(self, container, a_codec, v_codec, + a_bitrate, a_samplingrate, a_channels, + v_bitrate, v_framerate, v_resolution, v_dar): - print config.BT_TRACKER, duration, source + self.container = container + self.a_codec = a_codec + self.v_codec = v_codec + self.a_bitrate = a_bitrate + self.a_samplingrate = a_samplingrate + self.a_channels = a_channels + self.v_bitrate = v_bitrate + self.v_framerate = v_framerate + self.v_resolution = v_resolution + self.v_dar = v_dar - tdef = TorrentDef() - tdef.add_content(usource, playtime=duration) - tdef.set_tracker(config.BT_TRACKER) - tdef.set_piece_length(32768) - tdef.finalize() - tdef.save(source + '.tstream') +class Job: + """ + Structure that contains information about a job. + + Members are documented in the constructor. + """ + + def __init__(self, raw_video, name, transcode_configs): + """ + @param raw_video the input raw video file name transfered from WS + @param name video name (must be a valid file name) + @param transcode_configs a list of TranscodeConfig instances + """ + + self.raw_video = raw_video + self.name = name + self.transcode_configs if __name__ == '__main__': - pass + # Jobs queue. + queue = Queue() + + # Worker thread. + ci_worker = CIWorker(queue) + ci_worker.daemon = True + ci_worker.start() + + while True: + raw_video = sys.stdin.readline().strip() + if raw_video == 'x': + break + + job = Job(raw_video) + queue.put(job) + + queue.join() + + + + # transcoder = config.TRANSCODER_CLASS(sys.argv[1]) # transcoder.transcode('webm', "vorbis", "vp8", a_bitrate="128k", a_samplingrate=22050, a_channels=2, v_bitrate="256k", v_framerate=15, v_resolution="320x240", v_dar="4:3") @@ -43,4 +139,12 @@ if __name__ == '__main__': # #file_transfer.put(['cisd.py']) # file_transfer.close() - create_torrent(sys.argv[1]) +# create_torrent(sys.argv[1]) + +# bt_inst = bt.BitTorrent() +# +# bt_inst.download(sys.argv[1], '/tmp') +# bt_inst.download(sys.argv[2], '/tmp') +# +# print threading.active_count(), threading.enumerate() +# time.sleep(30) diff --git a/cis/config.py b/cis/config.py index e3b1cca..3e9ff69 100644 --- a/cis/config.py +++ b/cis/config.py @@ -4,16 +4,40 @@ from api import avhandling from api import file_transfer -# BitTorrent configurations. + +# === FILE TRANSFER CONFIGURATIONS === +# Path from the Web Server where the raw input video file is stored. +INPUT_PATH = 'upload' +# Path from the Web Server where the output torrent files will be stored. +OUTPUT_TORRENTS_PATH = 'torrents' +# Path from the Web Server where the output thumbnail image files will be +# stored. +OUTPUT_THUMBS_PATH = 'thumbs' + + +# === BITTORRENT CONFIGURATIONS === BT_TRACKER = "http://p2p-next-10.grid.pub.ro:6969/announce" -# External programs API classes. + +# === EXTERNAL PROGRAMS API CLASSES === +# API class for a prgram which retrives audio/video information, like duration. AVINFO_CLASS = avhandling.FFmpegAVInfo +# API class for a prgram which transcodes an audio/video file. TRANSCODER_CLASS = avhandling.FFmpegTranscoder +# API class for a prgram which extracts thumbnail images from a file. THUMB_EXTRACTOR_CLASS = avhandling.FFmpegThumbExtractor +# API class for a prgram which transfers files between Web Server and CIS. FILE_TRANSFERER_CLASS = file_transfer.FTPFileTransferer -# External programs binary file. None means default. + +# === EXTERNAL PROGRAMS BINARY FILES +# Set this values to None if you want default values provided by the API +# class to be used. +# Binary of a prgram which retrives audio/video information, like duration. +AVINFO_BIN = None +# Binary of a prgram which transcodes an audio/video file. TRANSCODER_BIN = None -THUMB_EXTRACTER_BIN = None +# Binary of a prgram which extracts thumbnail images from a file. +THUMB_EXTRACTOR_BIN = None +# Binary of a prgram which transfers files between Web Server and CIS. FILE_TRANSFERER_BIN = None -- 2.20.1