From 42b8f76ab32990f2668a4e3346374de7bad91be2 Mon Sep 17 00:00:00 2001 From: Calin-Andrei Burloiu Date: Wed, 21 Dec 2011 12:03:32 +0200 Subject: [PATCH] CIS: CIWorker works; now we need communication via web services --- cis/api/file_transfer.py | 24 +++++++++++--- cis/cisd.py | 70 +++++++++++++++++++++------------------- 2 files changed, 56 insertions(+), 38 deletions(-) diff --git a/cis/api/file_transfer.py b/cis/api/file_transfer.py index b7435d6..0425280 100644 --- a/cis/api/file_transfer.py +++ b/cis/api/file_transfer.py @@ -30,6 +30,14 @@ class FTPFileTransferer(base.BaseFileTransferer): self.ftp.set_pasv(True) def get(self, files): + try: + self.ftp.cwd(self.remote_path) + except ftplib.error_perm as e: + raise api_exceptions.FileTransferException( \ + "Could not change remote directory '%s': %s" \ + % (self.remote_path, repr(e))) + + for crt_fn in files: local_fn = os.path.join(self.local_path, crt_fn) remote_fn = os.path.join(self.remote_path, crt_fn) @@ -41,7 +49,6 @@ class FTPFileTransferer(base.BaseFileTransferer): % (local_fn, repr(e))) try: - self.ftp.cwd(self.remote_path) self.ftp.retrbinary('RETR %s' % crt_fn, file_local.write) file_local.close() except ftplib.error_perm as e: @@ -50,9 +57,15 @@ class FTPFileTransferer(base.BaseFileTransferer): % (remote_fn, repr(e))) def put(self, files): + try: + self.ftp.cwd(self.remote_path) + except ftplib.error_perm as e: + raise api_exceptions.FileTransferException( \ + "Could not change remote directory '%s': %s" \ + % (self.remote_path, repr(e))) + for crt_fn in files: local_fn = os.path.join(self.local_path, crt_fn) - remote_fn = os.path.join(self.local_path, crt_fn) try: file_local = open(local_fn, 'rb') @@ -62,13 +75,14 @@ class FTPFileTransferer(base.BaseFileTransferer): % (local_fn, repr(e))) try: - self.ftp.cwd(self.remote_path) + print('remote_path=' + self.remote_path \ + + '; crt_fn=' + crt_fn) self.ftp.storbinary('STOR %s' % crt_fn, file_local) file_local.close() except ftplib.error_perm as e: raise api_exceptions.FileTransferException( \ - "Could not get file '%s' from Web Server: %s" \ - % (remote_fn, repr(e))) + "Could not put file '%s' to Web Server: %s" \ + % (local_fn, repr(e))) def close(self): if self.ftp is not None: diff --git a/cis/cisd.py b/cis/cisd.py index b042817..483017b 100755 --- a/cis/cisd.py +++ b/cis/cisd.py @@ -2,6 +2,7 @@ import sys import os +import fnmatch import shutil import time import threading @@ -113,7 +114,7 @@ class CIWorker(threading.Thread): @param transcode_configs a list of dictionaries with format settings """ - for transcode_config in transcode_cofigs: + for transcode_config in transcode_configs: # * CREATE TORRENTS FOR EACH TRANSCODED VIDEO # Create torrent file. bt.create_torrent(transcode_config['output_file']) @@ -123,10 +124,13 @@ class CIWorker(threading.Thread): shutil.move(transcode_config['output_file'] + '.tstream', \ self.torrents_dir) + output_file = transcode_config['output_file'] + '.tstream' + output_file = output_file[(output_file.rindex('/') + 1):] + # * SEED TORRENTS bit_torrent.start_download( \ - transcode_config['output_file'] + '.tstream', - self_transcoded_videos_dir) + os.path.join(self.torrents_dir, output_file), + self.transcoded_videos_dir) print '** Creating torrents and seeding finished.' @@ -145,7 +149,7 @@ class CIWorker(threading.Thread): print '** Creating torrents and seeding finished.' - def remove_file(self, files, path): + def remove_files(self, files, path): """ Deletes files from a specified path. """ @@ -171,35 +175,35 @@ class CIWorker(threading.Thread): self.extract_thumbs(job['raw_video'], job['name'], \ job['thumbs']) -# # * CREATE TORRENTS AND START SEEDING OF TRANSCODED VIDEOS -# self.seed(job['transcode_configs']) -# -# # Torrent files. -# files = [f for f in os.listdir(self.torrents_dir) \ -# if os.path.isfile(os.path.join( \ -# self.torrents_dir, f))] -# torrent_files = fnmatch.filter(files, name + "_*") -# -# # Thumbnail images files. -# files = [f for f in os.listdir(self.thumbs_dir) \ -# if os.path.isfile(os.path.join( \ -# self.thumbs_dir, f))] -# thumb_files = fnmatch.filter(files, name + "_*") -# -# # Raw video files. -# raw_files = [f for f in os.listdir(self.raw_videos_dir) \ -# if os.path.isfile(os.path.join( \ -# self.raw_videos_dir, f))] -# -# # * TRANSFER TORRENTS AND THUMBNAIL IMAGES OUT -# self.transfer_out(torrent_files, self.torrents_dir, \ -# config.OUTPUT_TORRENTS_PATH) -# self.transfer_out(thumb_files, self.thumbs_dir, \ -# config.OUTPUT_THUMBS_PATH) -# -# # * CLEANUP RAW VIDEOS AND THUMBNAIL IMAGES -# self.remove_files(raw_files, self.raw_videos_dir) -# self.remove_files(thumb_files, self.thumbs_dir) + # * CREATE TORRENTS AND START SEEDING OF TRANSCODED VIDEOS + self.seed(job['transcode_configs']) + + # Torrent files. + files = [f for f in os.listdir(self.torrents_dir) \ + if os.path.isfile(os.path.join( \ + self.torrents_dir, f))] + torrent_files = fnmatch.filter(files, name + "_*") + + # Thumbnail images files. + files = [f for f in os.listdir(self.thumbs_dir) \ + if os.path.isfile(os.path.join( \ + self.thumbs_dir, f))] + thumb_files = fnmatch.filter(files, name + "_*") + + # Raw video files. + raw_files = [f for f in os.listdir(self.raw_videos_dir) \ + if os.path.isfile(os.path.join( \ + self.raw_videos_dir, f))] + + # * TRANSFER TORRENTS AND THUMBNAIL IMAGES OUT + self.transfer_out(torrent_files, self.torrents_dir, \ + config.OUTPUT_TORRENTS_PATH) + self.transfer_out(thumb_files, self.thumbs_dir, \ + config.OUTPUT_THUMBS_PATH) + + # * CLEANUP RAW VIDEOS AND THUMBNAIL IMAGES + self.remove_files(raw_files, self.raw_videos_dir) + self.remove_files(thumb_files, self.thumbs_dir) # * JOB FINISHED queue.task_done() -- 2.20.1