From 080b37a97e93691b3ba1c4aa3982a143167115a7 Mon Sep 17 00:00:00 2001 From: =?utf8?q?C=C4=83lin-Andrei=20Burloiu?= Date: Mon, 13 Feb 2012 19:58:35 +0200 Subject: [PATCH] cis notified web server of a job completion; upload form interface and validation finished --- application/config/form_validation.php | 22 +++ application/config/p2p-tube.php | 2 +- application/controllers/video.php | 151 ++++++++++++++++++ .../language/english/form_validation_lang.php | 5 + application/language/english/ui_lang.php | 3 + application/language/english/user_lang.php | 1 - application/language/english/video_lang.php | 11 ++ application/views/header.php | 4 + application/views/user/register_view.php | 2 +- application/views/video/upload_view.php | 44 +++++ cis/api/base.py | 19 ++- cis/api/ffmpeg.py | 10 +- cis/api/ftp.py | 19 ++- cis/bt.py | 14 ++ cis/cis.py | 49 +++++- .../api_exceptions.py => cis_exceptions.py} | 3 + cis/config.py | 9 +- cis/job_test.json | 2 +- cis/post-json.sh | 2 +- 19 files changed, 341 insertions(+), 31 deletions(-) create mode 100644 application/views/video/upload_view.php rename cis/{api/api_exceptions.py => cis_exceptions.py} (89%) diff --git a/application/config/form_validation.php b/application/config/form_validation.php index 67ed55c..85c5ca8 100644 --- a/application/config/form_validation.php +++ b/application/config/form_validation.php @@ -124,6 +124,28 @@ $config = array( 'label'=>'lang:video_comment', 'rules'=>'trim|required|xss_clean|callback__is_user_loggedin' ) + ), + 'upload'=> array( + array( + 'field'=>'video-upload-file', + 'label'=>'lang:video_upload_file', + 'rules'=>'callback__required_upload|callback__valid_upload' + ), + array( + 'field'=>'video-title', + 'label'=>'lang:video_title', + 'rules'=>'trim|required|xss_clean' + ), + array( + 'field'=>'video-description', + 'label'=>'lang:video_description', + 'rules'=>'trim|required|xss_clean' + ), + array( + 'field'=>'video-tags', + 'label'=>'lang:video_tags', + 'rules'=>'required|callback__valid_tags' + ) ) ); diff --git a/application/config/p2p-tube.php b/application/config/p2p-tube.php index a35cddc..bcbe0f5 100644 --- a/application/config/p2p-tube.php +++ b/application/config/p2p-tube.php @@ -155,4 +155,4 @@ $config['available_languages_list'] = array( | like account activation e-mail or password recovery. | */ -$config['noreply_email'] = 'no-reply@p2p-next.cs.pub.ro'; \ No newline at end of file +$config['noreply_email'] = 'no-reply@p2p-next.cs.pub.ro'; diff --git a/application/controllers/video.php b/application/controllers/video.php index 4e31540..eb4ed80 100644 --- a/application/controllers/video.php +++ b/application/controllers/video.php @@ -117,6 +117,109 @@ class Video extends CI_Controller { $this->load->view('footer'); $this->load->view('html_end'); } + +// public function upload() +// { +// $this->load->library('form_validation'); +// +// $this->form_validation->set_error_delimiters('', +// ''); +// $error_upload = ''; +// +// if ($this->form_validation->run('upload')) +// { +// if ($_FILES['video-upload-file']['tmp_name']) +// { +// // Upload library +// $config_upload['upload_path'] = './data/upload'; +// $this->load->library('upload', $config_upload); +// +// $b_validation = $this->upload->do_upload('video-upload-file'); +// $error_upload = +// $this->upload->display_errors('', +// ''); +// } +// else +// { +// $b_validation = FALSE; +// } +// } +// else +// $b_validation = FALSE; +// +// if ($b_validation === FALSE) +// { +// $params = array('title' => +// $this->lang->line('ui_nav_menu_upload') +// .' – ' +// . $this->config->item('site_name'), +// //'metas' => array('description'=>'') +// ); +// $this->load->library('html_head_params', $params); +// +// // ** +// // ** LOADING VIEWS +// // ** +// $this->load->view('html_begin', $this->html_head_params); +// $this->load->view('header', +// array('selected_menu' => 'upload')); +// +// $main_params['content'] = $this->load->view( +// 'video/upload_view', +// array('error_upload'=> $error_upload), +// TRUE); +// $main_params['side'] = $this->load->view('side_default', NULL, TRUE); +// $this->load->view('main', $main_params); +// +// $this->load->view('footer'); +// $this->load->view('html_end'); +// } +// else +// { +// +// } +// } + + public function upload() + { + $this->load->library('form_validation'); + + $this->form_validation->set_error_delimiters('', + ''); + $error_upload = ''; + + if ($this->form_validation->run('upload') === FALSE) + { + $params = array('title' => + $this->lang->line('ui_nav_menu_upload') + .' – ' + . $this->config->item('site_name'), + //'metas' => array('description'=>'') + ); + $this->load->library('html_head_params', $params); + + // ** + // ** LOADING VIEWS + // ** + $this->load->view('html_begin', $this->html_head_params); + $this->load->view('header', + array('selected_menu' => 'upload')); + + $main_params['content'] = $this->load->view( + 'video/upload_view', + array('error_upload'=> $error_upload), + TRUE); + $main_params['side'] = $this->load->view('side_default', NULL, TRUE); + $this->load->view('main', $main_params); + + $this->load->view('footer'); + $this->load->view('html_end'); + } + else + { + + } + } /** * Increments (dis)likes count for video with the specified id and returns to @@ -233,6 +336,54 @@ class Video extends CI_Controller { $this->videos_model->comment_video($video_id, $user_id, $comment); } + public function _valid_tags($tags) + { + $tok = strtok($tags, ','); + while ($tok != FALSE) + { + $tok = trim($tok); + if (!ctype_alnum($tok)) + return FALSE; + + $tok = strtok(','); + } + + return TRUE; + } + + public function _required_upload($file) + { + if ($_FILES['video-upload-file']['tmp_name']) + { + return TRUE; + } + + return FALSE; + } + + public function _valid_upload($file) + { + if ($_FILES['video-upload-file']['tmp_name']) + { + // Upload library + $config_upload = array(); + $config_upload['upload_path'] = './data/upload'; + $config_upload['allowed_types'] = '*'; + $this->load->library('upload', $config_upload); + + if ($this->upload->do_upload('video-upload-file')) + { + return TRUE; + } + + $this->form_validation->set_message('_valid_upload', + $this->upload->display_errors('', + '')); + } + + return FALSE; + } + /** * OBSOLETE: AJAX page which retrieves a video plugin. * diff --git a/application/language/english/form_validation_lang.php b/application/language/english/form_validation_lang.php index b8b6d29..bb6fba1 100644 --- a/application/language/english/form_validation_lang.php +++ b/application/language/english/form_validation_lang.php @@ -27,5 +27,10 @@ $lang['_do_recover_password'] = 'Username and e-mail address are not associated // Comment Video $lang['_is_user_loggedin'] = 'In order to comment a video you must be logged in.'; +// Upload Video +$lang['_valid_tags'] = 'Tags are alphanumeric words separated by commas.'; +$lang['_required_upload'] = 'The %s field is required.'; +$lang['_valid_upload'] = 'The uploaded video file is not supported.'; + /* End of file form_validation_lang.php */ /* Location: ./system/language/english/form_validation_lang.php */ \ No newline at end of file diff --git a/application/language/english/ui_lang.php b/application/language/english/ui_lang.php index 4d6d3dc..1d59b75 100644 --- a/application/language/english/ui_lang.php +++ b/application/language/english/ui_lang.php @@ -6,6 +6,7 @@ $lang['ui_nav_menu_install_plugins'] = 'Install Plugins'; $lang['ui_nav_menu_about'] = 'About'; $lang['ui_nav_menu_help'] = 'Help'; $lang['ui_nav_menu_contact'] = 'Contact'; +$lang['ui_nav_menu_upload'] = 'Upload'; $lang['ui_nav_menu_log_in'] = 'Log In'; $lang['ui_nav_menu_register'] = 'Register'; $lang['ui_nav_menu_account'] = 'Account'; @@ -69,5 +70,7 @@ $lang['ui_captcha'] = 'CAPTCHA'; $lang['ui_captcha_instructions'] = 'Please insert the text from the image below in order to demonstrate that you are a human:'; $lang['ui_change_captcha'] = 'Change CAPTCHA image'; +$lang['ui_required_fields'] = '* Required fields!'; + /* End of file ui_lang.php */ /* Location: ./application/language/english/ui_lang.php */ \ No newline at end of file diff --git a/application/language/english/user_lang.php b/application/language/english/user_lang.php index 898e0ae..39eb4db 100644 --- a/application/language/english/user_lang.php +++ b/application/language/english/user_lang.php @@ -33,7 +33,6 @@ $lang['user_video_prefs'] = 'Video Preferences'; $lang['user_registration_date'] = 'Registration Date'; $lang['user_last_login'] = 'Last Login Time'; -$lang['user_note_required_fields'] = '* Required fields!'; $lang['user_submit_register'] = 'Register'; $lang['user_submit_save'] = 'Save'; diff --git a/application/language/english/video_lang.php b/application/language/english/video_lang.php index fbe373e..056b751 100644 --- a/application/language/english/video_lang.php +++ b/application/language/english/video_lang.php @@ -1,5 +1,6 @@ >lang->line('ui_nav_menu_account') ?> +
  • >lang->line('ui_nav_menu_upload') ?> +
    • +
  • lang->line('ui_hello'). ', '. $username. '!   ' ?>
    • diff --git a/application/views/user/register_view.php b/application/views/user/register_view.php index aeb6dc5..eedc199 100644 --- a/application/views/user/register_view.php +++ b/application/views/user/register_view.php @@ -20,7 +20,7 @@ else - + diff --git a/application/views/video/upload_view.php b/application/views/video/upload_view.php new file mode 100644 index 0000000..702251a --- /dev/null +++ b/application/views/video/upload_view.php @@ -0,0 +1,44 @@ + +
    lang->line('user_note_required_fields') ?>lang->line('ui_required_fields') ?>
     
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    lang->line('ui_required_fields') ?>
     
    lang->line('video_upload_file') ?> * :
     
    lang->line('video_title') ?> * :
    lang->line('video_description') ?> * :
    lang->line('video_tags') ?> * : (lang->line('video_tags_hint') ?>)
     
    + \ No newline at end of file diff --git a/cis/api/base.py b/cis/api/base.py index 500aada..b4e18a8 100644 --- a/cis/api/base.py +++ b/cis/api/base.py @@ -8,7 +8,7 @@ import os import re import random -import api_exceptions +import cis_exceptions import cis_util class BaseTranscoder: @@ -109,6 +109,10 @@ class BaseTranscoder: raise ValueError('Video display aspect ratio must be a float or a string like :.') self.output_file = os.path.join(self.dest_path, self.name) + if os.path.exists(self.output_file): + raise cis_exceptions.FileAlreadyExistsException( \ + 'file "%s" already exists' % self.output_file) + if v_resolution is not None: self.output_file += '_' self.output_file += v_resolution[(v_resolution.rindex('x')+1):] @@ -137,7 +141,7 @@ class BaseTranscoder: """ Translates container API name into external program identifier.""" if not self.containers.has_key(name) or self.containers[name] is None: - raise api_exceptions.NotImplementedException("Container " + name \ + raise cis_exceptions.NotImplementedException("Container " + name \ + "not implemented") return self.containers[name] @@ -159,7 +163,7 @@ class BaseTranscoder: """ Translates audio codec API name into external program identifier.""" if not self.a_codecs.has_key(name) or self.a_codecs[name] is None: - raise api_exceptions.NotImplementedException("Audio Codec " + name \ + raise cis_exceptions.NotImplementedException("Audio Codec " + name \ + "not implemented") return self.a_codecs[name] @@ -168,7 +172,7 @@ class BaseTranscoder: """ Translates video codec API name into external program identifier.""" if not self.v_codecs.has_key(name) or self.v_codecs[name] is None: - raise api_exceptions.NotImplementedException("Video Codec " + name \ + raise cis_exceptions.NotImplementedException("Video Codec " + name \ + "not implemented") return self.v_codecs[name] @@ -231,7 +235,7 @@ class BaseThumbExtractor: thumb_extracted = True try: self.extract_thumb(seek_pos, resolution, n_thumbs_extracted) - except api_exceptions.ThumbExtractionException as e: + except cis_exceptions.ThumbExtractionException as e: thumb_extracted = False if thumb_extracted: @@ -245,6 +249,11 @@ class BaseThumbExtractor: """ Returns the name required as output file name based on index. """ output_file_name = os.path.join(self.dest_path, self.name) \ + '_t' + ("%02d" % index) + '.jpg' + + if os.path.exists(output_file_name): + raise cis_exceptions.FileAlreadyExistsException( \ + 'file "%s" already exists' % output_file_name) + return output_file_name def get_video_duration(self): diff --git a/cis/api/ffmpeg.py b/cis/api/ffmpeg.py index 27c9a5d..1974497 100644 --- a/cis/api/ffmpeg.py +++ b/cis/api/ffmpeg.py @@ -7,7 +7,7 @@ videos and thumbnail extraction from videos using FFmpeg CLI program. """ import base -import api_exceptions +import cis_exceptions import subprocess import re import os @@ -91,7 +91,7 @@ class FFmpegTranscoder(base.BaseTranscoder): exit_code = p.wait() if exit_code > 0: - raise api_exceptions.TranscodingException( \ + raise cis_exceptions.TranscodingException( \ 'FFmpeg exited with code ' + str(exit_code) + '.') log.close() @@ -133,14 +133,14 @@ class FFmpegThumbExtractor(base.BaseThumbExtractor): exit_code = p.wait() if exit_code > 0: - raise api_exceptions.ThumbExtractionException( \ + raise cis_exceptions.ThumbExtractionException( \ 'FFmpeg exited with code ' + str(exit_code) + '.') # FFmpeg bug: when no key frame is found from seek_pos to the # end of file an empty image file is created. if os.path.getsize(output_file) == 0L: os.unlink(output_file) - raise api_exceptions.ThumbExtractionException( \ + raise cis_exceptions.ThumbExtractionException( \ 'FFmpeg created an empty file.') def get_video_duration(self): @@ -194,5 +194,5 @@ class FFprobeAVInfo(base.BaseAVInfo): exit_code = p.wait() if exit_code > 0: - raise api_exceptions.AVInfoException( \ + raise cis_exceptions.AVInfoException( \ 'ffprobe exited with code ' + str(exit_code) + '.') diff --git a/cis/api/ftp.py b/cis/api/ftp.py index 567978f..3798fd3 100644 --- a/cis/api/ftp.py +++ b/cis/api/ftp.py @@ -11,7 +11,7 @@ import ftplib import base import ftp_config import socket -import api_exceptions +import cis_exceptions import os import logger @@ -35,7 +35,7 @@ class FTPFileTransferer(base.BaseFileTransferer): try: self.ftp.cwd(self.remote_path) except ftplib.error_perm as e: - raise api_exceptions.FileTransferException( \ + raise cis_exceptions.FileTransferException( \ "Could not change remote directory '%s': %s" \ % (self.remote_path, repr(e))) @@ -43,10 +43,15 @@ class FTPFileTransferer(base.BaseFileTransferer): for crt_fn in files: local_fn = os.path.join(self.local_path, crt_fn) remote_fn = os.path.join(self.remote_path, crt_fn) + + if os.path.exists(local_fn): + raise cis_exceptions.FileAlreadyExistsException( \ + 'file "%s" already exists' % local_fn) + try: file_local = open(local_fn, 'wb') except IOError as e: - raise api_exceptions.FileTransferException( \ + raise cis_exceptions.FileTransferException( \ "Could not open local file '%s' for writing: %s" \ % (local_fn, repr(e))) @@ -54,7 +59,7 @@ class FTPFileTransferer(base.BaseFileTransferer): self.ftp.retrbinary('RETR %s' % crt_fn, file_local.write) file_local.close() except ftplib.error_perm as e: - raise api_exceptions.FileTransferException( \ + raise cis_exceptions.FileTransferException( \ "Could not get file '%s' from Web Server: %s" \ % (remote_fn, repr(e))) @@ -62,7 +67,7 @@ class FTPFileTransferer(base.BaseFileTransferer): try: self.ftp.cwd(self.remote_path) except ftplib.error_perm as e: - raise api_exceptions.FileTransferException( \ + raise cis_exceptions.FileTransferException( \ "Could not change remote directory '%s': %s" \ % (self.remote_path, repr(e))) @@ -72,7 +77,7 @@ class FTPFileTransferer(base.BaseFileTransferer): try: file_local = open(local_fn, 'rb') except IOError as e: - raise api_exceptions.FileTransferException( \ + raise cis_exceptions.FileTransferException( \ "Could not open local file '%s' for reading: %s" \ % (local_fn, repr(e))) @@ -80,7 +85,7 @@ class FTPFileTransferer(base.BaseFileTransferer): self.ftp.storbinary('STOR %s' % crt_fn, file_local) file_local.close() except ftplib.error_perm as e: - raise api_exceptions.FileTransferException( \ + raise cis_exceptions.FileTransferException( \ "Could not put file '%s' to Web Server: %s" \ % (local_fn, repr(e))) diff --git a/cis/bt.py b/cis/bt.py index 74d43ab..894ea94 100644 --- a/cis/bt.py +++ b/cis/bt.py @@ -101,3 +101,17 @@ class BitTorrent: self.session.remove_download(dl, remove_content) logger.log_msg('torrent "%s" stopped' % torrent) break + + def get_torrent_list(self): + """ + Returns a list of all torrents started. + """ + + downloads = self.session.get_downloads() + torrent_list = [] + + for dl in downloads: + tdef = dl.get_def() + torrent_list.append(tdef.get_name() + '.tstream') + + return torrent_list diff --git a/cis/cis.py b/cis/cis.py index f88aad5..3ae47cf 100755 --- a/cis/cis.py +++ b/cis/cis.py @@ -10,11 +10,13 @@ from Queue import Queue import web import json from web.wsgiserver import CherryPyWSGIServer +import urllib import config import bt import users import logger +import cis_exceptions if config.SECURITY: CherryPyWSGIServer.ssl_certificate = "cacert.pem" @@ -149,26 +151,41 @@ class CIWorker(threading.Thread): for f in files: os.unlink(os.path.join(path, f)) + def notify_completion(self): + logger.log_msg('#%s: notifying web server about the job completion...'\ + % self.job_id) + + f = urllib.urlopen(config.WS_COMPLETION) + f.read() + def run(self): while True: job = Server.queue.get() - self.job_id = job['id'] + self.job_id = job['code'] # * TRANSFER RAW VIDEO IN try: self.transfer_in(job['raw_video']) + except cis_exceptions.FileAlreadyExistsException as e: + logger.log_msg('#%s: %s' \ + % (job['code'], repr(e)), logger.LOG_LEVEL_ERROR) + continue except Exception as e: logger.log_msg('#%s: error while transferring in: %s' \ - % (job['id'], str(e)), logger.LOG_LEVEL_FATAL) + % (job['code'], repr(e)), logger.LOG_LEVEL_FATAL) continue # * TRANSCODE RAW VIDEO try: self.transcode(job['raw_video'], job['name'], \ job['transcode_configs']) + except cis_exceptions.FileAlreadyExistsException as e: + logger.log_msg('#%s: %s' \ + % (job['code'], repr(e)), logger.LOG_LEVEL_ERROR) + continue except Exception as e: logger.log_msg('#%s: error while transcoding: %s' \ - % (job['id'], str(e)), logger.LOG_LEVEL_FATAL) + % (job['code'], repr(e)), logger.LOG_LEVEL_FATAL) continue # * EXTRACT THUMBNAIL IMAGES @@ -176,10 +193,14 @@ class CIWorker(threading.Thread): try: self.extract_thumbs(job['raw_video'], job['name'], \ job['thumbs']) + except cis_exceptions.FileAlreadyExistsException as e: + logger.log_msg('#%s: %s' \ + % (job['code'], repr(e)), logger.LOG_LEVEL_ERROR) + continue except Exception as e: logger.log_msg( \ '#%s: error while extracting thumbnail images: %s' \ - % (job['id'], str(e)), logger.LOG_LEVEL_FATAL) + % (job['code'], repr(e)), logger.LOG_LEVEL_FATAL) continue # * CREATE TORRENTS AND START SEEDING OF TRANSCODED VIDEOS @@ -205,7 +226,17 @@ class CIWorker(threading.Thread): config.WS_THUMBS_PATH) except Exception as e: logger.log_msg('#%s: error while transferring out: %s' \ - % (job['id'], str(e)), logger.LOG_LEVEL_FATAL) + % (job['code'], repr(e)), logger.LOG_LEVEL_FATAL) + continue + + # * NOTIFY WEB SERVER ABOUT CONTENT INGESTION COMPLETION + # TODO in the future web server should also be notified about errors + try: + self.notify_completion() + except Exception as e: + logger.log_msg( + '#%s: error while notifying web server about the job completion: %s' \ + % (job['code'], repr(e)), logger.LOG_LEVEL_FATAL) continue # * CLEANUP RAW VIDEOS AND THUMBNAIL IMAGES @@ -236,13 +267,17 @@ class Server: resp = {"load": Server.load} web.header('Content-Type', 'application/json') return json.dumps(resp) + elif request == 'get_torrent_list': + resp = Server.bit_torrent.get_torrent_list() + web.header('Content-Type', 'application/json') + return json.dumps(resp) #elif request == 'shutdown': - #sys.exit(0) + #exit(0) elif request == 'test': return '' else: web.badrequest() - return "" + return '' def POST(self, request): diff --git a/cis/api/api_exceptions.py b/cis/cis_exceptions.py similarity index 89% rename from cis/api/api_exceptions.py rename to cis/cis_exceptions.py index a8e4f78..11fbf8a 100644 --- a/cis/api/api_exceptions.py +++ b/cis/cis_exceptions.py @@ -11,6 +11,9 @@ class NotImplementedException(Exception): def __str__(self): return repr(self.value) +class FileAlreadyExistsException(Exception): + pass + class TranscodingException(Exception): pass diff --git a/cis/config.py b/cis/config.py index 094e552..f19604c 100644 --- a/cis/config.py +++ b/cis/config.py @@ -23,14 +23,19 @@ WS_TORRENTS_PATH = 'devel/data/torrents' WS_THUMBS_PATH = 'devel/data/thumbs' -# === BITTORRENT CONFIGURATIONS === +# === URLS === #BT_TRACKER = "http://p2p-next-10.grid.pub.ro:6969/announce" +# BitTorrent tracker URL. BT_TRACKER = 'http://p2p-next-10.grid.pub.ro:6969/announce' +# Web server's URL for content ingestion completion. +WS_COMPLETION = 'http://p2p-next-03.grid.pub.ro:8081/cis_completion' +#WS_COMPLETION = 'http://koala.cs.pub.ro/video/cis_completion' +# === CIS PATHS === RAW_VIDEOS_PATH = 'tmp/raw' -MEDIA_PATH = 'tmp/media' THUMBS_PATH = 'tmp/thumbs' +MEDIA_PATH = '/home/p2p/export/p2p-tube/media' # In a distributed file system for multi-CIS. TORRENTS_PATH = '/home/p2p/export/p2p-tube/torrents' diff --git a/cis/job_test.json b/cis/job_test.json index e63e8b8..f148510 100644 --- a/cis/job_test.json +++ b/cis/job_test.json @@ -1,5 +1,5 @@ { - "id": "33", + "code": "33", "raw_video": "test.ogv", "name": "test", "weight": 3, diff --git a/cis/post-json.sh b/cis/post-json.sh index 1132520..aaca00e 100755 --- a/cis/post-json.sh +++ b/cis/post-json.sh @@ -6,6 +6,6 @@ if [ $# -ne 1 ]; then fi JSON_FILE="$1" -CIS_URL="http://p2p-next-02.grid.pub.ro:8080/" +CIS_URL="http://p2p-next-03.grid.pub.ro:8080/" curl -H 'Content-Type: application/json' --data-binary @"$JSON_FILE" ${CIS_URL}ingest_content -- 2.20.1