/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: Ricardo Garcia Gonzalez
# Author: Danny Colligan
# License: Public domain code
import htmlentitydefs
import httplib
import locale
import math
import netrc
import os
import os.path
import re
import socket
import string
import sys
import time
import urllib
import urllib2
std_headers = {
'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.5)
Gecko/2008120122 Firefox/3.0.5',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
'Accept':
'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,i
mage/png,*/*;q=0.5',
'Accept-Language': 'en-us,en;q=0.5',
}
simple_title_chars = string.ascii_letters.decode('ascii') +
string.digits.decode('ascii')
class DownloadError(Exception):
"""Download Error exception.
class SameFileError(Exception):
"""Same File exception.
class PostProcessingError(Exception):
"""Post Processing exception.
Available options:
_params = None
_ies = []
_pps = []
@staticmethod
def pmkdir(filename):
"""Create directory components in filename. Similar to Unix "mkdir
-p"."""
components = filename.split(os.sep)
aggregate = [os.sep.join(components[0:x]) for x in xrange(1,
len(components))]
aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names
with separator
for dir in aggregate:
if not os.path.exists(dir):
os.mkdir(dir)
@staticmethod
def format_bytes(bytes):
if bytes is None:
return 'N/A'
if bytes == 0:
exponent = 0
else:
exponent = long(math.log(float(bytes), 1024.0))
suffix = 'bkMGTPEZY'[exponent]
converted = float(bytes) / float(1024**exponent)
return '%.2f%s' % (converted, suffix)
@staticmethod
def calc_percent(byte_counter, data_len):
if data_len is None:
return '---.-%'
return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) *
100.0))
@staticmethod
def calc_eta(start, now, total, current):
if total is None:
return '--:--'
dif = now - start
if current == 0 or dif < 0.001: # One millisecond
return '--:--'
rate = float(current) / dif
eta = long((float(total) - float(current)) / rate)
(eta_mins, eta_secs) = divmod(eta, 60)
if eta_mins > 99:
return '--:--'
return '%02d:%02d' % (eta_mins, eta_secs)
@staticmethod
def calc_speed(start, now, bytes):
dif = now - start
if bytes == 0 or dif < 0.001: # One millisecond
return '%10s' % '---b/s'
return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) /
dif))
@staticmethod
def best_block_size(elapsed_time, bytes):
new_min = max(bytes / 2.0, 1.0)
new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
if elapsed_time < 0.001:
return int(new_max)
rate = bytes / elapsed_time
if rate > new_max:
return int(new_max)
if rate < new_min:
return int(new_min)
return int(rate)
@staticmethod
def parse_bytes(bytestr):
"""Parse a string indicating a byte quantity into a long integer."""
matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
if matchobj is None:
return None
number = float(matchobj.group(1))
multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
return long(round(number * multiplier))
def get_params(self):
"""Get parameters."""
return self._params
def fixed_template(self):
"""Checks if the output template is fixed."""
return (re.search(ur'(?u)%\(.+?\)s', self._params['outtmpl']) is None)
def report_finish(self):
"""Report download finished."""
self.to_stdout(u'')
try:
filename = self._params['outtmpl'] % result
self.report_destination(filename)
except (ValueError, KeyError), err:
retcode = self.trouble('ERROR: invalid output
template or system charset: %s' % str(err))
continue
if self._params['nooverwrites'] and
os.path.exists(filename):
self.to_stderr('WARNING: file exists: %s;
skipping' % filename)
continue
try:
self.pmkdir(filename)
except (OSError, IOError), err:
retcode = self.trouble('ERROR: unable to create
directories: %s' % str(err))
continue
try:
outstream = open(filename, 'wb')
except (OSError, IOError), err:
retcode = self.trouble('ERROR: unable to open
for writing: %s' % str(err))
continue
try:
self._do_download(outstream, result['url'])
outstream.close()
except (OSError, IOError), err:
retcode = self.trouble('ERROR: unable to write
video data: %s' % str(err))
continue
except (urllib2.URLError, httplib.HTTPException,
socket.error), err:
retcode = self.trouble('ERROR: unable to
download video data: %s' % str(err))
continue
try:
self.post_process(filename, result)
except (PostProcessingError), err:
retcode = self.trouble('ERROR: postprocessing:
%s' % str(err))
continue
break
if not suitable_found:
retcode = self.trouble('ERROR: no suitable InfoExtractor:
%s' % url)
return retcode
self.report_finish()
if data_len is not None and str(byte_counter) != data_len:
raise ValueError('Content too short: %s/%s bytes' %
(byte_counter, data_len))
class InfoExtractor(object):
"""Information Extractor class.
_ready = False
_downloader = None
@staticmethod
def suitable(url):
"""Receives a URL and returns True if suitable for this IE."""
return False
def initialize(self):
"""Initializes an instance (authentication, etc)."""
if not self._ready:
self._real_initialize()
self._ready = True
def _real_initialize(self):
"""Real initialization process. Redefine in subclasses."""
pass
class YoutubeIE(InfoExtractor):
"""Information extractor for youtube.com."""
_VALID_URL = r'^((?:http://)?(?:\w+\.)?youtube\.com/(?:(?:v/)|
(?:(?:watch(?:\.php)?)?\?(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
_LANG_URL =
r'http://uk.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
_LOGIN_URL = 'http://www.youtube.com/signup?next=/&gl=US&hl=en'
_AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
_NETRC_MACHINE = 'youtube'
@staticmethod
def suitable(url):
return (re.match(YoutubeIE._VALID_URL, url) is not None)
def report_lang(self):
"""Report attempt to set language."""
self.to_stdout(u'[youtube] Setting language')
def report_login(self):
"""Report attempt to log in."""
self.to_stdout(u'[youtube] Logging in')
def report_age_confirmation(self):
"""Report attempt to confirm age."""
self.to_stdout(u'[youtube] Confirming age')
def _real_initialize(self):
if self._downloader is None:
return
username = None
password = None
downloader_params = self._downloader.get_params()
# No authentication to be performed
if username is None:
return
# Set language
request = urllib2.Request(self._LOGIN_URL, None, std_headers)
try:
self.report_lang()
urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
self.to_stderr(u'WARNING: unable to set language: %s' % str(err))
return
# Log in
login_form = {
'current_form': 'loginForm',
'next': '/',
'action_login': 'Log In',
'username': username,
'password': password,
}
request = urllib2.Request(self._LOGIN_URL,
urllib.urlencode(login_form), std_headers)
try:
self.report_login()
login_results = urllib2.urlopen(request).read()
if re.search(r'(?i)<form[^>]* name="loginForm"', login_results)
is not None:
self.to_stderr(u'WARNING: unable to log in: bad username or
password')
return
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
self.to_stderr(u'WARNING: unable to log in: %s' % str(err))
return
# Confirm age
age_form = {
'next_url': '/',
'action_confirm': 'Confirm',
}
request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form),
std_headers)
try:
self.report_age_confirmation()
age_results = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
self.to_stderr(u'ERROR: unable to confirm age: %s' % str(err))
return
# Extension
video_extension = {'18': 'mp4', '17': '3gp'}.get(format_param, 'flv')
# "t" param
mobj = re.search(r', "t": "([^"]+)"', video_webpage)
if mobj is None:
self.to_stderr(u'ERROR: unable to extract "t" parameter')
return [None]
video_real_url = 'http://www.youtube.com/get_video?video_id=%s&t=%s' %
(video_id, mobj.group(1))
if format_param is not None:
video_real_url = '%s&fmt=%s' % (video_real_url, format_param)
self.report_video_url(video_id, video_real_url)
# uploader
mobj = re.search(r"var watchUsername = '([^']+)';", video_webpage)
if mobj is None:
self.to_stderr(u'ERROR: unable to extract uploader nickname')
return [None]
video_uploader = mobj.group(1)
# title
mobj = re.search(r'(?im)<title>YouTube - ([^<]*)</title>',
video_webpage)
if mobj is None:
self.to_stderr(u'ERROR: unable to extract video title')
return [None]
video_title = mobj.group(1).decode('utf-8')
video_title = re.sub(ur'(?u)&(.+?);', lambda x:
unichr(htmlentitydefs.name2codepoint[x.group(1)]), video_title)
video_title = video_title.replace(os.sep, u'%')
# simplified title
simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_',
video_title)
simple_title = simple_title.strip(ur'_')
# Return information
return [{
'id': video_id.decode('utf-8'),
'url': video_real_url.decode('utf-8'),
'uploader': video_uploader.decode('utf-8'),
'title': video_title,
'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
}]
class MetacafeIE(InfoExtractor):
"""Information Extractor for metacafe.com."""
_VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
_DISCLAIMER = 'http://www.metacafe.com/family_filter/'
_youtube_ie = None
@staticmethod
def suitable(url):
return (re.match(MetacafeIE._VALID_URL, url) is not None)
def report_disclaimer(self):
"""Report disclaimer retrieval."""
self.to_stdout(u'[metacafe] Retrieving disclaimer')
def report_age_confirmation(self):
"""Report attempt to confirm age."""
self.to_stdout(u'[metacafe] Confirming age')
def _real_initialize(self):
# Retrieve disclaimer
request = urllib2.Request(self._DISCLAIMER, None, std_headers)
try:
self.report_disclaimer()
disclaimer = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
self.to_stderr(u'ERROR: unable to retrieve disclaimer: %s' %
str(err))
return
# Confirm age
disclaimer_form = {
'filters': '0',
'submit': "Continue - I'm over 18",
}
request = urllib2.Request('http://www.metacafe.com/',
urllib.urlencode(disclaimer_form), std_headers)
try:
self.report_age_confirmation()
disclaimer = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
self.to_stderr(u'ERROR: unable to confirm age: %s' % str(err))
return
video_id = mobj.group(1)
simple_title = mobj.group(2).decode('utf-8')
video_extension = 'flv'
# Return information
return [{
'id': video_id.decode('utf-8'),
'url': video_url.decode('utf-8'),
'uploader': video_uploader.decode('utf-8'),
'title': video_title,
'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
}]
class YoutubeSearchIE(InfoExtractor):
"""Information Extractor for YouTube search queries."""
_VALID_QUERY = r'ytsearch(\d+|all)?:[\s\S]+'
_TEMPLATE_URL =
'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
_VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
_MORE_PAGES_INDICATOR = r'>Next</a>'
_youtube_ie = None
@staticmethod
def suitable(url):
return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
def _real_initialize(self):
self._youtube_ie.initialize()
video_ids = []
already_seen = set()
pagenum = 1
while True:
self.report_download_page(query, pagenum)
result_url = self._TEMPLATE_URL % (urllib.quote_plus(query),
pagenum)
request = urllib2.Request(result_url, None, std_headers)
try:
page = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error),
err:
self.to_stderr(u'ERROR: unable to download webpage: %s' %
str(err))
return [None]
information.extend(self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' %
id))
return information
information.extend(self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' %
id))
return information
pagenum = pagenum + 1
class YoutubePlaylistIE(InfoExtractor):
"""Information Extractor for YouTube playlists."""
_VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/view_play_list\?p=(.+)'
_TEMPLATE_URL =
'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
_VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
_MORE_PAGES_INDICATOR = r'/view_play_list?p=%s&page=%s'
_youtube_ie = None
@staticmethod
def suitable(url):
return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
def _real_initialize(self):
self._youtube_ie.initialize()
while True:
self.report_download_page(playlist_id, pagenum)
request = urllib2.Request(self._TEMPLATE_URL % (playlist_id,
pagenum), None, std_headers)
try:
page = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error),
err:
self.to_stderr(u'ERROR: unable to download webpage: %s' %
str(err))
return [None]
information = []
for id in video_ids:
information.extend(self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' %
id))
return information
class PostProcessor(object):
"""Post Processor class.
The chain will be stopped if one of them ever returns None or the end
of the chain is reached.
_downloader = None
# General configuration
urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor()))
socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last
words)
# Information extractors
youtube_ie = YoutubeIE()
metacafe_ie = MetacafeIE(youtube_ie)
youtube_pl_ie = YoutubePlaylistIE(youtube_ie)
youtube_search_ie = YoutubeSearchIE(youtube_ie)
# File downloader
charset = locale.getdefaultlocale()[1]
if charset is None:
charset = 'ascii'
fd = FileDownloader({
'usenetrc': opts.usenetrc,
'username': opts.username,
'password': opts.password,
'quiet': (opts.quiet or opts.geturl or opts.gettitle),
'forceurl': opts.geturl,
'forcetitle': opts.gettitle,
'simulate': (opts.simulate or opts.geturl or opts.gettitle),
'format': opts.format,
'outtmpl': ((opts.outtmpl is not None and
opts.outtmpl.decode(charset))
or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
or u'%(id)s.%(ext)s'),
'ignoreerrors': opts.ignoreerrors,
'ratelimit': opts.ratelimit,
'nooverwrites': opts.nooverwrites,
})
fd.add_info_extractor(youtube_search_ie)
fd.add_info_extractor(youtube_pl_ie)
fd.add_info_extractor(metacafe_ie)
fd.add_info_extractor(youtube_ie)
retcode = fd.download(all_urls)
sys.exit(retcode)
except DownloadError:
sys.exit(1)
except SameFileError:
sys.exit(u'ERROR: fixed output name but more than one file to
download')
except KeyboardInterrupt:
sys.exit(u'\nERROR: Interrupted by user')