release 2014.08.24.3

youtube_dl/extractor/__init__.py

@@ -184,6 +184,7 @@ from .malemotion import MalemotionIE
 from .mdr import MDRIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
+from .ministrygrid import MinistryGridIE
 from .mit import TechTVMITIE, MITIE, OCWMITIE
 from .mitele import MiTeleIE
 from .mixcloud import MixcloudIE
@@ -390,6 +391,7 @@ from .vuclip import VuClipIE
 from .vulture import VultureIE
 from .washingtonpost import WashingtonPostIE
 from .wat import WatIE
+from .wayofthemaster import WayOfTheMasterIE
 from .wdr import (
     WDRIE,
     WDRMobileIE,

youtube_dl/extractor/ebaumsworld.py

@@ -1,19 +1,21 @@
+from __future__ import unicode_literals
+
 import re
 
 from .common import InfoExtractor
-from ..utils import determine_ext
 
 
 class EbaumsWorldIE(InfoExtractor):
     _VALID_URL = r'https?://www\.ebaumsworld\.com/video/watch/(?P<id>\d+)'
 
     _TEST = {
-        u'url': u'http://www.ebaumsworld.com/video/watch/83367677/',
-        u'file': u'83367677.mp4',
-        u'info_dict': {
-            u'title': u'A Giant Python Opens The Door',
-            u'description': u'This is how nightmares start...',
-            u'uploader': u'jihadpizza',
+        'url': 'http://www.ebaumsworld.com/video/watch/83367677/',
+        'info_dict': {
+            'id': '83367677',
+            'ext': 'mp4',
+            'title': 'A Giant Python Opens The Door',
+            'description': 'This is how nightmares start...',
+            'uploader': 'jihadpizza',
         },
     }
 
@@ -28,7 +30,6 @@ class EbaumsWorldIE(InfoExtractor):
             'id': video_id,
             'title': config.find('title').text,
             'url': video_url,
-            'ext': determine_ext(video_url),
             'description': config.find('description').text,
             'thumbnail': config.find('image').text,
             'uploader': config.find('username').text,

youtube_dl/extractor/generic.py

@@ -22,6 +22,7 @@ from ..utils import (
     smuggle_url,
     unescapeHTML,
     unified_strdate,
+    unsmuggle_url,
     url_basename,
 )
 from .brightcove import BrightcoveIE
@@ -330,6 +331,18 @@ class GenericIE(InfoExtractor):
             'info_dict': {
                 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
             }
+        },
+        # Flowplayer
+        {
+            'url': 'http://www.handjobhub.com/video/busty-blonde-siri-tit-fuck-while-wank-6313.html',
+            'md5': '9d65602bf31c6e20014319c7d07fba27',
+            'info_dict': {
+                'id': '5123ea6d5e5a7',
+                'ext': 'mp4',
+                'age_limit': 18,
+                'uploader': 'www.handjobhub.com',
+                'title': 'Busty Blonde Siri Tit Fuck While Wank at Handjob Hub',
+            }
         }
     ]
 
@@ -487,7 +500,14 @@ class GenericIE(InfoExtractor):
             else:
                 assert ':' in default_search
                 return self.url_result(default_search + url)
-        video_id = os.path.splitext(url.rstrip('/').split('/')[-1])[0]
+
+        url, smuggled_data = unsmuggle_url(url)
+        force_videoid = None
+        if smuggled_data and 'force_videoid' in smuggled_data:
+            force_videoid = smuggled_data['force_videoid']
+            video_id = force_videoid
+        else:
+            video_id = os.path.splitext(url.rstrip('/').split('/')[-1])[0]
 
         self.to_screen('%s: Requesting header' % video_id)
 
@@ -498,6 +518,9 @@ class GenericIE(InfoExtractor):
             new_url = response.geturl()
             if url != new_url:
                 self.report_following_redirect(new_url)
+                if force_videoid:
+                    new_url = smuggle_url(
+                        new_url, {'force_videoid': force_videoid})
                 return self.url_result(new_url)
 
             # Check for direct link to a video
@@ -559,6 +582,16 @@ class GenericIE(InfoExtractor):
             r'(?s)<title>(.*?)</title>', webpage, 'video title',
             default='video')
 
+        # Try to detect age limit automatically
+        age_limit = self._rta_search(webpage)
+        # And then there are the jokers who advertise that they use RTA,
+        # but actually don't.
+        AGE_LIMIT_MARKERS = [
+            r'Proudly Labeled <a href="http://www.rtalabel.org/" title="Restricted to Adults">RTA</a>',
+        ]
+        if any(re.search(marker, webpage) for marker in AGE_LIMIT_MARKERS):
+            age_limit = 18
+
         # video uploader is domain name
         video_uploader = self._search_regex(
             r'^(?:https?://)?([^/]*)/.*', url, 'video uploader')
@@ -822,6 +855,15 @@ class GenericIE(InfoExtractor):
         if not found:
             # Broaden the findall a little bit: JWPlayer JS loader
             found = re.findall(r'[^A-Za-z0-9]?file["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage)
+        if not found:
+            # Flow player
+            found = re.findall(r'''(?xs)
+                flowplayer\("[^"]+",\s*
+                    \{[^}]+?\}\s*,
+                    \s*{[^}]+? ["']?clip["']?\s*:\s*\{\s*
+                        ["']?url["']?\s*:\s*["']([^"']+)["']
+            ''', webpage)
+            assert found
         if not found:
             # Try to find twitter cards info
             found = re.findall(r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage)
@@ -873,6 +915,7 @@ class GenericIE(InfoExtractor):
                 'url': video_url,
                 'uploader': video_uploader,
                 'title': video_title,
+                'age_limit': age_limit,
             })
 
         if len(entries) == 1:

youtube_dl/extractor/ministrygrid.py

@@ -0,0 +1,57 @@
+from __future__ import unicode_literals
+
+import json
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    smuggle_url,
+)
+
+
+class MinistryGridIE(InfoExtractor):
+    _VALID_URL = r'https?://www\.ministrygrid.com/([^/?#]*/)*(?P<id>[^/#?]+)/?(?:$|[?#])'
+
+    _TEST = {
+        'url': 'http://www.ministrygrid.com/training-viewer/-/training/t4g-2014-conference/the-gospel-by-numbers-4/the-gospel-by-numbers',
+        'md5': '844be0d2a1340422759c2a9101bab017',
+        'info_dict': {
+            'id': '3453494717001',
+            'ext': 'mp4',
+            'title': 'The Gospel by Numbers',
+            'description': 'Coming soon from T4G 2014!',
+            'uploader': 'LifeWay Christian Resources (MG)',
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        webpage = self._download_webpage(url, video_id)
+        portlets_json = self._search_regex(
+            r'Liferay\.Portlet\.list=(\[.+?\])', webpage, 'portlet list')
+        portlets = json.loads(portlets_json)
+        pl_id = self._search_regex(
+            r'<!--\s*p_l_id - ([0-9]+)<br>', webpage, 'p_l_id')
+
+        for i, portlet in enumerate(portlets):
+            portlet_url = 'http://www.ministrygrid.com/c/portal/render_portlet?p_l_id=%s&p_p_id=%s' % (pl_id, portlet)
+            portlet_code = self._download_webpage(
+                portlet_url, video_id,
+                note='Looking in portlet %s (%d/%d)' % (portlet, i + 1, len(portlets)),
+                fatal=False)
+            video_iframe_url = self._search_regex(
+                r'<iframe.*?src="([^"]+)"', portlet_code, 'video iframe',
+                default=None)
+            if video_iframe_url:
+                surl = smuggle_url(
+                    video_iframe_url, {'force_videoid': video_id})
+                return {
+                    '_type': 'url',
+                    'id': video_id,
+                    'url': surl,
+                }
+
+        raise ExtractorError('Could not find video iframe in any portlets')

youtube_dl/extractor/wayofthemaster.py

@@ -0,0 +1,52 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class WayOfTheMasterIE(InfoExtractor):
+    _VALID_URL = r'https?://www\.wayofthemaster\.com/([^/?#]*/)*(?P<id>[^/?#]+)\.s?html(?:$|[?#])'
+
+    _TEST = {
+        'url': 'http://www.wayofthemaster.com/hbks.shtml',
+        'md5': '5316b57487ada8480606a93cb3d18d24',
+        'info_dict': {
+            'id': 'hbks',
+            'ext': 'mp4',
+            'title': 'Intelligent Design vs. Evolution',
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._search_regex(
+            r'<img src="images/title_[^"]+".*?alt="([^"]+)"',
+            webpage, 'title', default=None)
+        if title is None:
+            title = self._html_search_regex(
+                r'<title>(.*?)</title>', webpage, 'page title')
+
+        url_base = self._search_regex(
+            r'<param\s+name="?movie"?\s+value=".*?/wotm_videoplayer_highlow[0-9]*\.swf\?vid=([^"]+)"',
+            webpage, 'URL base')
+        formats = [{
+            'format_id': 'low',
+            'quality': 1,
+            'url': url_base + '_low.mp4',
+        }, {
+            'format_id': 'high',
+            'quality': 2,
+            'url': url_base + '_high.mp4',
+        }]
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+        }

youtube_dl/version.py

@@ -1,2 +1,2 @@
 
-__version__ = '2014.08.24.1'
+__version__ = '2014.08.24.3'