First step , install requests-html
- upgrade pip
pip install --upgrade pip
- upgrade urllib3
sudo python3 -m pip install urllib3 --upgrade
- install requests-html
sudo python3 -m pip install requests-html
The first 1.1 Step , For the project , install requests-html
- modify setup.py file ,
add to
install_requires=[
'requests-html',
],
- modify launch.json
add to
"pythonPath": "/usr/bin/python3"
- Command line , install
sudo python3 -m setup install
- python In file , Use
from requests_html import HTMLSession
The first 2 Step , Continue to use youtube - dl
- Create a new information extraction class
class XxxIE(InfoExtractor):
- Set up matching rules
_VALID_URL = r'https?://(?:www\.|m\.)?xxx\.com.+posts?.+'
Corresponding source code
After starting ,
- Go ahead YoutubeDL.py Of documents
def extract_info(self, url, download=True, ie_key=None, extra_info={},
process=True, force_generic_extractor=False):
# ...
for ie in ies:
if not ie.suitable(url):
continue
# ...
- To walk again extractor Under the folder common.py Of documents
@classmethod
def suitable(cls, url):
if '_VALID_URL_RE' not in cls.__dict__:
cls._VALID_URL_RE = re.compile(cls._VALID_URL)
# ...
2.1 The rest to
class XxxIE(InfoExtractor):
- First in extractor Under folder
extractors.py
Let's quote
- XxxIE Download and crawl , that will do
from requests_html import HTML
class XxxIE(InfoExtractor):
_GEO_COUNTRIES = ['CN']
IE_NAME = 'xxx: blog'
IE_DESC = 'wo qu'
_VALID_URL = r'https?://(?:www\.|m\.)?xxx\.com.+posts?.+'
_TEMPLATE_URL = '%s://www.xxx.com/%s/posts/%s/'
_LIST_VIDEO_RE = r'<a[^>]+?href="(?P<url>/%s/sound/(?P<id>\d+)/?)"[^>]+?title="(?P<title>[^>]+)">'
def _real_extract(self, url):
scheme = 'https' if url.startswith('https') else 'http'
print("start ya yay ya")
print("\n\n\n")
self.downloadX(url, 1)
small = list(range(2, 20))
for index in small:
# ?page=2
src = url + "?page=" + str(index)
self.downloadX(src, index)
print("\n\n\n")
return {}
def downloadX(self, src, index):
audio_id = 123456
webpage = self._download_webpage(src, audio_id,
note='Download sound page for %s' % audio_id,
errnote='Unable to get sound page')
html = HTML(html=webpage)
# print(webpage)
jsonElement = html.find('#js-initialData')
jsonInfo = jsonElement[0].text
jsonX = json.loads(jsonInfo)
dic = jsonX['initialState']['entities']['articles']
print("page: " + str(index) + " : ")
for k, v in dic.items():
# pprint(v)
t = v.get('title')
print(t)
print("\n")