First step , install requests-html

• upgrade pip

pip install --upgrade pip

• upgrade urllib3

sudo python3 -m pip install urllib3 --upgrade

• install requests-html

 sudo python3 -m pip install requests-html

The first 1.1 Step , For the project , install requests-html

• modify setup.py file ,

add to

install_requires=[
'requests-html',
],

• modify launch.json

add to

 "pythonPath": "/usr/bin/python3"

• Command line , install

sudo python3 -m setup install

• python In file , Use

from requests_html import HTMLSession

The first 2 Step , Continue to use youtube - dl

• Create a new information extraction class

 class XxxIE(InfoExtractor):

• Set up matching rules

_VALID_URL = r'https?://(?:www\.|m\.)?xxx\.com.+posts?.+'

Corresponding source code

After starting ,

• Go ahead YoutubeDL.py Of documents

 def extract_info(self, url, download=True, ie_key=None, extra_info={},
process=True, force_generic_extractor=False):
# ...
for ie in ies:
if not ie.suitable(url):
continue
# ...

• To walk again extractor Under the folder common.py Of documents

@classmethod
def suitable(cls, url):
if '_VALID_URL_RE' not in cls.__dict__:
cls._VALID_URL_RE = re.compile(cls._VALID_URL)
# ...

2.1 The rest to

class XxxIE(InfoExtractor):

• First in extractor Under folder

extractors.py

Let's quote

• XxxIE Download and crawl , that will do

from requests_html import HTML
class XxxIE(InfoExtractor):
_GEO_COUNTRIES = ['CN']
IE_NAME = 'xxx: blog'
IE_DESC = 'wo qu'
_VALID_URL = r'https?://(?:www\.|m\.)?xxx\.com.+posts?.+'
_TEMPLATE_URL = '%s://www.xxx.com/%s/posts/%s/'
_LIST_VIDEO_RE = r'<a[^>]+?href="(?P<url>/%s/sound/(?P<id>\d+)/?)"[^>]+?title="(?P<title>[^>]+)">'
def _real_extract(self, url):
scheme = 'https' if url.startswith('https') else 'http'
print("start ya yay ya")
print("\n\n\n")
self.downloadX(url, 1)
small = list(range(2, 20))
for index in small:
# ?page=2
src = url + "?page=" + str(index)
self.downloadX(src, index)
print("\n\n\n")
return {}
def downloadX(self, src, index):
audio_id = 123456
webpage = self._download_webpage(src, audio_id,
note='Download sound page for %s' % audio_id,
errnote='Unable to get sound page')
html = HTML(html=webpage)
# print(webpage)
jsonElement = html.find('#js-initialData')
jsonInfo = jsonElement[0].text
jsonX = json.loads(jsonInfo)
dic = jsonX['initialState']['entities']['articles']
print("page: " + str(index) + " : ")
for k, v in dic.items():
# pprint(v)
t = v.get('title')
print(t)
print("\n")

Code link