bslathi19/buildroot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

support/scripts/pkg-stats: use aiohttp for upstream URL checking

Browse Source 
This commit reworks the code that checks if the upstream URL of each
package (specified by its Config.in file) using the aiohttp
module. This makes the implementation much more elegant, and avoids
the problematic multiprocessing Pool which is causing issues in some
situations.

Suggested-by: Titouan Christophe <titouan.christophe@railnova.eu>
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
This commit is contained in:
Thomas Petazzoni
2020-08-08 20:08:24 +02:00
parent d28b2bc481
commit c8ba6aa133
1 changed files with 25 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
support/scripts/pkg-stats
View File

@@ -25,14 +25,13 @@ import os
from collections import defaultdict from collections import defaultdict
import re import re
import subprocess import subprocess
import requests # URL checking import requests # NVD database download
import json import json
import ijson import ijson
import distutils.version import distutils.version
import time import time
import gzip import gzip
import sys import sys
from multiprocessing import Pool
sys.path.append('utils/') sys.path.append('utils/')
from getdeveloperlib import parse_developers # noqa: E402 from getdeveloperlib import parse_developers # noqa: E402
@@ -499,26 +498,30 @@ def package_init_make_info():
Package.all_ignored_cves[pkgvar] = value.split() Package.all_ignored_cves[pkgvar] = value.split()
def check_url_status_worker(url, url_status): async def check_url_status(session, pkg, retry=True):
if url_status[0] == 'ok': try:
try: async with session.get(pkg.url) as resp:
url_status_code = requests.head(url, timeout=30).status_code if resp.status >= 400:
if url_status_code >= 400: pkg.status['url'] = ("error", "invalid {}".format(resp.status))
return ("error", "invalid {}".format(url_status_code)) return
except requests.exceptions.RequestException: except (aiohttp.ClientError, asyncio.TimeoutError):
return ("error", "invalid (err)") if retry:
return ("ok", "valid") return await check_url_status(session, pkg, retry=False)
return url_status else:
pkg.status['url'] = ("error", "invalid (err)")
return
pkg.status['url'] = ("ok", "valid")
def check_package_urls(packages): async def check_package_urls(packages):
pool = Pool(processes=64) tasks = []
for pkg in packages: connector = aiohttp.TCPConnector(limit_per_host=5)
pkg.url_worker = pool.apply_async(check_url_status_worker, (pkg.url, pkg.status['url'])) async with aiohttp.ClientSession(connector=connector, trust_env=True) as sess:
for pkg in packages: packages = [p for p in packages if p.status['url'][0] == 'ok']
pkg.status['url'] = pkg.url_worker.get(timeout=3600) for pkg in packages:
del pkg.url_worker tasks.append(check_url_status(sess, pkg))
pool.terminate() await asyncio.wait(tasks)
def check_package_latest_version_set_status(pkg, status, version, identifier): def check_package_latest_version_set_status(pkg, status, version, identifier):
@@ -1068,7 +1071,8 @@ def __main__():
pkg.set_url() pkg.set_url()
pkg.set_developers(developers) pkg.set_developers(developers)
print("Checking URL status") print("Checking URL status")
check_package_urls(packages) loop = asyncio.get_event_loop()
loop.run_until_complete(check_package_urls(packages))
print("Getting latest versions ...") print("Getting latest versions ...")
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
loop.run_until_complete(check_package_latest_version(packages)) loop.run_until_complete(check_package_latest_version(packages))