เว็บบอร์ด PHP เว็บส่งเสริมการเรียนรู้ Hosting CRM ERP Server Programming ถาม-ตอบปัญหา

โค้ด: เลือกทั้งหมด

from urllib.request import urlopen, Request

req = Request(URL, headers={'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'})
html = str(urlopen(req).read())

โค้ด: เลือกทั้งหมด

from flask import  Flask, render_template, request
import socket, time
from urllib.parse import urlparse
from urllib.request import urlopen
from bs4 import BeautifulSoup
import psycopg2 as p
import re

@app.route('/url' , methods=['POST', 'GET'])
def url():
    if request.method == 'POST':
        URL = request.form['url']
        print (URL, 'timing:')
        try:
            con = p.connect(host='127.0.0.1', database='test', user='postgres', password='0845770809')
            cur = con.cursor()
            urlinfo = urlparse(URL)

            start = time.time()
            ip = socket.gethostbyname(urlinfo.netloc)
            dns_tm = time.time() - start
            print('DNS:\t\t{:.3f} seconds'.format(dns_tm))

            start = time.time()
            html = str(urlopen(URL).read())
            load_tm = time.time() - start
            print('load:\t\t{:.3f} seconds'.format(load_tm))
            print('w/o DNS:\t{:.3f} seconds'.format(load_tm - dns_tm))

            sql = """INSERT INTO ur (url,dns,load_ur,wo_dns)
                        VALUES ('%s','%s','%s','%s') RETURNING id;""" % (URL,dns_tm,load_tm,load_tm - dns_tm)
            sql = sql.encode('utf-8')
            cur.execute(sql)
            id1 = str(cur.fetchone()[0])
            con.commit()
            soup = BeautifulSoup(html,"html.parser")

            for link in soup.find_all('a' ,href=re.compile('^https')):

                # time.sleep(1)
                print(link.get('href'))
                urlinfo2 = urlparse(link.get('href'))

                start = time.time()
                ip = socket.gethostbyname(urlinfo2.netloc)
                dns_tm2 = time.time() - start
                print('DNS:\t\t{:.3f} seconds'.format(dns_tm2))

                start = time.time()
                html = str(urlopen(link.get('href')).read())
                load_tm2 = time.time() - start
                print('load:\t\t{:.3f} seconds'.format(load_tm2))
                print('w/o DNS:\t{:.3f} seconds'.format(load_tm2 - dns_tm2))


                sql = """INSERT INTO des_ur (url,dns,load_ur,wo_dns,ur_id)
                                            VALUES ('%s','%s','%s','%s','%s')""" % (
                link.get('href'), dns_tm, load_tm, load_tm - dns_tm,id1)
                sql = sql.encode('utf-8')
                cur.execute(sql)
                con.commit()


            return html

        except p.DatabaseError:
            print('Error %s' % p.DatabaseError)

โค้ด: เลือกทั้งหมด

html = urlopen(URL).read()

โค้ด: เลือกทั้งหมด

for link in soup.find_all('a' ,href=re.compile('^https')):

โค้ด: เลือกทั้งหมด

html = urlopen(link.get('href')).read()

โค้ด: เลือกทั้งหมด

html = urlopen(URL).read()